diff --git a/lib/entropy/client/interfaces/loaders.py b/lib/entropy/client/interfaces/loaders.py index 8ac122ccf..2424efeb7 100644 --- a/lib/entropy/client/interfaces/loaders.py +++ b/lib/entropy/client/interfaces/loaders.py @@ -9,6 +9,8 @@ B{Entropy Package Manager Client Instance Loaders Interface}. """ +import warnings + from entropy.spm.plugins.factory import get_default_instance as get_spm, \ get_default_class as get_spm_default_class @@ -25,12 +27,15 @@ class LoadersMixin: # during complete system upgrades from entropy.client.interfaces.trigger import Trigger from entropy.client.interfaces.repository import Repository - from entropy.client.interfaces.package import Package + from entropy.client.interfaces.package import \ + PackageActionFactory, PackageActionFactoryWrapper from entropy.client.interfaces.sets import Sets from entropy.client.misc import ConfigurationUpdates from entropy.client.services.interfaces import \ ClientWebServiceFactory, RepositoryWebServiceFactory - self.__package_loader = Package + + self.__package_factory = PackageActionFactory + self.__package_loader = PackageActionFactoryWrapper self.__repository_loader = Repository self.__trigger_loader = Trigger self.__sets_loader = Sets @@ -141,14 +146,16 @@ class LoadersMixin: def Package(self): """ Load Entropy Package instance object - - @return: - @rtype: entropy.client.interfaces.package.Package """ + warnings.warn("deprecated, use PackageActionFactory", + DeprecationWarning) return self.__package_loader(self) + def PackageActionFactory(self): """ + Load Entropy PackageActionFactory instance object """ + return self.__package_factory(self) def ConfigurationUpdates(self): """ diff --git a/lib/entropy/client/interfaces/package/__init__.py b/lib/entropy/client/interfaces/package/__init__.py index fd52127cc..e72f77ac3 100644 --- a/lib/entropy/client/interfaces/package/__init__.py +++ b/lib/entropy/client/interfaces/package/__init__.py @@ -16,11 +16,12 @@ import stat import shutil import time import codecs +import collections from entropy.const import etpConst, const_setup_perms, const_mkstemp, \ const_isunicode, const_convert_to_unicode, const_debug_write, \ const_debug_enabled, const_convert_to_rawstring, const_is_python3 -from entropy.exceptions import PermissionDenied, SPMError +from entropy.exceptions import EntropyException, PermissionDenied, SPMError from entropy.i18n import _, ngettext from entropy.output import brown, blue, bold, darkgreen, \ darkblue, red, purple, darkred, teal @@ -34,4985 +35,166 @@ import entropy.tools from . import _content as Content -class Package(object): +from .actions.action import PackageAction +from .actions.config import _PackageConfigAction +from .actions.fetch import _PackageFetchAction +from .actions.install import _PackageInstallAction +from .actions.multifetch import _PackageMultiFetchAction +from .actions.remove import _PackageRemoveAction +from .actions.source import _PackageSourceAction + + +class PackageActionFactory(object): + """ + Package action factory. + + This factory object returns PackageAction instances + that can be used to perform a specific activity (like, + for instance, the removal, installation or download of a + single package). + + Example code: + + >>> factory = PackageActionFactory(entropy_client) + >>> install = PackageActionFactory.INSTALL_ACTION + >>> obj = factory.get(install, (123, "sabayon-weekly")) + >>> exit_status = obj.start() + >>> obj.finalize() + + You can reuse the factory as many times as you want. + If you pass an invalid action string, InvalidAction() will be raised. + The PackageAction objects (well, their methods) are not thread-safe. + """ + + class InvalidAction(EntropyException): + """ + Raised when the factory is passed an invalid action string. + """ + + INSTALL_ACTION = _PackageInstallAction.NAME + REMOVE_ACTION = _PackageRemoveAction.NAME + CONFIG_ACTION = _PackageConfigAction.NAME + FETCH_ACTION = _PackageFetchAction.NAME + MULTI_FETCH_ACTION = _PackageMultiFetchAction.NAME + SOURCE_ACTION = _PackageSourceAction.NAME def __init__(self, entropy_client): + """ + Object constructor. + @param entropy_client: a valid Client instance. + @type entropy_client: entropy.client.interfaces.Client + """ self._entropy = entropy_client - self._settings = SystemSettings() + self._actions = { + self.SOURCE_ACTION: _PackageSourceAction, + self.FETCH_ACTION: _PackageFetchAction, + self.MULTI_FETCH_ACTION: _PackageMultiFetchAction, + self.REMOVE_ACTION: _PackageRemoveAction, + self.INSTALL_ACTION: _PackageInstallAction, + self.CONFIG_ACTION: _PackageConfigAction, + } + self._action_instance = None + + def supported_actions(self): + """ + Return a list of supported actions. + + @return: a list of supported actions + @rtype: list + """ + return sorted(self._actions.keys()) + + def get(self, action, package_match, opts = None): + """ + Return the PackageAction instance associated with the given action. + + @param action: the action string, see supported_actions() + @type action: string + @param package_match: an Entropy package match tuple + (package_id, repository_id) + @type package_match: tuple + @keyword opts: metadata options to pass to the PackageAction + instance + @type opts: dict + """ + action_class = self._actions.get(action) + if action_class is None: + raise PackageActionFactory.InvalidAction( + "action does not exist") + return action_class(self._entropy, package_match, opts = opts) + + +class PackageActionFactoryWrapper(PackageActionFactory): + """ + Compatibility class that provides the old Entropy Package() interface. + It will be dropped at the end of 2014 (hopefully). + """ + + def __init__(self, entropy_client): + super(PackageActionFactoryWrapper, self).__init__(entropy_client) + self._action_instance = None self.pkgmeta = {} - self.__prepared = False - self._package_match = () - self._valid_actions = ("source", "fetch", "multi_fetch", "remove", - "remove_conflict", "install", "config" - ) - self._action = None - self._xterm_title = '' - - def __repr__(self): - return "<%s.Package at %s | metadata: %s | action: %s, prepared: %s>" \ - % (__name__, hex(id(self)), self.pkgmeta, self._action, - self.__prepared) - - def __str__(self): - return repr(self) - - def __unicode__(self): - return unicode(repr(self)) - - def kill(self): - - # remove temporary content files - # created by __generate_content_file() - content_files = self.pkgmeta.get( - '__content_files__', []) - for content_file in content_files: - try: - os.remove(content_file) - except (OSError, IOError): - pass - - self.pkgmeta.clear() - - self._package_match = () - self._valid_actions = () - self._action = None - self.__prepared = False - - def _error_on_prepared(self): - if self.__prepared: - mytxt = _("Already prepared") - raise PermissionDenied("PermissionDenied: %s" % (mytxt,)) - - def _error_on_not_prepared(self): - if not self.__prepared: - mytxt = _("Not yet prepared") - raise PermissionDenied("PermissionDenied: %s" % (mytxt,)) - - def _check_action_validity(self, action): - if action not in self._valid_actions: - raise AttributeError("Action must be in %s" % ( - self._valid_actions,)) - - _INFO_EXTS = ( - const_convert_to_unicode(".gz"), - const_convert_to_unicode(".bz2") - ) - - def _get_info_directories(self): - """ - Return a list of `info` directories as declared in the - INFOPATH and INFODIR environment variable. - """ - info_dirs = os.getenv("INFOPATH", "").split(":") - info_dirs += os.getenv("INFODIR", "").split(":") - info_dirs = [const_convert_to_unicode( - os.path.normpath(x)) for x in info_dirs] - info_dirs.sort() - return info_dirs - - @staticmethod - def get_standard_fetch_disk_path(download): - """ - Return standard path where package is going to be downloaded. - "download" argument passed must come from - EntropyRepository.retrieveDownloadURL() - """ - return os.path.join(etpConst['entropypackagesworkdir'], download) - - def __get_fetch_disk_path(self, download): - """ - Return proper Entropy package store path - """ - if 'fetch_path' in self.pkgmeta: - # only supported by fetch action, multifetch also unsupported - pkg_disk_path = os.path.join(self.pkgmeta['fetch_path'], - os.path.basename(download)) - else: - pkg_disk_path = Package.get_standard_fetch_disk_path(download) - return pkg_disk_path - - def __escape_path(self, path): - """ - Some applications (like ld) don't like ":" in path, others just don't - escape paths at all. So, it's better to avoid to use field separators - in path. - """ - path = path.replace(":", "_") - path = path.replace("~", "_") - return path - - def __check_pkg_path_download(self, download, checksum = None): - # is the file available - pkg_path = self.__get_fetch_disk_path(download) - if os.path.isfile(pkg_path): - - if checksum is None: - return 0 - # check digest - md5res = entropy.tools.compare_md5(pkg_path, checksum) - if md5res: - return 0 - return -2 - - return -1 - - def __fetch_files(self, url_data_list, checksum = True, resume = True): - - def _generate_checksum_map(url_data): - if not checksum: - return {} - ck_map = {} - ck_map_id = 0 - for _pkg_id, _repo, _url, _dest_path, cksum in url_data: - ck_map_id += 1 - if cksum is not None: - ck_map[ck_map_id] = cksum - return ck_map - - fetch_abort_function = self.pkgmeta.get('fetch_abort_function') - # avoid tainting data pointed by url_data_list - url_data = url_data_list[:] - diff_map = {} - - # setup directories - for pkg_id, repo, url, dest_path, _cksum in url_data: - dest_dir = os.path.dirname(dest_path) - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir, 0o775) - const_setup_perms(dest_dir, etpConst['entropygid']) - - checksum_map = _generate_checksum_map(url_data) - fetched_url_data, data_transfer, abort = self.__try_edelta_multifetch( - url_data, resume) - if abort: - return -100, {}, 0 - - for url_data_item in fetched_url_data: - url_data.remove(url_data_item) - - # some packages haven't been downloaded using edelta - if url_data: - - url_path_list = [] - for pkg_id, repo, url, dest_path, _cksum in url_data: - url_path_list.append((url, dest_path,)) - self._setup_differential_download( - self._entropy._multiple_url_fetcher, url, resume, dest_path, - repo, pkg_id) - - # load class - fetch_intf = self._entropy._multiple_url_fetcher(url_path_list, - resume = resume, abort_check_func = fetch_abort_function, - url_fetcher_class = self._entropy._url_fetcher, - checksum = checksum) - try: - # make sure that we don't need to abort already - # doing the check here avoids timeouts - if fetch_abort_function != None: - fetch_abort_function() - - data = fetch_intf.download() - except KeyboardInterrupt: - return -100, {}, 0 - # update transfer rate information - data_transfer = fetch_intf.get_transfer_rate() - - checksum_map = _generate_checksum_map(url_data) - # if checksum_map is empty, it means that checksum == False - for ck_id in checksum_map: - orig_checksum = checksum_map.get(ck_id) - if orig_checksum != data.get(ck_id): - diff_map[url_path_list[ck_id-1][0]] = orig_checksum - - if diff_map: - defval = -1 - for key, val in tuple(diff_map.items()): - if val == UrlFetcher.GENERIC_FETCH_WARN: - diff_map[key] = -2 - elif val == UrlFetcher.TIMEOUT_FETCH_ERROR: - diff_map[key] = -4 - elif val == UrlFetcher.GENERIC_FETCH_ERROR: - diff_map[key] = -3 - elif val == -100: - defval = -100 - return defval, diff_map, data_transfer - - return 0, diff_map, data_transfer - - def _get_url_name(self, url): - """ - Given a mirror URL, returns a smaller string representing the URL name. - - @param url: URL string - @type url: string - @return: representative URL string - @rtype: string - """ - url_data = entropy.tools.spliturl(url) - url_name = url_data.netloc - url_scheme = url_data.scheme - if not url_scheme: - url_scheme = "unknown" - return "%s://%s" % (url_scheme, url_name,) - - def _download_packages(self, download_list, checksum = False): - - avail_data = self._settings['repositories']['available'] - excluded_data = self._settings['repositories']['excluded'] - - repo_uris = {} - for pkg_id, repo, fname, cksum, _signatures in download_list: - repo_db = self._entropy.open_repository(repo) - # grab original repo, if any and use it if available - # this is done in order to support "equo repo merge" feature - # allowing client-side repository package metadata moves. - original_repo = repo_db.getInstalledPackageRepository(pkg_id) - - if (original_repo != repo) and (original_repo not in avail_data) \ - and (original_repo is not None): - # build up a new uris list, at least try, hoping that - # repository is just shadowing original_repo - # for example: original_repo got copied to repository, without - # copying packages, which would be useless. like it happens - # with sabayon-weekly - uris = self.__build_uris_list(original_repo, repo) - else: - if original_repo in avail_data: - uris = avail_data[original_repo]['packages'][::-1] - uris += avail_data[repo]['packages'][::-1] - elif original_repo in excluded_data: - uris = excluded_data[original_repo]['packages'][::-1] - uris += avail_data[repo]['packages'][::-1] - else: - uris = avail_data[repo]['packages'][::-1] - - obj = repo_uris.setdefault(repo, []) - # append at the beginning - new_ones = [x for x in uris if x not in obj][::-1] - for new_obj in new_ones: - obj.insert(0, new_obj) - - - remaining = repo_uris.copy() - my_download_list = download_list[:] - mirror_status = StatusInterface() - - def get_best_mirror(repository): - try: - return remaining[repository][0] - except IndexError: - return None - - def update_download_list(down_list, failed_down): - newlist = [] - for pkg_id, repo, fname, cksum, signatures in down_list: - myuri = get_best_mirror(repo) - myuri = os.path.join(myuri, fname) - if myuri not in failed_down: - continue - newlist.append((pkg_id, repo, fname, cksum, signatures,)) - return newlist - - # return True: for failing, return False: for fine - def mirror_fail_check(repository, best_mirror): - # check if uri is sane - if not mirror_status.get_failing_mirror_status(best_mirror) >= 30: - return False - # set to 30 for convenience - mirror_status.set_failing_mirror_status(best_mirror, 30) - mirrorcount = repo_uris[repository].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - mytxt += blue(" %s: ") % (_("Mirror"),) - mytxt += red(self._get_url_name(best_mirror)) - mytxt += " - %s." % (_("maximum failure threshold reached"),) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = red(" ## ") - ) - - if mirror_status.get_failing_mirror_status(best_mirror) == 30: - mirror_status.add_failing_mirror(best_mirror, 45) - elif mirror_status.get_failing_mirror_status(best_mirror) > 31: - mirror_status.add_failing_mirror(best_mirror, -4) - else: - mirror_status.set_failing_mirror_status(best_mirror, 0) - - try: - remaining[repository].remove(best_mirror) - except ValueError: - # ignore - pass - return True - - def show_download_summary(down_list): - for _pkg_id, repo, fname, _cksum, _signatures in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - basef = os.path.basename(fname) - mytxt += "[%s] %s " % (brown(basef), blue("@"),) - mytxt += red(self._get_url_name(best_mirror)) - # now fetch the new one - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - def show_successful_download(down_list, data_transfer): - for _pkg_id, repo, fname, _cksum, _signatures in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - basef = os.path.basename(fname) - mytxt += "[%s] %s %s " % (brown(basef), - darkred(_("success")), blue("@"),) - mytxt += red(self._get_url_name(best_mirror)) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - mytxt = " %s: %s%s%s" % ( - blue(_("Aggregated transfer rate")), - bold(entropy.tools.bytes_into_human(data_transfer)), - darkred("/"), - darkblue(_("second")), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - def show_download_error(down_list, rc): - for _pkg_id, repo, _fname, _cksum, _signatures in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - mytxt += blue("%s: %s") % ( - _("Error downloading from"), - red(self._get_url_name(best_mirror)), - ) - if rc == -1: - mytxt += " - %s." % ( - _("data not available on this mirror"),) - elif rc == -2: - mirror_status.add_failing_mirror(best_mirror, 1) - mytxt += " - %s." % (_("wrong checksum"),) - elif rc == -3: - mytxt += " - %s." % (_("not found"),) - elif rc == -4: # timeout! - mytxt += " - %s." % (_("timeout error"),) - elif rc == -100: - mytxt += " - %s." % (_("discarded download"),) - else: - mirror_status.add_failing_mirror(best_mirror, 5) - mytxt += " - %s." % (_("unknown reason"),) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = red(" ## ") - ) - - def remove_failing_mirrors(repos): - for repo in repos: - get_best_mirror(repo) - if remaining[repo]: - remaining[repo].pop(0) - - def check_remaining_mirror_failure(repos): - return [x for x in repos if not remaining.get(x)] - - while True: - - do_resume = True - timeout_try_count = 50 - while True: - - fetch_files_list = [] - for pkg_id, repo, fname, cksum, _signatures in my_download_list: - best_mirror = get_best_mirror(repo) - # set working mirror, dont care if its None - mirror_status.set_working_mirror(best_mirror) - if best_mirror is not None: - mirror_fail_check(repo, best_mirror) - best_mirror = get_best_mirror(repo) - if best_mirror is None: - # at least one package failed to download - # properly, give up with everything - return 3, my_download_list - myuri = os.path.join(best_mirror, fname) - pkg_path = Package.get_standard_fetch_disk_path(fname) - fetch_files_list.append( - (pkg_id, repo, myuri, pkg_path, cksum,)) - - try: - - show_download_summary(my_download_list) - rc, failed_downloads, data_transfer = self.__fetch_files( - fetch_files_list, checksum = checksum, - resume = do_resume - ) - if rc == 0: - show_successful_download(my_download_list, - data_transfer) - return 0, [] - - # update my_download_list - my_download_list = update_download_list(my_download_list, - failed_downloads) - if rc not in (-3, -4, -100,) and failed_downloads and \ - do_resume: - # disable resume - do_resume = False - continue - else: - show_download_error(my_download_list, rc) - if rc == -4: # timeout - timeout_try_count -= 1 - if timeout_try_count > 0: - continue - elif rc == -100: # user discarded fetch - return 1, [] - myrepos = set([x[1] for x in my_download_list]) - remove_failing_mirrors(myrepos) - # make sure we don't have nasty issues - remaining_failure = check_remaining_mirror_failure( - myrepos) - if remaining_failure: - return 3, my_download_list - break - - except KeyboardInterrupt: - return 1, [] - - return 0, [] - - def _setup_differential_download(self, fetcher, url, resume, - fetch_path, repository, package_id): - """ - Setup differential download in case of URL supporting it. - Internal function. - - @param fetcher: UrlFetcher or MultipleUrlFetcher class - @param url: URL to check differential download against - @type url: string - @param resume: resume support - @type resume: bool - @param fetch_path: path where package file will be saved - @type fetch_path: string - @param repository: repository identifier belonging to package file - @type repository: string - @param package_id: package identifier belonging to repository identifier - @type package_id: int - """ - # no resume? no party? - if not resume: - const_debug_write(__name__, - "_setup_differential_download(%s) %s" % ( - url, "resume disabled")) - return - - if not fetcher.supports_differential_download(url): - # no differential download support - const_debug_write(__name__, - "_setup_differential_download(%s) %s" % ( - url, "no differential download support")) - return - - # this is the fetch path of the file that is going to be downloaded - # not going to overwrite it if a file is located there because - # user would want a resume for sure in first place - if os.path.isfile(fetch_path): - const_debug_write(__name__, - "_setup_differential_download(%s) %s" % ( - url, "fetch path already exists, not overwriting")) - return - - pkg_repo = self._entropy.open_repository(repository) - keyslot = pkg_repo.retrieveKeySlot(package_id) - if keyslot is None: - # fucked up entry, not dealing with it here - const_debug_write(__name__, - "_setup_differential_download(%s) %s" % ( - url, "key, slot fucked up")) - return - - key, slot = keyslot - inst_repo = self._entropy.installed_repository() - pkg_ids = inst_repo.searchKeySlot(key, slot) - if not pkg_ids: - # not installed, nothing to use as diff download - const_debug_write(__name__, - "_setup_differential_download(%s) %s" % ( - url, "no installed packages")) - return - - # grab the highest, we don't know if user was able to mess up - # its installed packages repository - pkg_id = max(pkg_ids) - download_url = inst_repo.retrieveDownloadURL(pkg_id) - installed_fetch_path = self.__get_fetch_disk_path(download_url) - - try: - shutil.copyfile(installed_fetch_path, fetch_path) - except (OSError, IOError, shutil.Error) as err: - const_debug_write( - __name__, - "_setup_differential_download(%s), copyfile error: %s" % ( - url, err)) - try: - os.remove(fetch_path) - except OSError: - pass - return - - try: - user = os.stat(installed_fetch_path)[stat.ST_UID] - group = os.stat(installed_fetch_path)[stat.ST_GID] - os.chown(fetch_path, user, group) - except (OSError, IOError) as err: - const_debug_write( - __name__, - "_setup_differential_download(%s), chown error: %s" % ( - url, err)) - return - - try: - shutil.copystat(installed_fetch_path, fetch_path) - except (OSError, IOError, shutil.Error) as err: - const_debug_write( - __name__, - "_setup_differential_download(%s), copystat error: %s" % ( - url, err)) - return - - const_debug_write( - __name__, - "_setup_differential_download(%s) copied to %s" % ( - url, fetch_path)) - - def __approve_edelta(self, url, installed_package_id, package_digest): - """ - Approve Entropy package delta support for given url, checking if - a previously fetched package is available. - - @return: edelta URL to download and previously downloaded package path - or None if edelta is not available - @rtype: tuple of strings or None - """ - inst_repo = self._entropy.installed_repository() - download_url = inst_repo.retrieveDownloadURL(installed_package_id) - installed_digest = inst_repo.retrieveDigest(installed_package_id) - installed_fetch_path = self.__get_fetch_disk_path(download_url) - - edelta_local_approved = False - try: - edelta_local_approved = entropy.tools.compare_md5( - installed_fetch_path, installed_digest) - except (OSError, IOError) as err: - const_debug_write( - __name__, "__approve_edelta, error: %s" % (err,)) - return - - if edelta_local_approved: - hash_tag = installed_digest + package_digest - edelta_file_name = entropy.tools.generate_entropy_delta_file_name( - os.path.basename(download_url), - os.path.basename(url), - hash_tag) - edelta_url = os.path.join( - os.path.dirname(url), - etpConst['packagesdeltasubdir'], - edelta_file_name) - - return edelta_url, installed_fetch_path - - def __try_edelta_multifetch(self, url_data, resume): - - # no edelta support enabled - if not self.pkgmeta.get('edelta_support'): - return [], 0.0, False - # edelta enabled? - if not entropy.tools.is_entropy_delta_available(): - return [], 0.0, False - - url_path_list = [] - url_data_map = {} - url_data_map_idx = 0 - for pkg_id, repo, url, dest_path, cksum in url_data: - - repo_db = self._entropy.open_repository(repo) - if cksum is None: - # cannot setup edelta without checksum, get from repository - cksum = repo_db.retrieveDigest(pkg_id) - if cksum is None: - # still nothing - continue - - key_slot = repo_db.retrieveKeySlot(pkg_id) - if key_slot is None: - # wtf corrupted entry, skip - continue - - pkg_key, pkg_slot = key_slot - installed_package_id = self.__setup_package_to_remove(pkg_key, - pkg_slot) - if installed_package_id == -1: - # package is not installed - continue - - edelta_approve = self.__approve_edelta(url, installed_package_id, - cksum) - if edelta_approve is None: - # no edelta support - continue - edelta_url, installed_fetch_path = edelta_approve - - edelta_save_path = dest_path + etpConst['packagesdeltaext'] - key = (edelta_url, edelta_save_path) - url_path_list.append(key) - url_data_map_idx += 1 - url_data_map[url_data_map_idx] = (pkg_id, repo, url, dest_path, - cksum, edelta_url, edelta_save_path, installed_fetch_path) - - if not url_path_list: - # no martini, no party! - return [], 0.0, False - - fetch_abort_function = self.pkgmeta.get('fetch_abort_function') - fetch_intf = self._entropy._multiple_url_fetcher(url_path_list, - resume = resume, abort_check_func = fetch_abort_function, - url_fetcher_class = self._entropy._url_fetcher) - try: - # make sure that we don't need to abort already - # doing the check here avoids timeouts - if fetch_abort_function != None: - fetch_abort_function() - - data = fetch_intf.download() - except KeyboardInterrupt: - return [], 0.0, True - data_transfer = fetch_intf.get_transfer_rate() - - fetch_errors = ( - UrlFetcher.TIMEOUT_FETCH_ERROR, - UrlFetcher.GENERIC_FETCH_ERROR, - UrlFetcher.GENERIC_FETCH_WARN, - ) - - valid_idxs = [] - for url_data_map_idx, cksum in tuple(data.items()): - - if cksum in fetch_errors: - # download failed - continue - - pkg_id, repo, url, dest_path, orig_cksum, edelta_url, \ - edelta_save_path, installed_fetch_path = \ - url_data_map[url_data_map_idx] - - # now check - tmp_dest_path = dest_path + ".edelta_pkg_tmp" - # yay, we can apply the delta and cook the new package file! - try: - entropy.tools.apply_entropy_delta(installed_fetch_path, - edelta_save_path, tmp_dest_path) - except IOError: - # give up with this edelta - try: - os.remove(tmp_dest_path) - except (OSError, IOError): - pass - continue - - os.rename(tmp_dest_path, dest_path) - valid_idxs.append(url_data_map_idx) - - # now check md5 - fetched_url_data = [] - for url_data_map_idx in valid_idxs: - pkg_id, repo, url, dest_path, orig_cksum, edelta_url, \ - edelta_save_path, installed_fetch_path = \ - url_data_map[url_data_map_idx] - - try: - valid = entropy.tools.compare_md5(dest_path, orig_cksum) - except (IOError, OSError): - valid = False - - if valid: - url_data_item = (pkg_id, repo, url, dest_path, orig_cksum) - fetched_url_data.append(url_data_item) - - return fetched_url_data, data_transfer, False - - - def __try_edelta_fetch(self, url, save_path, resume): - - # no edelta support enabled - if not self.pkgmeta.get('edelta_support'): - return 1, 0.0 - # edelta enabled? - if not entropy.tools.is_entropy_delta_available(): - return 1, 0.0 - - # when called by __fetch_file, which is called by _download_package - # which is called by _match_checksum, which is called by - # multi_match_checksum, removepackage_id metadatum is not available - # So, be fault tolerant. - installed_package_id = self.pkgmeta.get('removepackage_id', -1) - - # fresh install, cannot fetch edelta, edelta only works for installed - # packages, by design. - if installed_package_id == -1: - return 1, 0.0 - - edelta_approve = self.__approve_edelta(url, installed_package_id, - self.pkgmeta['checksum']) - - if edelta_approve is None: - # edelta not available, give up - return 1, 0.0 - edelta_url, installed_fetch_path = edelta_approve - - # check if edelta file is available online - edelta_save_path = save_path + etpConst['packagesdeltaext'] - - max_tries = 2 - edelta_approved = False - data_transfer = 0 - download_plan = [(edelta_url, edelta_save_path) for _x in \ - range(max_tries)] - delta_resume = resume - fetch_abort_function = self.pkgmeta.get('fetch_abort_function') - fetch_errors = ( - UrlFetcher.TIMEOUT_FETCH_ERROR, - UrlFetcher.GENERIC_FETCH_ERROR, - UrlFetcher.GENERIC_FETCH_WARN, - ) - - for delta_url, delta_save in download_plan: - - delta_fetcher = self._entropy._url_fetcher(delta_url, - delta_save, resume = delta_resume, - abort_check_func = fetch_abort_function) - - try: - # make sure that we don't need to abort already - # doing the check here avoids timeouts - if fetch_abort_function != None: - fetch_abort_function() - - delta_checksum = delta_fetcher.download() - data_transfer = delta_fetcher.get_transfer_rate() - del delta_fetcher - except KeyboardInterrupt: - return -100, data_transfer - except NameError: - raise - except Exception: - return -1, data_transfer - - if delta_checksum in fetch_errors: - # make sure this points to hell - delta_resume = False - # retry - continue - - # now check - tmp_save_path = save_path + ".edelta_pkg_tmp" - # yay, we can apply the delta and cook the new package file! - try: - entropy.tools.apply_entropy_delta(installed_fetch_path, - delta_save, tmp_save_path) - except IOError: - # make sure this points to the hell - delta_resume = False - # retry - try: - os.remove(tmp_save_path) - except (OSError, IOError): - pass - continue - - os.rename(tmp_save_path, save_path) - edelta_approved = True - break - - if edelta_approved: - # we can happily return - return 0, data_transfer - # error, give up with the edelta stuff - return 1, data_transfer - - def __fetch_file(self, url, save_path, digest = None, resume = True, - download = None, package_id = None, repository = None): - - def do_stfu_rm(xpath): - try: - os.remove(xpath) - except OSError: - pass - - def do_get_md5sum(path): - try: - return entropy.tools.md5sum(path) - except IOError: - return None - except OSError: - return None - - fetch_abort_function = self.pkgmeta.get('fetch_abort_function') - filepath_dir = os.path.dirname(save_path) - # symlink support - if not os.path.isdir(os.path.realpath(filepath_dir)): - try: - os.remove(filepath_dir) - except OSError as err: - const_debug_write(__name__, - "__fetch_file.remove, %s, error: %s" % ( - filepath_dir, err)) - try: - os.makedirs(filepath_dir, 0o755) - except OSError as err: - const_debug_write(__name__, - "__fetch_file.makedirs, %s, error: %s" % ( - filepath_dir, err)) - return -1, 0, False - - rc, data_transfer = self.__try_edelta_fetch(url, save_path, resume) - if rc == 0: - return rc, data_transfer, False - elif rc < 0: # < 0 errors are unrecoverable - return rc, data_transfer, False - # otherwise, just fallback to package download - - existed_before = False - if os.path.isfile(save_path) and os.path.exists(save_path): - existed_before = True - - fetch_intf = self._entropy._url_fetcher( - url, save_path, resume = resume, - abort_check_func = fetch_abort_function) - if (download is not None) and (package_id is not None) and \ - (repository is not None) and (rc == 0): - fetch_path = self.__get_fetch_disk_path(download) - self._setup_differential_download( - self._entropy._url_fetcher, url, - resume, fetch_path, repository, package_id) - - # start to download - data_transfer = 0 - resumed = False - try: - # make sure that we don't need to abort already - # doing the check here avoids timeouts - if fetch_abort_function != None: - fetch_abort_function() - - fetch_checksum = fetch_intf.download() - data_transfer = fetch_intf.get_transfer_rate() - resumed = fetch_intf.is_resumed() - except KeyboardInterrupt: - return -100, data_transfer, resumed - except NameError: - raise - except: - if const_debug_enabled(): - self._entropy.output( - "fetch_file:", - importance = 1, - level = "warning", - header = red(" ## ") - ) - entropy.tools.print_traceback() - if (not existed_before) or (not resume): - do_stfu_rm(save_path) - return -1, data_transfer, resumed - if fetch_checksum == UrlFetcher.GENERIC_FETCH_ERROR: - # !! not found - # maybe we already have it? - # this handles the case where network is unavailable - # but file is already downloaded - fetch_checksum = do_get_md5sum(save_path) - if (fetch_checksum != digest) or fetch_checksum is None: - return -3, data_transfer, resumed - elif fetch_checksum == UrlFetcher.TIMEOUT_FETCH_ERROR: - # maybe we already have it? - # this handles the case where network is unavailable - # but file is already downloaded - fetch_checksum = do_get_md5sum(save_path) - if (fetch_checksum != digest) or fetch_checksum is None: - return -4, data_transfer, resumed - - del fetch_intf - - if digest and (fetch_checksum != digest): - # not properly downloaded - if (not existed_before) or (not resume): - do_stfu_rm(save_path) - return -2, data_transfer, resumed - - return 0, data_transfer, resumed - - def __build_uris_list(self, original_repo, repository): - - avail_data = self._settings['repositories']['available'] - product = self._settings['repositories']['product'] - uris = [] - plain_packages = avail_data[repository]['plain_packages'] - for uri in plain_packages: - expanded_uri = entropy.tools.expand_plain_package_mirror( - uri, product, original_repo) - uris.append(expanded_uri) - uris.reverse() - uris.extend(avail_data[repository]['packages'][::-1]) - return uris - - - def _download_package(self, package_id, repository, download, save_path, - digest = False, resume = True): - - avail_data = self._settings['repositories']['available'] - excluded_data = self._settings['repositories']['excluded'] - repo_db = self._entropy.open_repository(repository) - # grab original repo, if any and use it if available - # this is done in order to support "equo repo merge" feature - # allowing client-side repository package metadata moves. - original_repo = repo_db.getInstalledPackageRepository(package_id) - if (original_repo != repository) and (original_repo not in avail_data) \ - and (original_repo is not None): - # build up a new uris list, at least try, hoping that - # repository is just shadowing original_repo - # for example: original_repo got copied to repository, without - # copying packages, which would be useless. like it happens - # with sabayon-weekly - uris = self.__build_uris_list(original_repo, repository) - else: - if original_repo in avail_data: - uris = avail_data[original_repo]['packages'][::-1] - if repository in avail_data: - uris += avail_data[repository]['packages'][::-1] - elif original_repo in excluded_data: - uris = excluded_data[original_repo]['packages'][::-1] - if repository in avail_data: - uris += avail_data[repository]['packages'][::-1] - else: - uris = avail_data[repository]['packages'][::-1] - - remaining = set(uris) - mirror_status = StatusInterface() - - mirrorcount = 0 - for uri in uris: - - if not remaining: - # tried all the mirrors, quitting for error - mirror_status.set_working_mirror(None) - return 3 - - mirror_status.set_working_mirror(uri) - mirrorcount += 1 - mirror_count_txt = "( mirror #%s ) " % (mirrorcount,) - url = uri + "/" + download - - # check if uri is sane - if mirror_status.get_failing_mirror_status(uri) >= 30: - # ohohoh! - # set to 30 for convenience - mirror_status.set_failing_mirror_status(uri, 30) - mytxt = mirror_count_txt - mytxt += blue(" %s: ") % (_("Mirror"),) - mytxt += red(self._get_url_name(uri)) - mytxt += " - %s." % (_("maximum failure threshold reached"),) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = red(" ## ") - ) - - if mirror_status.get_failing_mirror_status(uri) == 30: - # put to 75 then decrement by 4 so we - # won't reach 30 anytime soon ahahaha - mirror_status.add_failing_mirror(uri, 45) - elif mirror_status.get_failing_mirror_status(uri) > 31: - # now decrement each time this point is reached, - # if will be back < 30, then equo will try to use it again - mirror_status.add_failing_mirror(uri, -4) - else: - # put to 0 - reenable mirror, welcome back uri! - mirror_status.set_failing_mirror_status(uri, 0) - - remaining.discard(uri) - continue - - do_resume = resume - timeout_try_count = 50 - while True: - try: - mytxt = mirror_count_txt - mytxt += blue("%s: ") % (_("Downloading from"),) - mytxt += red(self._get_url_name(uri)) - # now fetch the new one - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = red(" ## ") - ) - rc, data_transfer, resumed = \ - self.__fetch_file( - url, - save_path, - download = download, - package_id = package_id, - repository = repository, - digest = digest, - resume = do_resume - ) - if rc == 0: - mytxt = mirror_count_txt - mytxt += "%s: " % ( - blue(_("Successfully downloaded from")), - ) - mytxt += red(self._get_url_name(uri)) - human_bytes = entropy.tools.bytes_into_human( - data_transfer) - mytxt += " %s %s/%s" % (_("at"), - human_bytes, _("second"),) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - mirror_status.set_working_mirror(None) - return 0 - elif resumed and (rc not in (-3, -4, -100,)): - do_resume = False - continue - else: - error_message = mirror_count_txt - error_message += blue("%s: %s") % ( - _("Error downloading from"), - red(self._get_url_name(uri)), - ) - # something bad happened - if rc == -1: - error_message += " - %s." % ( - _("file not available on this mirror"),) - elif rc == -2: - mirror_status.add_failing_mirror(uri, 1) - error_message += " - %s." % (_("wrong checksum"),) - # If file is fetched (with no resume) and its - # complete better to enforce resume to False. - if (data_transfer < 1) and do_resume: - error_message += " %s." % ( - _("Disabling resume"),) - do_resume = False - continue - elif rc == -3: - mirror_status.add_failing_mirror(uri, 3) - error_message += " - %s." % (_("not found"),) - elif rc == -4: # timeout! - timeout_try_count -= 1 - if timeout_try_count > 0: - error_message += " - %s." % ( - _("timeout, retrying on this mirror"),) - else: - error_message += " - %s." % ( - _("timeout, giving up"),) - elif rc == -100: - error_message += " - %s." % ( - _("discarded download"),) - else: - mirror_status.add_failing_mirror(uri, 5) - error_message += " - %s." % (_("unknown reason"),) - self._entropy.output( - error_message, - importance = 1, - level = "warning", - header = red(" ## ") - ) - if rc == -4: # timeout - if timeout_try_count > 0: - continue - elif rc == -100: # user discarded fetch - mirror_status.set_working_mirror(None) - return 1 - remaining.discard(uri) - # make sure we don't have nasty issues - if not remaining: - mirror_status.set_working_mirror(None) - return 3 - break - except KeyboardInterrupt: - mirror_status.set_working_mirror(None) - return 1 - - mirror_status.set_working_mirror(None) - return 0 - - def _match_checksum(self, package_id, repository, checksum, download, - signatures): - - sys_settings = self._settings - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - enabled_hashes = sys_settings[sys_set_plg_id]['misc']['packagehashes'] - - pkg_disk_path = self.__get_fetch_disk_path(download) - pkg_disk_path_mtime = pkg_disk_path + etpConst['packagemtimefileext'] - - def do_mtime_validation(): - enc = etpConst['conf_encoding'] - try: - with codecs.open(pkg_disk_path_mtime, - "r", encoding=enc) as mt_f: - stored_mtime = mt_f.read().strip() - except (OSError, IOError) as err: - if err.errno != errno.ENOENT: - raise - return 1 - - # get pkg mtime - try: - cur_mtime = str(os.path.getmtime(pkg_disk_path)) - except (OSError, IOError) as err: - if err.errno != errno.ENOENT: - raise - return 2 - - if cur_mtime == stored_mtime: - return 0 - return 1 - - def do_store_mtime(): - enc = etpConst['conf_encoding'] - try: - with codecs.open(pkg_disk_path_mtime, - "w", encoding=enc) as mt_f: - cur_mtime = str(os.path.getmtime(pkg_disk_path)) - mt_f.write(cur_mtime) - except (OSError, IOError) as err: - if err.errno != errno.ENOENT: - raise - - def do_compare_gpg(pkg_path, hash_val): - - try: - repo_sec = self._entropy.RepositorySecurity() - except RepositorySecurity.GPGServiceNotAvailable: - return None - - # check if we have repository pubkey - try: - if not repo_sec.is_pubkey_available(repository): - return None - except repo_sec.KeyExpired: - # key is expired - return None - - # write gpg signature to disk for verification - tmp_fd, tmp_path = const_mkstemp(prefix="do_compare_gpg") - with os.fdopen(tmp_fd, "w") as tmp_f: - tmp_f.write(hash_val) - - try: - # actually verify - valid, err_msg = repo_sec.verify_file(repository, pkg_path, - tmp_path) - finally: - os.remove(tmp_path) - - if valid: - return True - - if err_msg: - self._entropy.output( - "%s: %s, %s" % ( - darkred(_("Package signature verification error for")), - purple("GPG"), - err_msg, - ), - importance = 0, - level = "error", - header = darkred(" ## ") - ) - return False - - signature_vry_map = { - 'sha1': entropy.tools.compare_sha1, - 'sha256': entropy.tools.compare_sha256, - 'sha512': entropy.tools.compare_sha512, - 'gpg': do_compare_gpg, - } - - def do_signatures_validation(signatures): - # check signatures, if available - if isinstance(signatures, dict): - for hash_type in sorted(signatures): - hash_val = signatures[hash_type] - # NOTE: workaround bug on unreleased - # entropy versions - if hash_val in signatures: - continue - if hash_val is None: - continue - if hash_type not in enabled_hashes: - self._entropy.output( - "%s %s" % ( - purple(hash_type.upper()), - darkgreen(_("disabled")), - ), - importance = 0, - level = "info", - header = " : " - ) - continue - - cmp_func = signature_vry_map.get(hash_type) - if cmp_func is None: - continue - - self._entropy.output( - "%s: %s" % (blue(_("Checking package signature")), - purple(hash_type.upper()),), - importance = 0, - level = "info", - header = red(" ## "), - back = True - ) - valid = cmp_func(pkg_disk_path, hash_val) - if valid is None: - self._entropy.output( - "%s '%s' %s" % ( - darkred(_("Package signature verification")), - purple(hash_type.upper()), - darkred(_("temporarily unavailable")), - ), - importance = 0, - level = "warning", - header = darkred(" ## ") - ) - continue - if not valid: - self._entropy.output( - "%s: %s %s" % ( - darkred(_("Package signature")), - purple(hash_type.upper()), - darkred(_("does not match the recorded one")), - ), - importance = 0, - level = "error", - header = darkred(" ## ") - ) - return 1 - self._entropy.output( - "%s %s" % ( - purple(hash_type.upper()), - darkgreen(_("matches")), - ), - importance = 0, - level = "info", - header = " : " - ) - return 0 - - dlcount = 0 - match = False - max_dlcount = 5 - - while dlcount <= max_dlcount: - - self._entropy.output( - blue(_("Checking package checksum...")), - importance = 0, - level = "info", - header = red(" ## "), - back = True - ) - - dlcheck = self.__check_pkg_path_download(download, - checksum = checksum) - if dlcheck == 0: - basef = os.path.basename(download) - self._entropy.output( - "%s: %s" % ( - blue(_("Package checksum matches")), - darkgreen(basef), - ), - importance = 0, - level = "info", - header = red(" ## ") - ) - - # check if package has been already checked - dlcheck = do_mtime_validation() - if dlcheck != 0: - dlcheck = do_signatures_validation(signatures) - - if dlcheck == 0: - do_store_mtime() - match = True - break # file downloaded successfully - - if dlcheck != 0: - dlcount += 1 - mytxt = _("Checksum does not match. Download attempt #%s") % ( - dlcount, - ) - self._entropy.output( - darkred(mytxt), - importance = 0, - level = "warning", - header = darkred(" ## ") - ) - # Unfortunately, disabling resume makes possible to recover - # from bad download data. trying to resume would do more harm - # than good in the majority of cases. - fetch = self._download_package( - package_id, - repository, - download, - pkg_disk_path, - checksum, - resume = False - ) - if fetch != 0: - self._entropy.output( - blue(_("Cannot properly fetch package! Quitting.")), - importance = 0, - level = "error", - header = darkred(" ## ") - ) - return fetch - - # package is fetched, let's loop one more time - # to make sure to run all the checksum checks - continue - - if not match: - mytxt = _("Cannot fetch package or checksum does not match") - mytxt2 = _("Try to download latest repositories") - for txt in (mytxt, mytxt2,): - self._entropy.output( - "%s." % (blue(txt),), - importance = 0, - level = "info", - header = red(" ## ") - ) - return 1 - - return 0 - - def multi_match_checksum(self): - rc = 0 - for pkg_id, repository, download, digest, signatures in \ - self.pkgmeta['multi_checksum_list']: - - rc = self._match_checksum(pkg_id, repository, digest, download, - signatures) - if rc != 0: - break - - if rc == 0: - self.pkgmeta['verified'] = True - return rc - - def __unpack_package(self, download, package_path, image_dir, pkg_dbpath): - - mytxt = "%s: %s" % ( - blue(_("Unpacking")), - red(os.path.basename(download)), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Unpacking package: %s" % (download,) - ) - - # removed in the meantime? at least try to cope - if not os.path.isfile(package_path): - - # must be fault-tolerant - if os.path.isdir(package_path): - shutil.rmtree(package_path) - if os.path.islink(package_path): - os.remove(package_path) - self.pkgmeta['verified'] = False - rc = self._fetch_step() - if rc != 0: - return rc - - # make sure image_dir always exists - # pkgs not providing any file would cause image_dir - # to not be created by uncompress_tarball - try: - os.makedirs(image_dir, 0o755) - except OSError as err: - if err.errno != errno.EEXIST: - self._entropy.logger.log( - "[Package]", etpConst['logging']['normal_loglevel_id'], - "Unable to mkdir: %s, error: %s" % ( - image_dir, repr(err),) - ) - self._entropy.output( - "%s: %s" % (brown(_("Unpack error")), err.errno,), - importance = 1, - level = "error", - header = red(" ## ") - ) - return 1 - - # pkg_dbpath is only non-None for the base package file - # extra package files don't carry any other edb information - if pkg_dbpath is not None: - # extract entropy database from package file - # in order to avoid having to read content data - # from the repository database, which, in future - # is allowed to not provide such info. - pkg_dbdir = os.path.dirname(pkg_dbpath) - if not os.path.isdir(pkg_dbdir): - os.makedirs(pkg_dbdir, 0o755) - # extract edb - dump_rc = entropy.tools.dump_entropy_metadata( - package_path, pkg_dbpath) - if not dump_rc: - # error during entropy db extraction from package file - # might be because edb entry point is not found or - # because there is not enough space for it - self._entropy.logger.log( - "[Package]", etpConst['logging']['normal_loglevel_id'], - "Unable to dump edb for: " + pkg_dbpath - ) - self._entropy.output( - brown(_("Unable to find Entropy metadata in package")), - importance = 1, - level = "error", - header = red(" ## ") - ) - return 1 - - unpack_tries = 3 - while True: - if unpack_tries <= 0: - return 1 - unpack_tries -= 1 - try: - rc = entropy.tools.uncompress_tarball( - package_path, - extract_path = image_dir, - catch_empty = True - ) - except EOFError as err: - self._entropy.logger.log( - "[Package]", etpConst['logging']['normal_loglevel_id'], - "EOFError on " + package_path + " " + \ - repr(err) - ) - entropy.tools.print_traceback() - # try again until unpack_tries goes to 0 - rc = 1 - except Exception as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Ouch! error while unpacking " + \ - package_path + " " + repr(err) - ) - entropy.tools.print_traceback() - # try again until unpack_tries goes to 0 - rc = 1 - - if rc == 0: - break - - # try to download it again - self.pkgmeta['verified'] = False - f_rc = self._fetch_step() - if f_rc != 0: - return f_rc - return 0 - - - def __configure_package(self): - - try: - spm = self._entropy.Spm() - except Exception as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Source Package Manager not available: %s | %s" % ( - type(Exception), err, - ) - ) - return 1 - - self._entropy.output( - "SPM: %s" % (brown(_("configuration phase")),), - importance = 0, - header = red(" ## ") - ) - try: - spm.execute_package_phase(self.pkgmeta, self.pkgmeta, - self._action, "configure") - - except spm.PhaseFailure as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Phase execution failed with %s, %d" % ( - err.message, err.code)) - return err.code - - except spm.OutdatedPhaseError as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Source Package Manager is too old: %s" % ( - err)) - - err_msg = "%s: %s" % ( - brown(_("Source Package Manager is too old, please update it")), - err) - self._entropy.output( - err_msg, - importance = 1, - header = darkred(" ## "), - level = "error" - ) - return 1 - - except spm.PhaseError as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Phase execution error: %s" % ( - err)) - return 1 - - return 0 - - def _setup_config_protect_and_mask(self, installed_repository, - installed_package_id): - """ - Setup the config_protect+mask metadata object. - Make sure to call this before the package goes away from the - repository. - """ - protect = self._get_config_protect( - installed_repository, installed_package_id) - mask = self._get_config_protect( - installed_repository, installed_package_id, mask = True) - - self.pkgmeta['config_protect+mask'] = (protect, mask) - - def __remove_package(self): - - self._entropy.clear_cache() - - self._entropy.logger.log("[Package]", - etpConst['logging']['normal_loglevel_id'], - "Removing package: %s" % (self.pkgmeta['removeatom'],)) - - mytxt = "%s: %s" % ( - blue(_("Removing from Entropy")), - red(self.pkgmeta['removeatom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - inst_repo = self._entropy.installed_repository() - automerge_metadata = inst_repo.retrieveAutomergefiles( - self.pkgmeta['removepackage_id'], get_dict = True) - # setup config_protect and config_protect_mask metadata before it's - # too late. - self._setup_config_protect_and_mask( - inst_repo, self.pkgmeta['removepackage_id']) - - inst_repo.removePackage(self.pkgmeta['removepackage_id']) - - # commit changes, to avoid users pressing CTRL+C and still having - # all the db entries in, so we need to commit at every iteration - inst_repo.commit() - - self._remove_content_from_system( - automerge_metadata = automerge_metadata) - - return 0 - - def _remove_content_from_system_loop( - self, remove_content, directories, directories_cache, - not_removed_due_to_collisions, colliding_path_messages, - automerge_metadata, col_protect, protect, mask, protectskip, - sys_root): - """ - Body of the _remove_content_from_system() method. - """ - inst_repo = self._entropy.installed_repository() - info_dirs = self._get_info_directories() - - for _pkg_id, item, _ftype in remove_content: - - if not item: - continue # empty element?? - - sys_root_item = sys_root + item - sys_root_item_encoded = sys_root_item - if not const_is_python3(): - # this is coming from the db, and it's pure utf-8 - sys_root_item_encoded = const_convert_to_rawstring( - sys_root_item, - from_enctype = etpConst['conf_raw_encoding']) - - # collision check - if col_protect > 0: - - if inst_repo.isFileAvailable(item) \ - and os.path.isfile(sys_root_item_encoded): - - # in this way we filter out directories - colliding_path_messages.add(sys_root_item) - not_removed_due_to_collisions.add(item) - continue - - protected = False - in_mask = False - - if not self.pkgmeta['removeconfig']: - - protected_item_test = sys_root_item - (in_mask, protected, _x, - do_continue) = self._handle_config_protect( - protect, mask, protectskip, None, protected_item_test, - do_allocation_check = False, do_quiet = True - ) - - if do_continue: - protected = True - - # when files have not been modified by the user - # and they are inside a config protect directory - # we could even remove them directly - if in_mask: - - oldprot_md5 = automerge_metadata.get(item) - if oldprot_md5: - - try: - in_system_md5 = entropy.tools.md5sum( - protected_item_test) - except (OSError, IOError) as err: - if err.errno != errno.ENOENT: - raise - in_system_md5 = "?" - - if oldprot_md5 == in_system_md5: - prot_msg = _("Removing config file, never modified") - mytxt = "%s: %s" % ( - darkgreen(prot_msg), - blue(item), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - protected = False - do_continue = False - - # Is file or directory a protected item? - if protected: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['verbose_loglevel_id'], - "[remove] Protecting config file: %s" % (sys_root_item,) - ) - mytxt = "[%s] %s: %s" % ( - red(_("remove")), - brown(_("Protecting config file")), - sys_root_item, - ) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = red(" ## ") - ) - continue - - try: - os.lstat(sys_root_item_encoded) - except OSError as err: - if err.errno in (errno.ENOENT, errno.ENOTDIR): - continue # skip file, does not exist - raise - - except UnicodeEncodeError: - msg = _("This package contains a badly encoded file !!!") - mytxt = brown(msg) - self._entropy.output( - red("QA: ")+mytxt, - importance = 1, - level = "warning", - header = darkred(" ## ") - ) - continue # file has a really bad encoding - - if os.path.isdir(sys_root_item_encoded) and \ - os.path.islink(sys_root_item_encoded): - # S_ISDIR returns False for directory symlinks, - # so using os.path.isdir valid directory symlink - if sys_root_item not in directories_cache: - # collect for Trigger - self.pkgmeta['affected_directories'].add(item) - directories.add((sys_root_item, "link")) - directories_cache.add(sys_root_item) - continue - - if os.path.isdir(sys_root_item_encoded): - # plain directory - if sys_root_item not in directories_cache: - # collect for Trigger - self.pkgmeta['affected_directories'].add(item) - directories.add((sys_root_item, "dir")) - directories_cache.add(sys_root_item) - continue - - # files, symlinks or not - # just a file or symlink or broken - # directory symlink (remove now) - - try: - os.remove(sys_root_item_encoded) - except OSError as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "[remove] Unable to remove %s, error: %s" % ( - sys_root_item, err,) - ) - continue - - # collect for Trigger - dir_name = os.path.dirname(item) - self.pkgmeta['affected_directories'].add(dir_name) - - # account for info files, if any - if dir_name in info_dirs: - for _ext in self._INFO_EXTS: - if item.endswith(_ext): - self.pkgmeta['affected_infofiles'].add(item) - break - - # add its parent directory - dirobj = const_convert_to_unicode( - os.path.dirname(sys_root_item_encoded)) - if dirobj not in directories_cache: - if os.path.isdir(dirobj) and os.path.islink(dirobj): - directories.add((dirobj, "link")) - elif os.path.isdir(dirobj): - directories.add((dirobj, "dir")) - - directories_cache.add(dirobj) - - def _remove_content_from_system(self, automerge_metadata = None): - """ - Remove installed package content (files/directories) from live system. - - @keyword automerge_metadata: Entropy "automerge metadata" - @type automerge_metadata: dict - """ - if automerge_metadata is None: - automerge_metadata = {} - - sys_root = etpConst['systemroot'] - # load CONFIG_PROTECT and CONFIG_PROTECT_MASK - sys_settings = self._settings - - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - col_protect = sys_settings[sys_set_plg_id]['misc']['collisionprotect'] - - # remove files from system - directories = set() - directories_cache = set() - not_removed_due_to_collisions = set() - colliding_path_messages = set() - - protect_mask = self.pkgmeta['config_protect+mask'] - if protect_mask is not None: - protect, mask = protect_mask - else: - protect, mask = set(), set() - protectskip = self._get_config_protect_skip() - - remove_content = None - try: - # simulate a removecontent list/set object - remove_content = [] - if self.pkgmeta['removecontent_file'] is not None: - remove_content = Content.FileContentReader( - self.pkgmeta['removecontent_file']) - - self._remove_content_from_system_loop( - remove_content, directories, directories_cache, - not_removed_due_to_collisions, colliding_path_messages, - automerge_metadata, col_protect, protect, mask, protectskip, - sys_root) - - finally: - if hasattr(remove_content, "close"): - remove_content.close() - - if colliding_path_messages: - self._entropy.output( - "%s:" % (_("Collision found during removal of"),), - importance = 1, - level = "warning", - header = red(" ## ") - ) - - for path in sorted(colliding_path_messages): - self._entropy.output( - purple(path), - importance = 0, - level = "warning", - header = red(" ## ") - ) - self._entropy.logger.log( - "[Package]", etpConst['logging']['normal_loglevel_id'], - "Collision found during removal of %s - cannot overwrite" % ( - path,) - ) - - # removing files not removed from removecontent. - # it happened that boot services not removed due to - # collisions got removed from their belonging runlevels - # by postremove step. - # since this is a set, it is a mapped type, so every - # other instance around will feature this update - if not_removed_due_to_collisions: - def _filter(_path): - return _path not in not_removed_due_to_collisions - Package._filter_content_file( - self.pkgmeta['removecontent_file'], - _filter) - - # now handle directories - directories = sorted(directories, reverse = True) - while True: - taint = False - for directory, dirtype in directories: - mydir = "%s%s" % (sys_root, directory,) - if dirtype == "link": - try: - mylist = os.listdir(mydir) - if not mylist: - try: - os.remove(mydir) - taint = True - except OSError: - pass - except OSError: - pass - elif dirtype == "dir": - try: - mylist = os.listdir(mydir) - if not mylist: - try: - os.rmdir(mydir) - taint = True - except OSError: - pass - except OSError: - pass - - if not taint: - break - - del directories_cache - del directories - - def _cleanup_package(self, unpack_dir): - # shutil.rmtree wants raw strings... otherwise it will explode - if const_isunicode(unpack_dir): - unpack_dir = const_convert_to_rawstring(unpack_dir) - - # remove unpack dir - try: - shutil.rmtree(unpack_dir, True) - except Exception as err: - # fault tolerance here dude - self._entropy.logger.log( - "[Package]", etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to cleanup directory %s," \ - " error: %s" % (unpack_dir, err,)) - try: - os.rmdir(unpack_dir) - except OSError: - pass - return 0 - - def __install_package(self): - - # clear on-disk cache - self._entropy.clear_cache() - - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Installing package: %s" % (self.pkgmeta['atom'],) - ) - - inst_repo = self._entropy.installed_repository() - - if self.pkgmeta['removepackage_id'] != -1: - self.pkgmeta['already_protected_config_files'] = \ - inst_repo.retrieveAutomergefiles( - self.pkgmeta['removepackage_id'], get_dict = True - ) - - # items_*installed will be filled by _move_image_to_system - # then passed to _add_installed_package() - items_installed = set() - items_not_installed = set() - rc = self._move_image_to_system( - items_installed, items_not_installed) - if rc != 0: - return rc - - # inject into database - mytxt = "%s: %s" % ( - blue(_("Updating database")), - red(self.pkgmeta['atom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - self.pkgmeta['installed_package_id'] = self._add_installed_package( - items_installed, items_not_installed) - return 0 - - def _spm_install_package(self, installed_package_id): - """ - Call Source Package Manager interface and tell it to register our - newly installed package. - - @param installed_package_id: Entropy repository package identifier - @type installed_package_id: int - @return: execution status - @rtype: int - """ - try: - Spm = self._entropy.Spm() - except Exception as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Source Package Manager not available: %s | %s" % ( - type(Exception), err, - ) - ) - return -1 - - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Installing new SPM entry: %s" % (self.pkgmeta['atom'],) - ) - - spm_uid = Spm.add_installed_package(self.pkgmeta) - if spm_uid != -1: - self._entropy.installed_repository().insertSpmUid( - installed_package_id, spm_uid) - - return 0 - - def _spm_update_package_uid(self, installed_package_id, entropy_atom): - """ - Update Source Package Manager <-> Entropy package identifiers coupling. - Entropy can handle multiple packages in the same scope from a SPM POV - (see the "package tag" feature to provide linux kernel module packages - for different kernel versions). This method just reassigns a new SPM - unique package identifier to Entropy. - - @param installed_package_id: Entropy package identifier bound to - given entropy_atom - @type installed_package_id: int - @param entropy_atom: Entropy package atom, must be converted to a valid - SPM package atom. - @type entropy_atom: string - @return: execution status - @rtype: int - """ - try: - Spm = self._entropy.Spm() - except Exception as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Source Package Manager not available: %s | %s" % ( - type(Exception), err, - ) - ) - return -1 - - spm_package = Spm.convert_from_entropy_package_name(entropy_atom) - try: - spm_uid = Spm.assign_uid_to_installed_package(spm_package) - except (SPMError, KeyError,): - # installed package not available, we must ignore it - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Spm uid not available for Spm package: %s (pkg not avail?)" % ( - spm_package, - ) - ) - return 0 - - if spm_uid != -1: - self._entropy.installed_repository().insertSpmUid( - installed_package_id, spm_uid) - - return 0 - - - def _spm_remove_package(self): - """ - Call Source Package Manager interface and tell it to remove our - just removed package. - - @return: execution status - @rtype: int - """ - try: - Spm = self._entropy.Spm() - except Exception as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Source Package Manager not available: %s | %s" % ( - type(Exception), err, - ) - ) - return -1 - - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Removing from SPM: %s" % (self.pkgmeta['removeatom'],) - ) - - return Spm.remove_installed_package(self.pkgmeta) - - - def _add_installed_package(self, items_installed, items_not_installed): - """ - For internal use only. - Copy package from repository to installed packages one. - """ - - def _merge_removecontent(inst_repo, repo, _package_id): - # NOTE: this could be a source of memory consumption - # but generally, the difference between two contents - # is really small - content_diff = list(inst_repo.contentDiff( - self.pkgmeta['removepackage_id'], - repo, - _package_id, - extended=True)) - - if content_diff: - - # reverse-order compare - def _cmp_func(_path, _spath): - if _path > _spath: - return -1 - elif _path == _spath: - return 0 - return 1 - - # must be sorted, and in reverse order - # or the merge step won't work - content_diff.sort(reverse=True) - - Package._merge_content_file( - self.pkgmeta['removecontent_file'], - content_diff, _cmp_func) - - inst_repo = self._entropy.installed_repository() - smart_pkg = self.pkgmeta['smartpackage'] - dbconn = self._entropy.open_repository(self.pkgmeta['repository']) - splitdebug, splitdebug_dirs = self.pkgmeta['splitdebug'], \ - self.pkgmeta['splitdebug_dirs'] - - if smart_pkg or self.pkgmeta['merge_from']: - - data = dbconn.getPackageData(self.pkgmeta['package_id'], - content_insert_formatted = True, - get_changelog = False, get_content = False, - get_content_safety = False) - - content = dbconn.retrieveContentIter( - self.pkgmeta['package_id']) - content_file = self.__generate_content_file( - content, package_id = self.pkgmeta['package_id'], - filter_splitdebug = True, - splitdebug = splitdebug, - splitdebug_dirs = splitdebug_dirs) - - content_safety = dbconn.retrieveContentSafetyIter( - self.pkgmeta['package_id']) - content_safety_file = self.__generate_content_safety_file( - content_safety) - - if self.pkgmeta['removepackage_id'] != -1 and \ - self.pkgmeta['removecontent_file'] is not None: - _merge_removecontent( - inst_repo, dbconn, self.pkgmeta['package_id']) - - else: - - # normal repositories - data = dbconn.getPackageData(self.pkgmeta['package_id'], - get_content = False, get_changelog = False) - - # indexing_override = False : no need to index tables - # xcache = False : no need to use on-disk cache - # skipChecks = False : creating missing tables is unwanted, - # and also no foreign keys update - # readOnly = True: no need to open in write mode - pkg_dbconn = self._entropy.open_generic_repository( - self.pkgmeta['pkgdbpath'], skip_checks = True, - indexing_override = False, read_only = True, - xcache = False) - - # it is safe to consider that package dbs coming from repos - # contain only one entry - pkg_package_id = sorted(pkg_dbconn.listAllPackageIds(), - reverse = True)[0] - content = pkg_dbconn.retrieveContentIter( - pkg_package_id) - content_file = self.__generate_content_file( - content, package_id = self.pkgmeta['package_id'], - filter_splitdebug = True, - splitdebug = splitdebug, - splitdebug_dirs = splitdebug_dirs) - - # setup content safety metadata, get from package - content_safety = pkg_dbconn.retrieveContentSafetyIter( - pkg_package_id) - content_safety_file = self.__generate_content_safety_file( - content_safety) - - if self.pkgmeta['removepackage_id'] != -1 and \ - self.pkgmeta['removecontent_file'] is not None: - _merge_removecontent(inst_repo, pkg_dbconn, pkg_package_id) - - pkg_dbconn.close() - - # items_installed is useful to avoid the removal of installed - # files by __remove_package just because - # there's a difference in the directory path, perhaps, - # which is not handled correctly by - # EntropyRepository.contentDiff for obvious reasons - # (think about stuff in /usr/lib and /usr/lib64, - # where the latter is just a symlink to the former) - # -- - # fix removecontent, need to check if we just installed files - # that resolves at the same directory path (different symlink) - if self.pkgmeta['removecontent_file'] is not None: - self.__filter_out_files_installed_on_diff_path( - self.pkgmeta['removecontent_file'], - items_installed) - - # setup config_protect and config_protect_mask metadata before it's - # too late. - installed_package_id = self.pkgmeta['removepackage_id'] - if installed_package_id != -1: - self._setup_config_protect_and_mask( - inst_repo, installed_package_id) - - # filter out files not installed from content metadata - # these include splitdebug files, when splitdebug is - # disabled. - if items_not_installed: - def _filter(_path): - return _path not in items_not_installed - Package._filter_content_file( - content_file, _filter) - - # this is needed to make postinstall trigger work properly - self.pkgmeta['triggers']['install']['affected_directories'] = \ - self.pkgmeta['affected_directories'] - self.pkgmeta['triggers']['install']['affected_infofiles'] = \ - self.pkgmeta['affected_infofiles'] - - # always set data['injected'] to False - # installed packages database SHOULD never have more - # than one package for scope (key+slot) - data['injected'] = False - # spm counter will be set in self._install_package_into_spm_database() - data['counter'] = -1 - # branch must be always set properly, it could happen it's not - # when installing packages through their .tbz2s - data['branch'] = self._settings['repositories']['branch'] - # there is no need to store needed paths into db - if "needed_paths" in data: - del data['needed_paths'] - # there is no need to store changelog data into db - if "changelog" in data: - del data['changelog'] - # we don't want it to be added now, we want to add install source - # info too. - if "original_repository" in data: - del data['original_repository'] - # rewrite extra_download metadata with the currently provided, - # and accepted extra_download items (in case of splitdebug being - # disable, we're not going to add those entries, for example) - data['extra_download'] = self.pkgmeta['extra_download'] - - data['content'] = None - data['content_safety'] = None - try: - # now we are ready to craft a 'content' iter object - data['content'] = Content.FileContentReader( - content_file) - data['content_safety'] = Content.FileContentSafetyReader( - content_safety_file) - package_id = inst_repo.handlePackage( - data, revision = data['revision'], - formattedContent = True) - finally: - if data['content'] is not None: - try: - data['content'].close() - data['content'] = None - except (OSError, IOError): - data['content'] = None - if data['content_safety'] is not None: - try: - data['content_safety'].close() - data['content_safety'] = None - except (OSError, IOError): - data['content_safety'] = None - - # update datecreation - ctime = time.time() - inst_repo.setCreationDate(package_id, str(ctime)) - - # add idpk to the installedtable - inst_repo.dropInstalledPackageFromStore(package_id) - inst_repo.storeInstalledPackage(package_id, - self.pkgmeta['repository'], self.pkgmeta['install_source']) - - automerge_data = self.pkgmeta.get('configprotect_data') - if automerge_data: - inst_repo.insertAutomergefiles(package_id, automerge_data) - - inst_repo.commit() - - # replace current empty "content" metadata info - # content metadata is required by - # _spm_install_package() -> Spm.add_installed_package() - # in case of injected packages (SPM metadata might be - # incomplete). - self.pkgmeta['triggers']['install']['content'] = \ - Content.FileContentReader(content_file) - - return package_id - - def __fill_image_dir(self, merge_from, image_dir): - - dbconn = self._entropy.open_repository(self.pkgmeta['repository']) - # this is triggered by merge_from pkgmeta metadata - # even if repositories are allowed to not have content - # metadata, in this particular case, it is mandatory - contents = dbconn.retrieveContentIter( - self.pkgmeta['package_id'], order_by = "file") - - # collect files - for path, ftype in contents: - # convert back to filesystem str - encoded_path = path - path = os.path.join(merge_from, encoded_path[1:]) - topath = os.path.join(image_dir, encoded_path[1:]) - path = const_convert_to_rawstring(path) - topath = const_convert_to_rawstring(topath) - - try: - exist = os.lstat(path) - except OSError: - continue # skip file - - if 'dir' == ftype and \ - not stat.S_ISDIR(exist.st_mode) and \ - os.path.isdir(path): - # workaround for directory symlink issues - path = os.path.realpath(path) - - copystat = False - # if our directory is a symlink instead, then copy the symlink - if os.path.islink(path): - tolink = os.readlink(path) - if os.path.islink(topath): - os.remove(topath) - os.symlink(tolink, topath) - elif os.path.isdir(path): - if not os.path.isdir(topath): - os.makedirs(topath) - copystat = True - elif os.path.isfile(path): - if os.path.isfile(topath): - os.remove(topath) # should never happen - shutil.copy2(path, topath) - copystat = True - - if copystat: - user = os.stat(path)[stat.ST_UID] - group = os.stat(path)[stat.ST_GID] - os.chown(topath, user, group) - shutil.copystat(path, topath) - - del contents - - def _get_config_protect(self, entropy_repository, package_id, mask = False): - """ - Return configuration protection (or mask) metadata for the given - package. - This method should not be used as source for storing metadata into - repositories since the returned objects may not be decoded in utf-8. - Data returned by this method is expected to be used only by internal - functions. - """ - cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] - misc_data = self._settings[cl_id]['misc'] - - if mask: - paths = entropy_repository.retrieveProtectMask(package_id).split() - misc_key = "configprotectmask" - else: - paths = entropy_repository.retrieveProtect(package_id).split() - misc_key = "configprotect" - - root = etpConst['systemroot'] - config = set(("%s%s" % (root, path) for path in paths)) - config.update(misc_data[misc_key]) - - # os.* methods in Python 2.x do not expect unicode strings - # This set of data is only used by _handle_config_protect atm. - if not const_is_python3(): - config = set((const_convert_to_rawstring(x) for x in config)) - - return config - - def _get_config_protect_skip(self): - """ - Return the configuration protection path set. - """ - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - client_settings = self._settings[sys_set_plg_id] - misc_settings = client_settings['misc'] - - protectskip = misc_settings['configprotectskip'] - if not const_is_python3(): - protectskip = set(( - const_convert_to_rawstring( - x, from_enctype = etpConst['conf_encoding']) for x in - misc_settings['configprotectskip'])) - return protectskip - - def __get_sys_root(self): - return self.pkgmeta.get('unittest_root', '') + \ - etpConst['systemroot'] - - def _move_image_to_system(self, items_installed, items_not_installed): - - # load CONFIG_PROTECT and its mask - repo = self._entropy.open_repository(self.pkgmeta['repository']) - protect = self._get_config_protect(repo, self.pkgmeta['package_id']) - mask = self._get_config_protect(repo, self.pkgmeta['package_id'], - mask = True) - protectskip = self._get_config_protect_skip() - - # support for unit testing settings - sys_root = self.__get_sys_root() - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - misc_data = self._settings[sys_set_plg_id]['misc'] - col_protect = misc_data['collisionprotect'] - splitdebug, splitdebug_dirs = self.pkgmeta['splitdebug'], \ - self.pkgmeta['splitdebug_dirs'] - info_dirs = self._get_info_directories() - - # setup image_dir properly - image_dir = self.pkgmeta['imagedir'][:] - if not const_is_python3(): - # image_dir comes from unpackdir, which comes from download - # metadatum, which is utf-8 (conf_encoding) - image_dir = const_convert_to_rawstring(image_dir, - from_enctype = etpConst['conf_encoding']) - movefile = entropy.tools.movefile - - def workout_subdir(currentdir, subdir): - - imagepath_dir = os.path.join(currentdir, subdir) - rel_imagepath_dir = imagepath_dir[len(image_dir):] - rootdir = sys_root + rel_imagepath_dir - - # splitdebug (.debug files) support - # If splitdebug is not enabled, do not create splitdebug directories - # and move on instead (return) - if not splitdebug: - for split_dir in splitdebug_dirs: - if rootdir.startswith(split_dir): - # also drop item from content metadata. In this way - # SPM has in sync information on what the package - # content really is. - # --- - # we should really use unicode - # strings for items_not_installed - unicode_rootdir = const_convert_to_unicode(rootdir) - items_not_installed.add(unicode_rootdir) - return - - # handle broken symlinks - if os.path.islink(rootdir) and not os.path.exists(rootdir): - # broken symlink - os.remove(rootdir) - - # if our directory is a file on the live system - elif os.path.isfile(rootdir): # really weird...! - - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! %s is a file when it should be " \ - "a directory !! Removing in 20 seconds..." % (rootdir,) - ) - mytxt = darkred(_("%s is a file when should be a " \ - "directory !! Removing in 20 seconds...") % (rootdir,)) - - self._entropy.output( - red("QA: ")+mytxt, - importance = 1, - level = "warning", - header = red(" !!! ") - ) - os.remove(rootdir) - - # if our directory is a symlink instead, then copy the symlink - if os.path.islink(imagepath_dir): - - # if our live system features a directory instead of - # a symlink, we should consider removing the directory - if not os.path.islink(rootdir) and os.path.isdir(rootdir): - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! %s is a directory when it should be " \ - "a symlink !! Removing in 20 seconds..." % ( - rootdir,) - ) - mytxt = "%s: %s" % ( - _("directory expected, symlink found"), - rootdir, - ) - mytxt2 = _("Removing in 20 seconds !!") - for txt in (mytxt, mytxt2,): - self._entropy.output( - darkred("QA: ") + darkred(txt), - importance = 1, - level = "warning", - header = red(" !!! ") - ) - - # fucking kill it in any case! - # rootdir must die! die die die die! - # /me brings chainsaw - try: - shutil.rmtree(rootdir, True) - except (shutil.Error, OSError,) as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to rm %s " \ - "directory ! [workout_subdir/1]: %s" % ( - rootdir, err, - ) - ) - - tolink = os.readlink(imagepath_dir) - live_tolink = None - if os.path.islink(rootdir): - live_tolink = os.readlink(rootdir) - - if tolink != live_tolink: - _symfail = False - if os.path.lexists(rootdir): - # at this point, it must be a file - try: - os.remove(rootdir) - except OSError as err: - _symfail = True - # must be atomic, too bad if it fails - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to remove %s " \ - "file ! [workout_file/0]: %s" % ( - rootdir, err, - ) - ) - msg = _("Cannot remove symlink") - mytxt = "%s: %s => %s" % ( - purple(msg), - blue(rootdir), - repr(err), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = brown(" ## ") - ) - if not _symfail: - os.symlink(tolink, rootdir) - - elif not os.path.isdir(rootdir): - # directory not found, we need to create it - - try: - # really force a simple mkdir first of all - os.mkdir(rootdir) - except OSError: - os.makedirs(rootdir) - - - if not os.path.islink(rootdir): - - # symlink doesn't need permissions, also - # until os.walk ends they might be broken - user = os.stat(imagepath_dir)[stat.ST_UID] - group = os.stat(imagepath_dir)[stat.ST_GID] - try: - os.chown(rootdir, user, group) - shutil.copystat(imagepath_dir, rootdir) - except (OSError, IOError) as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Error during workdir setup " \ - "%s, %s, errno: %s" % ( - rootdir, - err, - err.errno, - ) - ) - # skip some errors because we may have - # unwritable directories - if err.errno not in ( - errno.EPERM, errno.ENOENT, - errno.ENOTDIR): - mytxt = "%s: %s, %s, %s" % ( - brown("Error during workdir setup"), - purple(rootdir), err, - err.errno - ) - self._entropy.output( - mytxt, - importance = 1, - level = "error", - header = darkred(" !!! ") - ) - return 4 - - item_dir, item_base = os.path.split(rootdir) - item_dir = os.path.realpath(item_dir) - item_inst = os.path.join(item_dir, item_base) - item_inst = const_convert_to_unicode(item_inst) - items_installed.add(item_inst) - - - def workout_file(currentdir, item): - - fromfile = os.path.join(currentdir, item) - rel_fromfile = fromfile[len(image_dir):] - rel_fromfile_dir = os.path.dirname(rel_fromfile) - tofile = sys_root + rel_fromfile - - rel_fromfile_dir_utf = const_convert_to_unicode( - rel_fromfile_dir) - self.pkgmeta['affected_directories'].add( - rel_fromfile_dir_utf) - - # account for info files, if any - if rel_fromfile_dir_utf in info_dirs: - rel_fromfile_utf = const_convert_to_unicode( - rel_fromfile) - for _ext in self._INFO_EXTS: - if rel_fromfile_utf.endswith(_ext): - self.pkgmeta['affected_infofiles'].add( - rel_fromfile_utf) - break - - # splitdebug (.debug files) support - # If splitdebug is not enabled, do not create - # splitdebug directories and move on instead (return) - if not splitdebug: - for split_dir in splitdebug_dirs: - if tofile.startswith(split_dir): - # also drop item from content metadata. In this way - # SPM has in sync information on what the package - # content really is. - # --- - # we should really use unicode - # strings for items_not_installed - unicode_tofile = const_convert_to_unicode(tofile) - items_not_installed.add(unicode_tofile) - return 0 - - if col_protect > 1: - todbfile = fromfile[len(image_dir):] - myrc = self._handle_install_collision_protect(tofile, - todbfile) - if not myrc: - return 0 - - prot_old_tofile = tofile[len(sys_root):] - # configprotect_data is passed to insertAutomergefiles() - # which always expects unicode data. - # revert back to unicode (we previously called encode on - # image_dir (which is passed to os.walk, which generates - # raw strings) - prot_old_tofile = const_convert_to_unicode(prot_old_tofile) - - pre_tofile = tofile[:] - (in_mask, protected, - tofile, do_return) = self._handle_config_protect( - protect, mask, protectskip, fromfile, tofile) - - # collect new config automerge data - if in_mask and os.path.exists(fromfile): - try: - prot_md5 = const_convert_to_unicode( - entropy.tools.md5sum(fromfile)) - self.pkgmeta['configprotect_data'].append( - (prot_old_tofile, prot_md5,)) - except (IOError,) as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to get md5 of %s " \ - "file ! [workout_file/1]: %s" % ( - fromfile, err, - ) - ) - - # check if it's really necessary to protect file - if protected: - - # second task - # prot_old_tofile is always unicode, it must be, see above - oldprot_md5 = self.pkgmeta['already_protected_config_files'].get( - prot_old_tofile) - - if oldprot_md5: - - try: - in_system_md5 = entropy.tools.md5sum(pre_tofile) - except (OSError, IOError) as err: - if err.errno != errno.ENOENT: - raise - in_system_md5 = "?" - - if oldprot_md5 == in_system_md5: - # we can merge it, files, even if - # contains changes have not been modified - # by the user - msg = _("Automerging config file, never modified") - mytxt = "%s: %s" % ( - darkgreen(msg), - blue(pre_tofile), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - protected = False - do_return = False - tofile = pre_tofile - - if do_return: - return 0 - - try: - from_r_path = os.path.realpath(fromfile) - except RuntimeError: - # circular symlink, fuck! - # really weird...! - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! %s is a circular symlink !!!" % (fromfile,) - ) - mytxt = "%s: %s" % ( - _("Circular symlink issue"), - const_convert_to_unicode(fromfile), - ) - self._entropy.output( - darkred("QA: ") + darkred(mytxt), - importance = 1, - level = "warning", - header = red(" !!! ") - ) - from_r_path = fromfile - - try: - to_r_path = os.path.realpath(tofile) - except RuntimeError: - # circular symlink, fuck! - # really weird...! - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! %s is a circular symlink !!!" % (tofile,) - ) - mytxt = "%s: %s" % ( - _("Circular symlink issue"), - const_convert_to_unicode(tofile), - ) - self._entropy.output( - darkred("QA: ") + darkred(mytxt), - importance = 1, - level = "warning", - header = red(" !!! ") - ) - to_r_path = tofile - - if from_r_path == to_r_path and os.path.islink(tofile): - # there is a serious issue here, better removing tofile, - # happened to someone. - - try: - # try to cope... - os.remove(tofile) - except (OSError, IOError,) as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to cope to oddity of %s " \ - "file ! [workout_file/2]: %s" % ( - tofile, err, - ) - ) - - # if our file is a dir on the live system - if os.path.isdir(tofile) and not os.path.islink(tofile): - - # really weird...! - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! %s is a directory when it should " \ - "be a file !! Removing in 20 seconds..." % (tofile,) - ) - - mytxt = "%s: %s" % ( - _("file expected, directory found"), - const_convert_to_unicode(tofile), - ) - mytxt2 = _("Removing in 20 seconds !!") - for txt in (mytxt, mytxt2,): - self._entropy.output( - darkred("QA: ") + darkred(txt), - importance = 1, - level = "warning", - header = red(" !!! ") - ) - - try: - shutil.rmtree(tofile, True) - except (shutil.Error, IOError,) as err: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to cope to oddity of %s " \ - "file ! [workout_file/3]: %s" % ( - tofile, err, - ) - ) - - # moving file using the raw format - try: - done = movefile(fromfile, tofile, src_basedir = image_dir) - except (IOError,) as err: - # try to move forward, sometimes packages might be - # fucked up and contain broken things - if err.errno not in (errno.ENOENT, errno.EACCES,): - raise - - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Error during file move" \ - " to system: %s => %s | IGNORED: %s" % ( - const_convert_to_unicode(fromfile), - const_convert_to_unicode(tofile), - err, - ) - ) - done = True - - if not done: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Error during file move" \ - " to system: %s => %s" % (fromfile, tofile,) - ) - mytxt = "%s: %s => %s, %s" % ( - _("File move error"), - const_convert_to_unicode(fromfile), - const_convert_to_unicode(tofile), - _("please report"), - ) - self._entropy.output( - red("QA: ")+darkred(mytxt), - importance = 1, - level = "warning", - header = red(" !!! ") - ) - return 4 - - item_dir = os.path.realpath(os.path.dirname(tofile)) - item_inst = os.path.join(item_dir, os.path.basename(tofile)) - item_inst = const_convert_to_unicode(item_inst) - items_installed.add(item_inst) - - if protected and \ - os.getenv("ENTROPY_CLIENT_ENABLE_OLD_FILEUPDATES"): - # add to disk cache - file_updates = self._entropy.PackageFileUpdates() - file_updates.add(tofile, quiet = True) - - return 0 - - # merge data into system - for currentdir, subdirs, files in os.walk(image_dir): - - # create subdirs - for subdir in subdirs: - workout_subdir(currentdir, subdir) - - for item in files: - move_st = workout_file(currentdir, item) - if move_st != 0: - return move_st - - return 0 - - def __filter_out_files_installed_on_diff_path(self, content_file, - installed_content): - """ - Use case: if a package provided files in /lib then, a new version - of that package moved the same files under /lib64, we need to check - if both directory paths solve to the same inode and if so, - add to our set that we're going to return. - """ - sys_root = self.__get_sys_root() - second_pass_removal = set() - - if not installed_content: - # nothing to filter, no-op - return - def _main_filter(_path): - item_dir = os.path.dirname("%s%s" % ( - sys_root, _path,)) - item = os.path.join( - os.path.realpath(item_dir), - os.path.basename(_path)) - if item in installed_content: - second_pass_removal.add(item) - return False - return True - - # first pass, remove direct matches, schedule a second pass - # list of files - Package._filter_content_file(content_file, _main_filter) - - if not second_pass_removal: - # done then - return - - # second pass, drop remaining files - # unfortunately, this is the only way to work it out - # with iterators - def _filter(_path): - return _path not in second_pass_removal - Package._filter_content_file(content_file, _filter) - - def _handle_config_protect(self, protect, mask, protectskip, - fromfile, tofile, - do_allocation_check = True, - do_quiet = False): - """ - Handle configuration file protection. This method contains the logic - for determining if a file should be protected from overwrite. - """ - protected = False - do_continue = False - in_mask = False - - if tofile in protect: - protected = True - in_mask = True - - elif os.path.dirname(tofile) in protect: - protected = True - in_mask = True - - else: - tofile_testdir = os.path.dirname(tofile) - old_tofile_testdir = None - while tofile_testdir != old_tofile_testdir: - if tofile_testdir in protect: - protected = True - in_mask = True - break - old_tofile_testdir = tofile_testdir - tofile_testdir = os.path.dirname(tofile_testdir) - - if protected: # check if perhaps, file is masked, so unprotected - - if tofile in mask: - protected = False - in_mask = False - - elif os.path.dirname(tofile) in mask: - protected = False - in_mask = False - - else: - tofile_testdir = os.path.dirname(tofile) - old_tofile_testdir = None - while tofile_testdir != old_tofile_testdir: - if tofile_testdir in mask: - protected = False - in_mask = False - break - old_tofile_testdir = tofile_testdir - tofile_testdir = os.path.dirname(tofile_testdir) - - if not os.path.lexists(tofile): - protected = False # file doesn't exist - - # check if it's a text file - if protected: - protected = entropy.tools.istextfile(tofile) - in_mask = protected - - if fromfile is not None: - if protected and os.path.lexists(fromfile) and \ - (not os.path.exists(fromfile)) and os.path.islink(fromfile): - # broken symlink, don't protect - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Failed to handle file protection for: " \ - "%s, broken symlink in package" % ( - tofile, - ) - ) - msg = _("Cannot protect broken symlink") - mytxt = "%s:" % ( - purple(msg), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = brown(" ## ") - ) - self._entropy.output( - tofile, - level = "warning", - header = brown(" ## ") - ) - protected = False - - if not protected: - return in_mask, protected, tofile, do_continue - - ## ## - # file is protected # - ##__________________## - - # check if protection is disabled for this element - if tofile in protectskip: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Skipping config file installation/removal, " \ - "as stated in client.conf: %s" % (tofile,) - ) - if not do_quiet: - mytxt = "%s: %s" % ( - _("Skipping file installation/removal"), - tofile, - ) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = darkred(" ## ") - ) - do_continue = True - return in_mask, protected, tofile, do_continue - - ## ## - # file is protected (2) # - ##______________________## - - prot_status = True - if do_allocation_check: - spm_class = self._entropy.Spm_class() - tofile, prot_status = spm_class.allocate_protected_file(fromfile, - tofile) - - if not prot_status: - # a protected file with the same content - # is already in place, so not going to protect - # the same file twice - protected = False - return in_mask, protected, tofile, do_continue - - ## ## - # file is protected (3) # - ##______________________## - - oldtofile = tofile - if oldtofile.find("._cfg") != -1: - oldtofile = os.path.join(os.path.dirname(oldtofile), - os.path.basename(oldtofile)[10:]) - - if not do_quiet: - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Protecting config file: %s" % (oldtofile,) - ) - mytxt = red("%s: %s") % (_("Protecting config file"), oldtofile,) - self._entropy.output( - mytxt, - importance = 1, - level = "warning", - header = darkred(" ## ") - ) - - return in_mask, protected, tofile, do_continue - - - def _handle_install_collision_protect(self, tofile, todbfile): - - avail = self._entropy.installed_repository().isFileAvailable( - const_convert_to_unicode(todbfile), get_id = True) - - if (self.pkgmeta['removepackage_id'] not in avail) and avail: - mytxt = darkred(_("Collision found during install for")) - mytxt += " %s - %s" % ( - blue(tofile), - darkred(_("cannot overwrite")), - ) - self._entropy.output( - red("QA: ")+mytxt, - importance = 1, - level = "warning", - header = darkred(" ## ") - ) - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "WARNING!!! Collision found during install " \ - "for %s - cannot overwrite" % (tofile,) - ) - return False - - return True - - def _sources_fetch_step(self): - - down_data = self.pkgmeta['download'] - down_keys = list(down_data.keys()) - d_cache = set() - rc = 0 - key_cache = [os.path.basename(x) for x in down_keys] - - for key in sorted(down_keys): - - key_name = os.path.basename(key) - if key_name in d_cache: - continue - # first fine wins - - keyboard_interrupt = False - for url in down_data[key]: - - file_name = os.path.basename(url) - if self.pkgmeta.get('fetch_path'): - dest_file = os.path.join(self.pkgmeta['fetch_path'], - file_name) - else: - dest_file = os.path.join(self.pkgmeta['unpackdir'], - file_name) - - try: - rc = self._fetch_source(url, dest_file) - except KeyboardInterrupt: - keyboard_interrupt = True - break - - if rc == -100: - keyboard_interrupt = True - break - - if rc == 0: - d_cache.add(key_name) - break - - if keyboard_interrupt: - rc = 1 - break - - key_cache.remove(key_name) - if rc != 0 and key_name not in key_cache: - break - - rc = 0 - - return rc - - def _fetch_source(self, url, dest_file): - rc = 1 - - mytxt = "%s: %s" % (blue(_("Downloading")), brown(url),) - # now fetch the new one - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - rc, data_transfer, _resumed = self.__fetch_file( - url, - dest_file, - digest = None, - resume = False - ) - if rc == 0: - mytxt = blue("%s: ") % (_("Successfully downloaded from"),) - mytxt += red(self._get_url_name(url)) - human_bytes = entropy.tools.bytes_into_human(data_transfer) - mytxt += " %s %s/%s" % (_("at"), human_bytes, _("second"),) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - self._entropy.output( - "%s: %s" % (blue(_("Local path")), brown(dest_file),), - importance = 1, - level = "info", - header = red(" # ") - ) - else: - error_message = blue("%s: %s") % ( - _("Error downloading from"), - red(self._get_url_name(url)), - ) - # something bad happened - if rc == -1: - error_message += " - %s." % ( - _("file not available on this mirror"), - ) - elif rc == -3: - error_message += " - not found." - elif rc == -100: - error_message += " - %s." % (_("discarded download"),) - else: - error_message += " - %s: %s" % (_("unknown reason"), rc,) - self._entropy.output( - error_message, - importance = 1, - level = "warning", - header = red(" ## ") - ) - - return rc - - def _fetch_step(self): - - if self.pkgmeta['verified']: - return 0 - - def _fetch(download, checksum): - mytxt = "%s: %s" % (blue(_("Downloading")), - red(os.path.basename(download)),) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - pkg_disk_path = self.__get_fetch_disk_path(download) - return self._download_package( - self.pkgmeta['package_id'], - self.pkgmeta['repository'], - download, - pkg_disk_path, - checksum - ) - - rc = _fetch(self.pkgmeta['download'], self.pkgmeta['checksum']) - if rc == 0: - # go ahead with extra_download - for extra_download in self.pkgmeta['extra_download']: - rc = _fetch(extra_download['download'], - extra_download['md5']) - if rc != 0: - break - - if rc == 0: - return 0 - - mytxt = "%s. %s: %s" % ( - red(_("Package cannot be fetched. Try to update repositories")), - blue(_("Error")), - rc, - ) - self._entropy.output( - mytxt, - importance = 1, - level = "error", - header = darkred(" ## ") - ) - - return rc - - def _multi_fetch_step(self): - - m_fetch_len = len(self.pkgmeta['multi_fetch_list']) - mytxt = "%s: %s %s" % ( - blue(_("Downloading")), - darkred(str(m_fetch_len)), - ngettext("archive", "archives", m_fetch_len), - ) - - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - rc, err_list = self._download_packages( - self.pkgmeta['multi_fetch_list'], - self.pkgmeta['checksum'] - ) - - if rc == 0: - return 0 - - mytxt = _("Some packages cannot be fetched") - mytxt2 = _("Try to update your repositories and retry") - mytxt3 = "%s: %s" % (brown(_("Error")), bold(str(rc)),) - for txt in (mytxt, mytxt2,): - self._entropy.output( - "%s." % (darkred(txt),), - importance = 0, - level = "info", - header = red(" ## ") - ) - self._entropy.output( - mytxt3, - importance = 0, - level = "info", - header = red(" ## ") - ) - - for _pkg_id, repo, fname, cksum, _signatures in err_list: - self._entropy.output( - "[%s|%s] %s" % (blue(repo), darkgreen(cksum), darkred(fname),), - importance = 1, - level = "error", - header = darkred(" # ") - ) - - return rc - - def _fetch_not_available_step(self): - self._entropy.output( - blue(_("Package cannot be downloaded, unknown error.")), - importance = 1, - level = "info", - header = red(" ## ") - ) - return 0 - - def _vanished_step(self): - self._entropy.output( - blue(_("Installed package in queue vanished, skipping.")), - importance = 1, - level = "info", - header = red(" ## ") - ) - return 0 - - def _checksum_step(self): - base_pkg_rc = self._match_checksum(self.pkgmeta['package_id'], - self.pkgmeta['repository'], self.pkgmeta['checksum'], - self.pkgmeta['download'], self.pkgmeta['signatures']) - if base_pkg_rc != 0: - return base_pkg_rc - # now go with extra_download - for extra_download in self.pkgmeta['extra_download']: - download = extra_download['download'] - checksum = extra_download['md5'] - signatures = { - 'sha1': extra_download['sha1'], - 'sha256': extra_download['sha256'], - 'sha512': extra_download['sha512'], - 'gpg': extra_download['gpg'], - } - extra_rc = self._match_checksum(self.pkgmeta['package_id'], - self.pkgmeta['repository'], checksum, download, signatures) - if extra_rc != 0: - return extra_rc - self.pkgmeta['verified'] = True - return 0 - - def _multi_checksum_step(self): - return self.multi_match_checksum() - - def _merge_from_unpack_step(self): - - mytxt = "%s: %s" % ( - blue(_("Merging package")), - red(os.path.basename(self.pkgmeta['atom'])), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Merging package: %s" % (self.pkgmeta['atom'],) - ) - self.__fill_image_dir(self.pkgmeta['merge_from'], - self.pkgmeta['imagedir']) - spm_class = self._entropy.Spm_class() - # call Spm unpack hook - return spm_class.entropy_install_unpack_hook(self._entropy, - self.pkgmeta) - - def _unpack_step(self): - - unpack_dir = self.pkgmeta['unpackdir'] - if not const_is_python3(): - # unpackdir comes from download metadatum, which is utf-8 - # (conf_encoding) - unpack_dir = const_convert_to_rawstring(unpack_dir, - from_enctype = etpConst['conf_encoding']) - - if os.path.isdir(unpack_dir): - # this, if Python 2.x, must be fed with rawstrings - shutil.rmtree(unpack_dir) - elif os.path.isfile(unpack_dir): - os.remove(unpack_dir) - os.makedirs(unpack_dir) - - rc = self.__unpack_package(self.pkgmeta['download'], - self.pkgmeta['pkgpath'], self.pkgmeta['imagedir'], - self.pkgmeta['pkgdbpath']) - - if rc == 0: - for extra_download in self.pkgmeta['extra_download']: - download = extra_download['download'] - pkgpath = self.__get_fetch_disk_path(download) - rc = self.__unpack_package(download, pkgpath, - self.pkgmeta['imagedir'], None) - if rc != 0: - break - - if rc != 0: - if rc == 512: - errormsg = "%s. %s. %s: 512" % ( - red(_("You are running out of disk space")), - red(_("I bet, you're probably Michele")), - blue(_("Error")), - ) - else: - msg = _("An error occured while trying to unpack the package") - errormsg = "%s. %s. %s: %s" % ( - red(msg), - red(_("Check if your system is healthy")), - blue(_("Error")), - rc, - ) - self._entropy.output( - errormsg, - importance = 1, - level = "error", - header = red(" ## ") - ) - return rc - - spm_class = self._entropy.Spm_class() - # call Spm unpack hook - return spm_class.entropy_install_unpack_hook(self._entropy, - self.pkgmeta) - - def _install_step(self): - mytxt = "%s: %s" % ( - blue(_("Installing package")), - red(self.pkgmeta['atom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - if self.pkgmeta.get('description'): - mytxt = "[%s]" % (purple(self.pkgmeta.get('description')),) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - if self.pkgmeta['splitdebug']: - if self.pkgmeta.get('splitdebug_pkgfile'): - mytxt = "[%s]" % ( - teal(_("unsupported splitdebug usage (package files)")),) - level = "warning" - else: - mytxt = "[%s]" % ( - teal(_("<3 debug files installation enabled <3")),) - level = "info" - self._entropy.output( - mytxt, - importance = 1, - level = level, - header = red(" ## ") - ) - - rc = self.__install_package() - if rc != 0: - mytxt = "%s. %s. %s: %s" % ( - red(_("An error occured while trying to install the package")), - red(_("Check if your system is healthy")), - blue(_("Error")), - rc, - ) - self._entropy.output( - mytxt, - importance = 1, - level = "error", - header = red(" ## ") - ) - return rc - - def _remove_step(self): - mytxt = "%s: %s" % ( - blue(_("Removing data")), - red(self.pkgmeta['removeatom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - rc = self.__remove_package() - if rc != 0: - mytxt = "%s. %s. %s: %s" % ( - red(_("An error occured while trying to remove the package")), - red(_("Check if you have enough disk space on your hard disk")), - blue(_("Error")), - rc, - ) - self._entropy.output( - mytxt, - importance = 1, - level = "error", - header = red(" ## ") - ) - return rc - - def _cleanup_step(self): - mytxt = "%s: %s" % ( - blue(_("Cleaning")), - red(self.pkgmeta['atom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - self._cleanup_package(self.pkgmeta['unpackdir']) - # we don't care if cleanupPackage fails since it's not critical - return 0 - - def _post_install_step(self): - pkgdata = self.pkgmeta['triggers'].get('install') - action_data = self.pkgmeta['triggers'].get('install') - code = 0 - - if pkgdata: - trigger = self._entropy.Triggers( - self._action, "postinstall", - pkgdata, action_data) - do = trigger.prepare() - if do: - code = trigger.run() - trigger.kill() - - del pkgdata - return code - - def _pre_install_step(self): - pkgdata = self.pkgmeta['triggers'].get('install') - action_data = self.pkgmeta['triggers'].get('install') - code = 0 - - if pkgdata: - trigger = self._entropy.Triggers( - self._action, "preinstall", - pkgdata, action_data) - do = trigger.prepare() - if do: - code = trigger.run() - trigger.kill() - - del pkgdata - return code - - def _setup_step(self): - pkgdata = self.pkgmeta['triggers'].get('install') - action_data = self.pkgmeta['triggers'].get('install') - code = 0 - - if pkgdata: - trigger = self._entropy.Triggers( - self._action, "setup", - pkgdata, action_data) - do = trigger.prepare() - if do: - code = trigger.run() - trigger.kill() - - del pkgdata - if code != 0: - return code - - # NOTE: fixup permissions in the image directory - # the setup phase could have created additional users and groups - package_path = self.pkgmeta['pkgpath'] - prefix_dir = self.pkgmeta['imagedir'] - try: - entropy.tools.apply_tarball_ownership(package_path, prefix_dir) - except IOError as err: - msg = "%s: %s" % ( - brown(_("Error during package files permissions setup")), - err,) - self._entropy.output( - msg, - importance = 1, - level = "error", - header = darkred(" !!! ") - ) - return 1 - - return 0 - - def _pre_remove_step(self): - remdata = self.pkgmeta['triggers'].get('remove') - action_data = self.pkgmeta['triggers'].get('install') - code = 0 - - if remdata: - trigger = self._entropy.Triggers( - self._action, 'preremove', remdata, - action_data) - do = trigger.prepare() - if do: - code = trigger.run() - trigger.kill() - - del remdata - return code - - def _package_install_clean(self): - """ - Cleanup package files not used anymore by newly installed version. - This is part of the atomic install, which overwrites the live fs with - new files and removes old afterwards. - """ - self._entropy.output( - blue(_("Cleaning previously installed application data.")), - importance = 1, - level = "info", - header = red(" ## ") - ) - - self._remove_content_from_system( - automerge_metadata = self.pkgmeta['already_protected_config_files']) - - return 0 - - def _post_remove_step_install(self): - """ - Post-remove phase of package install, this step removes older SPM - package entries from SPM db. - """ - self._entropy.logger.log( - "[Package]", - etpConst['logging']['normal_loglevel_id'], - "Remove old package (spm data): %s" % (self.pkgmeta['removeatom'],) - ) - return self._spm_remove_package() - - def _post_remove_step_remove(self): - """ - Post-remove phase of package remove action, this step removes SPM - package entries if there are no other Entropy-tagged packages installed. - """ - # remove pkg - # -- now it's possible to remove SPM package entry. - # if another package with the same atom is installed in - # Entropy db, do not call SPM at all because it would cause - # to get that package removed from there resulting in matching - # inconsistencies. - # -- of course, we need to drop versiontag before being able to look - # for other pkgs with same atom but different tag (which is an - # entropy-only metadatum) - test_atom = entropy.dep.remove_tag(self.pkgmeta['removeatom']) - others_installed = self._entropy.installed_repository().getPackageIds( - test_atom) - - # It's obvious that clientdb cannot have more than one package_id - # featuring the same "atom" value, but still, let's be fault-tolerant. - spm_rc = 0 - - if not others_installed: - spm_rc = self._spm_remove_package() - - for other_installed in others_installed: - # we have one installed, we need to update SPM uid - spm_rc = self._spm_update_package_uid(other_installed, - self.pkgmeta['removeatom']) - if spm_rc != 0: - break # ohi ohi ohi - - return spm_rc - - def _post_remove_step(self): - remdata = self.pkgmeta['triggers'].get('remove') - action_data = self.pkgmeta['triggers'].get('install') - code = 0 - - if remdata: - trigger = self._entropy.Triggers( - self._action, "postremove", - remdata, action_data) - do = trigger.prepare() - if do: - code = trigger.run() - trigger.kill() - - del remdata - return code - - def _removeconflict_step(self): - - inst_repo = self._entropy.installed_repository() - confl_package_ids = [x for x in self.pkgmeta['conflicts'] if \ - inst_repo.isPackageIdAvailable(x)] - if not confl_package_ids: - return 0 - - # calculate removal dependencies - # system_packages must be False because we should not exclude - # them from the dependency tree in any case. Also, we cannot trigger - # DependenciesNotRemovable() exception, too. - proposed_pkg_ids = self._entropy.get_removal_queue(confl_package_ids, - system_packages = False) - # we don't want to remove the whole inverse dependencies of course, - # but just the conflicting ones, in a proper order - package_ids = [x for x in proposed_pkg_ids if x in confl_package_ids] - # make sure that every package is listed in package_ids before - # proceeding, cannot keep packages behind anyway, and must be fault - # tolerant. Besides, having missing packages here should never happen. - package_ids += [x for x in confl_package_ids if x not in \ - package_ids] - - for package_id in package_ids: - - pkg = self._entropy.Package() - pkg.prepare((package_id,), "remove_conflict", - self.pkgmeta['remove_metaopts']) - - rc = pkg.run(xterm_header = self._xterm_title) - pkg.kill() - if rc != 0: - return rc - - return 0 - - def _config_step(self): - - mytxt = "%s: %s" % ( - blue(_("Configuring package")), - red(self.pkgmeta['atom']), - ) - self._entropy.output( - mytxt, - importance = 1, - level = "info", - header = red(" ## ") - ) - - conf_rc = self.__configure_package() - if conf_rc == 1: - mytxt = _("An error occured while trying to configure the package") - mytxt2 = "%s. %s: %s" % ( - red(_("Make sure that your system is healthy")), - blue(_("Error")), - conf_rc, - ) - self._entropy.output( - darkred(mytxt), - importance = 1, - level = "error", - header = red(" ## ") - ) - self._entropy.output( - mytxt2, - importance = 1, - level = "error", - header = red(" ## ") - ) - - elif conf_rc == 2: - mytxt = _("An error occured while trying to configure the package") - mytxt2 = "%s. %s: %s" % ( - red(_("It seems that Source Package Manager entry is missing")), - blue(_("Error")), - conf_rc, - ) - self._entropy.output( - darkred(mytxt), - importance = 1, - level = "error", - header = red(" ## ") - ) - self._entropy.output( - mytxt2, - importance = 1, - level = "error", - header = red(" ## ") - ) - - return conf_rc - - def _stepper(self, xterm_header): - if xterm_header is None: - xterm_header = "" - - if 'remove_installed_vanished' in self.pkgmeta: - self._xterm_title += ' %s' % (_("Installed package vanished"),) - self._entropy.set_title(self._xterm_title) - return self._vanished_step() - - if 'fetch_not_available' in self.pkgmeta: - self._xterm_title += ' %s' % (_("Fetch not available"),) - self._entropy.set_title(self._xterm_title) - return self._fetch_not_available_step() - - def do_fetch(): - self._xterm_title += ' %s: %s' % ( - _("Fetching"), - os.path.basename(self.pkgmeta['download']), - ) - self._entropy.set_title(self._xterm_title) - return self._fetch_step() - - def do_multi_fetch(): - m_fetch_len = len(self.pkgmeta['multi_fetch_list']) / 2 - self._xterm_title += ' %s: %s %s' % (_("Multi Fetching"), - m_fetch_len, ngettext("package", "packages", m_fetch_len),) - self._entropy.set_title(self._xterm_title) - return self._multi_fetch_step() - - def do_sources_fetch(): - self._xterm_title += ' %s: %s' % ( - _("Fetching sources"), - os.path.basename(self.pkgmeta['atom']),) - self._entropy.set_title(self._xterm_title) - return self._sources_fetch_step() - - def do_checksum(): - self._xterm_title += ' %s: %s' % (_("Verifying"), - os.path.basename(self.pkgmeta['download']),) - self._entropy.set_title(self._xterm_title) - return self._checksum_step() - - def do_multi_checksum(): - m_checksum_len = len(self.pkgmeta['multi_checksum_list']) - self._xterm_title += ' %s: %s %s' % ( - _("Multi Verification"), - m_checksum_len, - ngettext("package", "packages", m_checksum_len),) - self._entropy.set_title(self._xterm_title) - return self._multi_checksum_step() - - def do_unpack(): - if self.pkgmeta['merge_from']: - mytxt = _("Merging") - self._xterm_title += ' %s: %s' % ( - mytxt, - os.path.basename(self.pkgmeta['atom']), - ) - self._entropy.set_title(self._xterm_title) - return self._merge_from_unpack_step() - - mytxt = _("Unpacking") - self._xterm_title += ' %s: %s' % ( - mytxt, - os.path.basename(self.pkgmeta['download']), - ) - self._entropy.set_title(self._xterm_title) - return self._unpack_step() - - def do_install(): - self._xterm_title += ' %s: %s' % ( - _("Installing"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._install_step() - - def do_install_spm(): - return self._spm_install_package( - self.pkgmeta['installed_package_id']) - - def do_remove(): - self._xterm_title += ' %s: %s' % ( - _("Removing"), - self.pkgmeta['removeatom'], - ) - self._entropy.set_title(self._xterm_title) - return self._remove_step() - - def do_cleanup(): - self._xterm_title += ' %s: %s' % ( - _("Cleaning"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._cleanup_step() - - def do_postinstall(): - self._xterm_title += ' %s: %s' % ( - _("Postinstall"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._post_install_step() - - def do_setup(): - self._xterm_title += ' %s: %s' % ( - _("Setup"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._setup_step() - - def do_preinstall(): - self._xterm_title += ' %s: %s' % ( - _("Preinstall"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._pre_install_step() - - def do_preremove(): - self._xterm_title += ' %s: %s' % ( - _("Preremove"), - self.pkgmeta['removeatom'], - ) - self._entropy.set_title(self._xterm_title) - return self._pre_remove_step() - - def do_postremove(): - self._xterm_title += ' %s: %s' % ( - _("Postremove"), - self.pkgmeta['removeatom'], - ) - self._entropy.set_title(self._xterm_title) - return self._post_remove_step() - - def do_config(): - self._xterm_title += ' %s: %s' % ( - _("Configuring"), - self.pkgmeta['atom'], - ) - self._entropy.set_title(self._xterm_title) - return self._config_step() - - steps_data = { - "fetch": do_fetch, - "multi_fetch": do_multi_fetch, - "multi_checksum": do_multi_checksum, - "sources_fetch": do_sources_fetch, - "checksum": do_checksum, - "unpack": do_unpack, - "remove_conflicts": self._removeconflict_step, - "install": do_install, - "install_spm": do_install_spm, - "install_clean": self._package_install_clean, - "remove": do_remove, - "cleanup": do_cleanup, - "postinstall": do_postinstall, - "setup": do_setup, - "preinstall": do_preinstall, - "postremove": do_postremove, - "postremove_install": self._post_remove_step_install, - "postremove_remove": self._post_remove_step_remove, - "preremove": do_preremove, - "config": do_config, - } - - rc = 0 - for step in self.pkgmeta['steps']: - self._xterm_title = xterm_header - rc = steps_data.get(step)() - if rc != 0: - break - return rc - - - def run(self, xterm_header = None): - self._error_on_not_prepared() - - gave_up = self._entropy.wait_resources() - if gave_up: - return 20 - # resources acquired - - try: - rc = self._stepper(xterm_header) - finally: - self._entropy.unlock_resources() - - if rc != 0: - self._entropy.output( - blue(_("An error occured. Action aborted.")), - importance = 2, - level = "error", - header = darkred(" ## ") - ) - return rc + self.metaopts = {} def prepare(self, package_match, action, metaopts = None): - self._error_on_prepared() + """ + Backward compatible method. + """ + if action in ("remove", "remove_conflict", "config"): + # the old API only required a tuple with just the + # installed package id. the new API requires the + # repository name (id) in every case. + if action == "remove_conflict": + action = "remove" + package_match = ( + package_match[0], + self._entropy.installed_repository().name) - self._check_action_validity(action) + self._action_instance = self.get( + action, package_match, opts = metaopts) + inst = self._action_instance - self._action = action - self._package_match = package_match + inst.setup() + self.pkgmeta = inst.metadata() + + # these are required by the old PackageKit backend + self.pkgmeta['repository'] = self._action_instance.repository_id() - if metaopts is None: - metaopts = {} self.metaopts = metaopts - # generate metadata dictionary - self._generate_metadata() + def run(self, xterm_header = None): + """ + Backward compatible method. + """ + if self._action_instance is None: + raise PermissionDenied("Not prepared") + if xterm_header is None: + xterm_header = "" + self._action_instance.set_xterm_header(xterm_header) + return self._action_instance.start() + + def kill(self): + """ + Backward compatible method. + """ + if self._action_instance is None: + raise PermissionDenied("Not prepared") + return self._action_instance.finalize() @classmethod def splitdebug_enabled(cls, entropy_client, pkg_match): """ - Return whether splitdebug is enabled for package. + Backward compatible method. """ - settings = entropy_client.Settings() - # this is a SystemSettings.CachingList object - splitdebug = settings['splitdebug'] - splitdebug_mask = settings['splitdebug_mask'] - _pkg_id, pkg_repo = pkg_match + return PackageAction.splitdebug_enabled(entropy_client, pkg_match) - def _generate_cache(lst_obj): - # compute the package matching then - pkg_matches = set() - for dep in lst_obj: - dep, repo_ids = entropy.dep.dep_get_match_in_repos(dep) - if repo_ids is not None: - if pkg_repo not in repo_ids: - # skip entry, not me - continue - dep_matches, _rc = entropy_client.atom_match( - dep, multi_match=True, multi_repo=True) - pkg_matches |= dep_matches - - # set cache back - lst_obj.set(pkg_matches) - return pkg_matches - - enabled = False - if not splitdebug: - # no entries, consider splitdebug always enabled - enabled = True - else: - # whitelist support - pkg_matches = splitdebug.get() - if pkg_matches is None: - pkg_matches = _generate_cache(splitdebug) - - # determine if it's enabled then - enabled = pkg_match in pkg_matches - - # if it's enabled, check whether it's blacklisted - if enabled: - # blacklist support - pkg_matches = splitdebug_mask.get() - if pkg_matches is None: - # compute the package matching - pkg_matches = _generate_cache(splitdebug_mask) - - enabled = pkg_match not in pkg_matches - - return enabled - - def _package_splitdebug_enabled(self, pkg_match): + @classmethod + def get_standard_fetch_disk_path(cls, download): """ - Determine if splitdebug is enabled for the package being installed - or just fetched. This method should be called only if system-wide - splitdebug setting in client.conf is enabled already. + Backward compatible method. """ - return Package.splitdebug_enabled(self._entropy, pkg_match) + return PackageAction.get_standard_fetch_disk_path(download) - def __get_base_metadata(self, _action): - def get_splitdebug_data(): - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - misc_data = self._settings[sys_set_plg_id]['misc'] - splitdebug = misc_data['splitdebug'] - splitdebug_dirs = misc_data['splitdebug_dirs'] - return splitdebug, splitdebug_dirs - - splitdebug, splitdebug_dirs = get_splitdebug_data() - metadata = { - 'splitdebug': splitdebug, - 'splitdebug_dirs': splitdebug_dirs, - '__content_files__': [], - } - return metadata - - def _generate_metadata(self): - self._error_on_prepared() - - self._check_action_validity(self._action) - self.pkgmeta.clear() - self.pkgmeta.update(self.__get_base_metadata(self._action)) - - if self._action == "fetch": - self.__generate_fetch_metadata() - elif self._action == "multi_fetch": - self.__generate_multi_fetch_metadata() - elif self._action in ("remove", "remove_conflict"): - self.__generate_remove_metadata() - elif self._action == "install": - self.__generate_install_metadata() - elif self._action == "source": - self.__generate_fetch_metadata(sources = True) - elif self._action == "config": - self.__generate_config_metadata() - - self.__prepared = True - - @staticmethod - def _generate_content_safety_file(content_safety): - """ - Generate a file containing the "content_safety" metadata, - reading by content_safety list or iterator. Each item - of "content_safety" must contain (path, sha256, mtime). - Each item shall be written to file, one per line, - in the following form: "||". - The order of the element in "content_safety" will be kept. - """ - tmp_dir = os.path.join( - etpConst['entropyunpackdir'], - "__generate_content_safety_file_f") - try: - os.makedirs(tmp_dir, 0o755) - except OSError as err: - if err.errno != errno.EEXIST: - raise - - tmp_fd, tmp_path = None, None - generated = False - try: - tmp_fd, tmp_path = const_mkstemp( - prefix="PackageContentSafety", - dir=tmp_dir) - with Content.FileContentSafetyWriter(tmp_fd) as tmp_f: - for path, sha256, mtime in content_safety: - tmp_f.write(path, sha256, mtime) - - generated = True - return tmp_path - finally: - if tmp_fd is not None: - try: - os.close(tmp_fd) - except OSError: - pass - if tmp_path is not None and not generated: - try: - os.remove(tmp_path) - except (OSError, IOError): - pass - - @staticmethod - def _generate_content_file(content, package_id = None, - filter_splitdebug = False, - splitdebug = None, - splitdebug_dirs = None): - """ - Generate a file containing the "content" metadata, - reading by content list or iterator. Each item - of "content" must contain (path, ftype). - Each item shall be written to file, one per line, - in the following form: "[|]|". - The order of the element in "content" will be kept. - """ - tmp_dir = os.path.join( - etpConst['entropyunpackdir'], - "__generate_content_file_f") - try: - os.makedirs(tmp_dir, 0o755) - except OSError as err: - if err.errno != errno.EEXIST: - raise - - tmp_fd, tmp_path = None, None - generated = False - try: - tmp_fd, tmp_path = const_mkstemp( - prefix="PackageContent", - dir=tmp_dir) - with Content.FileContentWriter(tmp_fd) as tmp_f: - for path, ftype in content: - if filter_splitdebug and not splitdebug: - # if filter_splitdebug is enabled, this - # code filters out all the paths starting - # with splitdebug_dirs, if splitdebug is - # disabled for package. - _skip = False - for split_dir in splitdebug_dirs: - if path.startswith(split_dir): - _skip = True - break - if _skip: - continue - tmp_f.write(package_id, path, ftype) - - generated = True - return tmp_path - finally: - if tmp_fd is not None: - try: - os.close(tmp_fd) - except OSError: - pass - if tmp_path is not None and not generated: - try: - os.remove(tmp_path) - except (OSError, IOError): - pass - - @staticmethod - def _merge_content_file(content_file, sorted_content, - cmp_func): - """ - Given a sorted content_file content and a sorted list of - content (sorted_content), apply the "merge" step of a merge - sort algorithm. In other words, add the sorted_content to - content_file keeping content_file content ordered. - It is of couse O(n+m) where n = lines in content_file and - m = sorted_content length. - """ - tmp_content_file = content_file + \ - Content.FileContentWriter.TMP_SUFFIX - - sorted_ptr = 0 - _sorted_path = None - _sorted_ftype = None - _package_id = 0 # will be filled - try: - with Content.FileContentWriter(tmp_content_file) as tmp_w: - with Content.FileContentReader(content_file) as tmp_r: - for _package_id, _path, _ftype in tmp_r: - - while True: - - try: - _sorted_path, _sorted_ftype = \ - sorted_content[sorted_ptr] - except IndexError: - _sorted_path = None - _sorted_ftype = None - - if _sorted_path is None: - tmp_w.write(_package_id, _path, _ftype) - break - - cmp_outcome = cmp_func(_path, _sorted_path) - if cmp_outcome < 0: - tmp_w.write(_package_id, _path, _ftype) - break - - # always privilege _ftype over _sorted_ftype - # _sorted_ftype might be invalid - tmp_w.write( - _package_id, _sorted_path, _ftype) - sorted_ptr += 1 - if cmp_outcome == 0: - # write only one - break - - # add the remainder - if sorted_ptr < len(sorted_content): - _sorted_rem = sorted_content[sorted_ptr:] - for _sorted_path, _sorted_ftype in _sorted_rem: - tmp_w.write( - _package_id, _sorted_path, _sorted_ftype) - - os.rename(tmp_content_file, content_file) - finally: - try: - os.remove(tmp_content_file) - except OSError as err: - if err.errno != errno.ENOENT: - raise - - @staticmethod - def _filter_content_file(content_file, filter_func): - """ - This method rewrites the content of content_file by applying - a filter to the path elements. - """ - tmp_content_file = content_file + \ - Content.FileContentWriter.TMP_SUFFIX - try: - with Content.FileContentWriter(tmp_content_file) as tmp_w: - with Content.FileContentReader(content_file) as tmp_r: - for _package_id, _path, _ftype in tmp_r: - if filter_func(_path): - tmp_w.write(_package_id, _path, _ftype) - os.rename(tmp_content_file, content_file) - finally: - try: - os.remove(tmp_content_file) - except OSError as err: - if err.errno != errno.ENOENT: - raise - - def __generate_content_file(self, content, package_id = None, - filter_splitdebug = False, - splitdebug = None, - splitdebug_dirs = None): - content_path = None - try: - content_path = Package._generate_content_file( - content, package_id = package_id, - filter_splitdebug = filter_splitdebug, - splitdebug = splitdebug, - splitdebug_dirs = splitdebug_dirs) - return content_path - finally: - if content_path is not None: - self.pkgmeta['__content_files__'].append( - content_path) - - def __generate_content_safety_file(self, content_safety): - content_path = None - try: - content_path = Package._generate_content_safety_file( - content_safety) - return content_path - finally: - if content_path is not None: - self.pkgmeta['__content_files__'].append( - content_path) - - def __generate_remove_metadata(self): - - package_id = self._package_match[0] - inst_repo = self._entropy.installed_repository() - - if not inst_repo.isPackageIdAvailable(package_id): - self.pkgmeta['remove_installed_vanished'] = True - return 0 - - self.pkgmeta['package_id'] = package_id - self.pkgmeta['removepackage_id'] = package_id - self.pkgmeta['configprotect_data'] = [] - self.pkgmeta['triggers'] = {} - self.pkgmeta['removeatom'] = \ - inst_repo.retrieveAtom(package_id) - self.pkgmeta['slot'] = \ - inst_repo.retrieveSlot(package_id) - self.pkgmeta['versiontag'] = \ - inst_repo.retrieveTag(package_id) - - remove_config = False - if 'removeconfig' in self.metaopts: - remove_config = self.metaopts.get('removeconfig') - self.pkgmeta['removeconfig'] = remove_config - - content = inst_repo.retrieveContentIter( - package_id, order_by="file", reverse=True) - self.pkgmeta['removecontent_file'] = \ - self.__generate_content_file(content) - # collects directories whose content has been modified - # this information is then handed to the Trigger - self.pkgmeta['affected_directories'] = set() - self.pkgmeta['affected_infofiles'] = set() - - # this will be set before calling removePackage() if we have - # a package to remove - self.pkgmeta['config_protect+mask'] = None - - self.pkgmeta['triggers']['remove'] = \ - inst_repo.getTriggerData(package_id) - if self.pkgmeta['triggers']['remove'] is None: - self.pkgmeta['remove_installed_vanished'] = True - return 0 - self.pkgmeta['triggers']['remove']['affected_directories'] = \ - self.pkgmeta['affected_directories'] - self.pkgmeta['triggers']['remove']['affected_infofiles'] = \ - self.pkgmeta['affected_infofiles'] - - self.pkgmeta['triggers']['remove']['spm_repository'] = \ - inst_repo.retrieveSpmRepository( - package_id) - self.pkgmeta['triggers']['remove'].update( - self.__get_base_metadata(self._action)) - - pkg_license = inst_repo.retrieveLicense( - package_id) - if pkg_license is None: - pkg_license = set() - else: - pkg_license = set(pkg_license.split()) - - self.pkgmeta['triggers']['remove']['accept_license'] = pkg_license - - self.pkgmeta['steps'] = ["preremove", "remove", "postremove", - "postremove_remove"] - - return 0 - - def __generate_config_metadata(self): - package_id = self._package_match[0] - inst_repo = self._entropy.installed_repository() - - self.pkgmeta['atom'] = inst_repo.retrieveAtom(package_id) - key, slot = inst_repo.retrieveKeySlot(package_id) - self.pkgmeta['key'], self.pkgmeta['slot'] = key, slot - self.pkgmeta['version'] = inst_repo.retrieveVersion(package_id) - self.pkgmeta['category'] = inst_repo.retrieveCategory(package_id) - self.pkgmeta['name'] = inst_repo.retrieveName(package_id) - self.pkgmeta['spm_repository'] = inst_repo.retrieveSpmRepository( - package_id) - - pkg_license = inst_repo.retrieveLicense(package_id) - if pkg_license is None: - pkg_license = set() - else: - pkg_license = set(pkg_license.split()) - - self.pkgmeta['accept_license'] = pkg_license - self.pkgmeta['steps'] = [] - self.pkgmeta['steps'].append("config") - - return 0 - - def __get_match_conflicts(self, match): - m_id, m_repo = match - - dbconn = self._entropy.open_repository(m_repo) - conflicts = dbconn.retrieveConflicts(m_id) - found_conflicts = set() - inst_repo = self._entropy.installed_repository() - - for conflict in conflicts: - my_m_id, _my_m_rc = inst_repo.atomMatch(conflict) - if my_m_id != -1: - # check if the package shares the same slot - match_data = dbconn.retrieveKeySlot(m_id) - installed_match_data = inst_repo.retrieveKeySlot(my_m_id) - if match_data != installed_match_data: - found_conflicts.add(my_m_id) - - # auto conflicts support - found_conflicts |= self._entropy._generate_dependency_inverse_conflicts( - match, just_id=True) - - return found_conflicts - - def __setup_package_to_remove(self, package_key, slot): - - inst_repo = self._entropy.installed_repository() - inst_package_id, _inst_rc = inst_repo.atomMatch(package_key, - matchSlot = slot) - - if inst_package_id != -1: - avail = inst_repo.isPackageIdAvailable(inst_package_id) - if avail: - inst_atom = inst_repo.retrieveAtom(inst_package_id) - self.pkgmeta['removeatom'] = inst_atom - else: - inst_package_id = -1 - return inst_package_id - - def __generate_install_metadata(self): - - package_id, repository = self._package_match - inst_repo = self._entropy.installed_repository() - self.pkgmeta['package_id'] = package_id - self.pkgmeta['repository'] = repository - cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] - edelta_support = self._settings[cl_id]['misc']['edelta_support'] - self.pkgmeta['edelta_support'] = edelta_support - is_package_repo = repository.endswith(etpConst['packagesext']) - - # if splitdebug is enabled, check if it's also enabled - # via package.splitdebug - if self.pkgmeta['splitdebug']: - # yeah, this has to affect exported splitdebug setting - # because it is read during package files installation - # Older splitdebug data was in the same package file of - # the actual content. Later on, splitdebug data was moved - # to its own package file that gets downloaded and unpacked - # only if required (if splitdebug is enabled) - self.pkgmeta['splitdebug'] = self._package_splitdebug_enabled( - self._package_match) - - # fetch abort function - self.pkgmeta['fetch_abort_function'] = \ - self.metaopts.get('fetch_abort_function') - - install_source = etpConst['install_sources']['unknown'] - meta_inst_source = self.metaopts.get('install_source', install_source) - if meta_inst_source in list(etpConst['install_sources'].values()): - install_source = meta_inst_source - self.pkgmeta['install_source'] = install_source - - self.pkgmeta['already_protected_config_files'] = {} - self.pkgmeta['configprotect_data'] = [] - dbconn = self._entropy.open_repository(repository) - self.pkgmeta['triggers'] = {} - self.pkgmeta['atom'] = dbconn.retrieveAtom(package_id) - self.pkgmeta['slot'] = dbconn.retrieveSlot(package_id) - - ver, tag, rev = dbconn.getVersioningData(package_id) - self.pkgmeta['version'] = ver - self.pkgmeta['versiontag'] = tag - self.pkgmeta['revision'] = rev - - self.pkgmeta['extra_download'] = [] - self.pkgmeta['splitdebug_pkgfile'] = True - if not is_package_repo: - self.pkgmeta['splitdebug_pkgfile'] = False - extra_download = dbconn.retrieveExtraDownload(package_id) - if not self.pkgmeta['splitdebug']: - extra_download = [x for x in extra_download if \ - x['type'] != "debug"] - self.pkgmeta['extra_download'] += extra_download - - self.pkgmeta['category'] = dbconn.retrieveCategory(package_id) - self.pkgmeta['download'] = dbconn.retrieveDownloadURL(package_id) - self.pkgmeta['name'] = dbconn.retrieveName(package_id) - self.pkgmeta['checksum'] = dbconn.retrieveDigest(package_id) - sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id) - signatures = { - 'sha1': sha1, - 'sha256': sha256, - 'sha512': sha512, - 'gpg': gpg, - } - self.pkgmeta['signatures'] = signatures - self.pkgmeta['conflicts'] = self.__get_match_conflicts( - self._package_match) - - description = dbconn.retrieveDescription(package_id) - if description: - if len(description) > 74: - description = description[:74].strip() - description += "..." - self.pkgmeta['description'] = description - - # this is set by __install_package() and required by spm_install - # phase - self.pkgmeta['installed_package_id'] = None - # fill action queue - self.pkgmeta['removepackage_id'] = -1 - remove_config = False - if 'removeconfig' in self.metaopts: - remove_config = self.metaopts.get('removeconfig') - - self.pkgmeta['remove_metaopts'] = { - 'removeconfig': True, - } - if 'remove_metaopts' in self.metaopts: - self.pkgmeta['remove_metaopts'] = \ - self.metaopts.get('remove_metaopts') - - self.pkgmeta['merge_from'] = None - mf = self.metaopts.get('merge_from') - if mf != None: - self.pkgmeta['merge_from'] = const_convert_to_unicode(mf) - self.pkgmeta['removeconfig'] = remove_config - - self.pkgmeta['removepackage_id'] = self.__setup_package_to_remove( - entropy.dep.dep_getkey(self.pkgmeta['atom']), - self.pkgmeta['slot']) - - # collects directories whose content has been modified - # this information is then handed to the Trigger - self.pkgmeta['affected_directories'] = set() - self.pkgmeta['affected_infofiles'] = set() - - # smartpackage ? - self.pkgmeta['smartpackage'] = False - # set unpack dir and image dir - if is_package_repo: - - try: - compiled_arch = dbconn.getSetting("arch") - arch_fine = compiled_arch == etpConst['currentarch'] - except KeyError: - arch_fine = True # sorry, old db, cannot check - - if not arch_fine: - self.__prepared = False - return -1 - - repo_data = self._settings['repositories'] - repo_meta = repo_data['available'][self.pkgmeta['repository']] - self.pkgmeta['smartpackage'] = repo_meta['smartpackage'] - self.pkgmeta['pkgpath'] = repo_meta['pkgpath'] - - else: - self.pkgmeta['pkgpath'] = self.__get_fetch_disk_path( - self.pkgmeta['download']) - - self.pkgmeta['unpackdir'] = etpConst['entropyunpackdir'] + \ - os.path.sep + self.__escape_path(self.pkgmeta['download']) - - self.pkgmeta['imagedir'] = self.pkgmeta['unpackdir'] + os.path.sep + \ - etpConst['entropyimagerelativepath'] - - self.pkgmeta['pkgdbpath'] = os.path.join(self.pkgmeta['unpackdir'], - "edb/pkg.db") - - # this will be set before calling handlePackage() - # if we have a package to remove - self.pkgmeta['config_protect+mask'] = None - - if self.pkgmeta['removepackage_id'] == -1: - # nothing to remove, fresh install - self.pkgmeta['removecontent_file'] = None - else: - # generate content file - content = inst_repo.retrieveContentIter( - self.pkgmeta['removepackage_id'], - order_by="file", reverse=True) - self.pkgmeta['removecontent_file'] = \ - self.__generate_content_file(content) - - # There is a pkg to remove, but... - # compare both versions and if they match, disable removepackage_id - trigger_data = inst_repo.getTriggerData( - self.pkgmeta['removepackage_id']) - - if trigger_data is None: - # installed repository entry is corrupted - self.pkgmeta['removepackage_id'] = -1 - else: - self.pkgmeta['removeatom'] = inst_repo.retrieveAtom( - self.pkgmeta['removepackage_id']) - - self.pkgmeta['triggers']['remove'] = trigger_data - # pass reference, not copy! nevva! - self.pkgmeta['triggers']['remove']['affected_directories'] = \ - self.pkgmeta['affected_directories'] - self.pkgmeta['triggers']['remove']['affected_infofiles'] = \ - self.pkgmeta['affected_infofiles'] - - self.pkgmeta['triggers']['remove']['spm_repository'] = \ - inst_repo.retrieveSpmRepository(package_id) - self.pkgmeta['triggers']['remove'].update( - self.__get_base_metadata(self._action)) - - pkg_rm_license = inst_repo.retrieveLicense( - self.pkgmeta['removepackage_id']) - if pkg_rm_license is None: - pkg_rm_license = set() - else: - pkg_rm_license = set(pkg_rm_license.split()) - self.pkgmeta['triggers']['remove']['accept_license'] = \ - pkg_rm_license - - # set steps - self.pkgmeta['steps'] = [] - if self.pkgmeta['conflicts']: - self.pkgmeta['steps'].append("remove_conflicts") - # install - self.pkgmeta['steps'].append("unpack") - # preinstall placed before preremove in order - # to respect Spm order - self.pkgmeta['steps'].append("setup") - self.pkgmeta['steps'].append("preinstall") - self.pkgmeta['steps'].append("install") - if self.pkgmeta['removepackage_id'] != -1: - self.pkgmeta['steps'].append("preremove") - self.pkgmeta['steps'].append("install_clean") - if self.pkgmeta['removepackage_id'] != -1: - self.pkgmeta['steps'].append("postremove") - self.pkgmeta['steps'].append("postremove_install") - self.pkgmeta['steps'].append("install_spm") - self.pkgmeta['steps'].append("postinstall") - self.pkgmeta['steps'].append("cleanup") - - self.pkgmeta['triggers']['install'] = dbconn.getTriggerData(package_id) - if self.pkgmeta['triggers']['install'] is None: - # wtf!? - return 1 - pkg_license = dbconn.retrieveLicense(package_id) - if pkg_license is None: - pkg_license = set() - else: - pkg_license = set(pkg_license.split()) - self.pkgmeta['accept_license'] = pkg_license - self.pkgmeta['triggers']['install']['accept_license'] = pkg_license - self.pkgmeta['triggers']['install']['unpackdir'] = \ - self.pkgmeta['unpackdir'] - self.pkgmeta['triggers']['install']['imagedir'] = \ - self.pkgmeta['imagedir'] - self.pkgmeta['triggers']['install']['spm_repository'] = \ - dbconn.retrieveSpmRepository(package_id) - self.pkgmeta['triggers']['install'].update( - self.__get_base_metadata(self._action)) - - spm_class = self._entropy.Spm_class() - # call Spm setup hook - return spm_class.entropy_install_setup_hook(self._entropy, self.pkgmeta) - - def __generate_fetch_metadata(self, sources = False): - - package_id, repository = self._package_match - dochecksum = True - - # fetch abort function - self.pkgmeta['fetch_abort_function'] = \ - self.metaopts.get('fetch_abort_function') - - if 'dochecksum' in self.metaopts: - dochecksum = self.metaopts.get('dochecksum') - - # NOTE: if you want to implement download-to-dir feature in your - # client, you've found what you were looking for. - # fetch_path is the path where data should be downloaded - # it overrides default path - if 'fetch_path' in self.metaopts: - fetch_path = self.metaopts.get('fetch_path') - if entropy.tools.is_valid_path(fetch_path): - self.pkgmeta['fetch_path'] = fetch_path - - # if splitdebug is enabled, check if it's also enabled - # via package.splitdebug - splitdebug = self.pkgmeta['splitdebug'] - if splitdebug: - splitdebug = self._package_splitdebug_enabled( - self._package_match) - - self.pkgmeta['repository'] = repository - self.pkgmeta['package_id'] = package_id - dbconn = self._entropy.open_repository(repository) - self.pkgmeta['atom'] = dbconn.retrieveAtom(package_id) - self.pkgmeta['slot'] = dbconn.retrieveSlot(package_id) - self.pkgmeta['removepackage_id'] = self.__setup_package_to_remove( - entropy.dep.dep_getkey(self.pkgmeta['atom']), self.pkgmeta['slot']) - - if sources: - self.pkgmeta['edelta_support'] = False - self.pkgmeta['extra_download'] = tuple() - self.pkgmeta['download'] = dbconn.retrieveSources(package_id, - extended = True) - # fake path, don't use - self.pkgmeta['pkgpath'] = etpConst['entropypackagesworkdir'] - else: - cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] - edelta_support = self._settings[cl_id]['misc']['edelta_support'] - self.pkgmeta['edelta_support'] = edelta_support - self.pkgmeta['checksum'] = dbconn.retrieveDigest(package_id) - sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id) - signatures = { - 'sha1': sha1, - 'sha256': sha256, - 'sha512': sha512, - 'gpg': gpg, - } - self.pkgmeta['signatures'] = signatures - extra_download = dbconn.retrieveExtraDownload(package_id) - if not splitdebug: - extra_download = [x for x in extra_download if \ - x['type'] != "debug"] - self.pkgmeta['extra_download'] = extra_download - self.pkgmeta['download'] = dbconn.retrieveDownloadURL(package_id) - - # export main package download path to metadata - # this is actually used by PackageKit backend in order - # to signal downloaded package files - self.pkgmeta['pkgpath'] = self.__get_fetch_disk_path( - self.pkgmeta['download']) - - if not self.pkgmeta['download']: - self.pkgmeta['fetch_not_available'] = True - return 0 - - self.pkgmeta['verified'] = False - self.pkgmeta['steps'] = [] - if not repository.endswith(etpConst['packagesext']) and not sources: - - dl_check = self.__check_pkg_path_download(self.pkgmeta['download'], - None) - dl_fetch = dl_check < 0 - - if not dl_fetch: - for extra_download in self.pkgmeta['extra_download']: - dl_check = self.__check_pkg_path_download( - extra_download['download'], None) - if dl_check < 0: - # dl_check checked again right below - break - - if dl_check < 0: - self.pkgmeta['steps'].append("fetch") - - if dochecksum: - self.pkgmeta['steps'].append("checksum") - - elif sources: - self.pkgmeta['steps'].append("sources_fetch") - - if sources: - # create sources destination directory - unpack_dir = os.path.join(etpConst['entropyunpackdir'], - "sources", self.pkgmeta['atom']) - self.pkgmeta['unpackdir'] = unpack_dir - - if not self.pkgmeta.get('fetch_path'): - if os.path.lexists(unpack_dir): - if os.path.isfile(unpack_dir): - os.remove(unpack_dir) - elif os.path.isdir(unpack_dir): - shutil.rmtree(unpack_dir, True) - if not os.path.lexists(unpack_dir): - os.makedirs(unpack_dir, 0o755) - const_setup_perms(unpack_dir, etpConst['entropygid'], - recursion = False, uid = etpConst['uid']) - return 0 - - def _check_matching_size(download, size): - d_path = self.__get_fetch_disk_path(download) - try: - st = os.stat(d_path) - return size == st.st_size - except OSError as err: - if err.errno != errno.ENOENT: - raise - return False - - matching_size = _check_matching_size(self.pkgmeta['download'], - dbconn.retrieveSize(package_id)) - if matching_size: - for extra_download in self.pkgmeta['extra_download']: - matching_size = _check_matching_size( - extra_download['download'], - extra_download['size']) - if not matching_size: - break - - # downloading binary package - # if file exists, first checksum then fetch - if matching_size: - self.pkgmeta['steps'].reverse() - - return 0 - - def __generate_multi_fetch_metadata(self): - - if not isinstance(self._package_match, list): - raise AttributeError( - "package_match must be a list of tuples, not %s" % ( - type(self._package_match,) - ) - ) - - dochecksum = True - - # meta options - self.pkgmeta['fetch_abort_function'] = \ - self.metaopts.get('fetch_abort_function') - - if 'dochecksum' in self.metaopts: - dochecksum = self.metaopts.get('dochecksum') - self.pkgmeta['checksum'] = dochecksum - - matches = self._package_match - cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] - edelta_support = self._settings[cl_id]['misc']['edelta_support'] - self.pkgmeta['edelta_support'] = edelta_support - self.pkgmeta['matches'] = matches - self.pkgmeta['atoms'] = [] - self.pkgmeta['repository_atoms'] = {} - temp_fetch_list = [] - temp_checksum_list = [] - temp_already_downloaded_count = 0 - - def _setup_download(download, size, package_id, repository, digest, - signatures): - - if dochecksum: - obj = (package_id, repository, download, digest, - signatures) - temp_checksum_list.append(obj) - - if self.__check_pkg_path_download(download, None) < 0: - obj = (package_id, repository, download, digest, signatures) - temp_fetch_list.append(obj) - else: - down_path = self.__get_fetch_disk_path(download) - if os.path.isfile(down_path): - with open(down_path, "rb") as f: - f.seek(0, os.SEEK_END) - disk_size = f.tell() - if size == disk_size: - return 1 - return 0 - - - for package_id, repository in matches: - - if repository.endswith(etpConst['packagesext']): - continue - - dbconn = self._entropy.open_repository(repository) - myatom = dbconn.retrieveAtom(package_id) - - # general purpose metadata - self.pkgmeta['atoms'].append(myatom) - if repository not in self.pkgmeta['repository_atoms']: - self.pkgmeta['repository_atoms'][repository] = set() - self.pkgmeta['repository_atoms'][repository].add(myatom) - - download = dbconn.retrieveDownloadURL(package_id) - digest = dbconn.retrieveDigest(package_id) - sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id) - size = dbconn.retrieveSize(package_id) - signatures = { - 'sha1': sha1, - 'sha256': sha256, - 'sha512': sha512, - 'gpg': gpg, - } - temp_already_downloaded_count += _setup_download(download, size, - package_id, repository, digest, signatures) - - extra_downloads = dbconn.retrieveExtraDownload(package_id) - - splitdebug = self.pkgmeta['splitdebug'] - # if splitdebug is enabled, check if it's also enabled - # via package.splitdebug - if splitdebug: - splitdebug = self._package_splitdebug_enabled( - (package_id, repository)) - - if not splitdebug: - extra_downloads = [x for x in extra_downloads if \ - x['type'] != "debug"] - for extra_download in extra_downloads: - download = extra_download['download'] - size = extra_download['size'] - digest = extra_download['md5'] - signatures = { - 'sha1': extra_download['sha1'], - 'sha256': extra_download['sha256'], - 'sha512': extra_download['sha512'], - 'gpg': extra_download['gpg'], - } - temp_already_downloaded_count += _setup_download(download, - size, package_id, repository, digest, signatures) - - self.pkgmeta['steps'] = [] - self.pkgmeta['multi_fetch_list'] = temp_fetch_list - self.pkgmeta['multi_checksum_list'] = temp_checksum_list - if self.pkgmeta['multi_fetch_list']: - self.pkgmeta['steps'].append("multi_fetch") - if self.pkgmeta['multi_checksum_list']: - self.pkgmeta['steps'].append("multi_checksum") - if temp_already_downloaded_count == len(temp_checksum_list): - self.pkgmeta['steps'].reverse() - - return 0 +# Complete the backward compatibility support bits +Package = PackageActionFactoryWrapper diff --git a/lib/entropy/client/interfaces/package/_content.py b/lib/entropy/client/interfaces/package/_content.py index e5865e777..feec5f5e6 100644 --- a/lib/entropy/client/interfaces/package/_content.py +++ b/lib/entropy/client/interfaces/package/_content.py @@ -10,9 +10,11 @@ """ import codecs +import errno import sys +import os -from entropy.const import etpConst +from entropy.const import etpConst, const_mkstemp import entropy.tools @@ -312,3 +314,189 @@ class FileContentSafetyReader(object): if self._file is not None: self._file.close() self._file = None + + +def generate_content_safety_file(content_safety): + """ + Generate a file containing the "content_safety" metadata, + reading by content_safety list or iterator. Each item + of "content_safety" must contain (path, sha256, mtime). + Each item shall be written to file, one per line, + in the following form: "||". + The order of the element in "content_safety" will be kept. + """ + tmp_dir = os.path.join( + etpConst['entropyunpackdir'], + "__generate_content_safety_file_f") + try: + os.makedirs(tmp_dir, 0o755) + except OSError as err: + if err.errno != errno.EEXIST: + raise + + tmp_fd, tmp_path = None, None + generated = False + try: + tmp_fd, tmp_path = const_mkstemp( + prefix="PackageContentSafety", + dir=tmp_dir) + with FileContentSafetyWriter(tmp_fd) as tmp_f: + for path, sha256, mtime in content_safety: + tmp_f.write(path, sha256, mtime) + + generated = True + return tmp_path + finally: + if tmp_fd is not None: + try: + os.close(tmp_fd) + except OSError: + pass + if tmp_path is not None and not generated: + try: + os.remove(tmp_path) + except (OSError, IOError): + pass + + +def generate_content_file(content, package_id = None, + filter_splitdebug = False, + splitdebug = None, + splitdebug_dirs = None): + """ + Generate a file containing the "content" metadata, + reading by content list or iterator. Each item + of "content" must contain (path, ftype). + Each item shall be written to file, one per line, + in the following form: "[|]|". + The order of the element in "content" will be kept. + """ + tmp_dir = os.path.join( + etpConst['entropyunpackdir'], + "__generate_content_file_f") + try: + os.makedirs(tmp_dir, 0o755) + except OSError as err: + if err.errno != errno.EEXIST: + raise + + tmp_fd, tmp_path = None, None + generated = False + try: + tmp_fd, tmp_path = const_mkstemp( + prefix="PackageContent", + dir=tmp_dir) + with FileContentWriter(tmp_fd) as tmp_f: + for path, ftype in content: + if filter_splitdebug and not splitdebug: + # if filter_splitdebug is enabled, this + # code filters out all the paths starting + # with splitdebug_dirs, if splitdebug is + # disabled for package. + _skip = False + for split_dir in splitdebug_dirs: + if path.startswith(split_dir): + _skip = True + break + if _skip: + continue + tmp_f.write(package_id, path, ftype) + + generated = True + return tmp_path + finally: + if tmp_fd is not None: + try: + os.close(tmp_fd) + except OSError: + pass + if tmp_path is not None and not generated: + try: + os.remove(tmp_path) + except (OSError, IOError): + pass + + +def merge_content_file(content_file, sorted_content, + cmp_func): + """ + Given a sorted content_file content and a sorted list of + content (sorted_content), apply the "merge" step of a merge + sort algorithm. In other words, add the sorted_content to + content_file keeping content_file content ordered. + It is of couse O(n+m) where n = lines in content_file and + m = sorted_content length. + """ + tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX + + sorted_ptr = 0 + _sorted_path = None + _sorted_ftype = None + _package_id = 0 # will be filled + try: + with FileContentWriter(tmp_content_file) as tmp_w: + with FileContentReader(content_file) as tmp_r: + for _package_id, _path, _ftype in tmp_r: + + while True: + + try: + _sorted_path, _sorted_ftype = \ + sorted_content[sorted_ptr] + except IndexError: + _sorted_path = None + _sorted_ftype = None + + if _sorted_path is None: + tmp_w.write(_package_id, _path, _ftype) + break + + cmp_outcome = cmp_func(_path, _sorted_path) + if cmp_outcome < 0: + tmp_w.write(_package_id, _path, _ftype) + break + + # always privilege _ftype over _sorted_ftype + # _sorted_ftype might be invalid + tmp_w.write( + _package_id, _sorted_path, _ftype) + sorted_ptr += 1 + if cmp_outcome == 0: + # write only one + break + + # add the remainder + if sorted_ptr < len(sorted_content): + _sorted_rem = sorted_content[sorted_ptr:] + for _sorted_path, _sorted_ftype in _sorted_rem: + tmp_w.write( + _package_id, _sorted_path, _sorted_ftype) + + os.rename(tmp_content_file, content_file) + finally: + try: + os.remove(tmp_content_file) + except OSError as err: + if err.errno != errno.ENOENT: + raise + + +def filter_content_file(content_file, filter_func): + """ + This method rewrites the content of content_file by applying + a filter to the path elements. + """ + tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX + try: + with FileContentWriter(tmp_content_file) as tmp_w: + with FileContentReader(content_file) as tmp_r: + for _package_id, _path, _ftype in tmp_r: + if filter_func(_path): + tmp_w.write(_package_id, _path, _ftype) + os.rename(tmp_content_file, content_file) + finally: + try: + os.remove(tmp_content_file) + except OSError as err: + if err.errno != errno.ENOENT: + raise diff --git a/lib/entropy/client/interfaces/package/actions/__init__.py b/lib/entropy/client/interfaces/package/actions/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lib/entropy/client/interfaces/package/actions/_manage.py b/lib/entropy/client/interfaces/package/actions/_manage.py new file mode 100644 index 000000000..889bad0ff --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/_manage.py @@ -0,0 +1,590 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import errno +import os + +from entropy.const import etpConst, const_convert_to_unicode, \ + const_convert_to_rawstring, const_is_python3 +from entropy.i18n import _ +from entropy.output import red, purple, brown, darkred, blue, darkgreen + +import entropy.tools + +from .. import _content as Content + +from .action import PackageAction + + +class _PackageInstallRemoveAction(PackageAction): + """ + Abstract class that exposes shared functions between install + and remove PackageAction classes. + """ + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + return self._meta + + def setup(self): + """ + Overridden from PackageAction. + """ + raise NotImplementedError() + + def _run(self): + """ + Overridden from PackageAction. + """ + raise NotImplementedError() + + def _get_system_root(self): + """ + Return the path to the system root directory. + """ + metadata = self.metadata() + return metadata.get('unittest_root', "") + etpConst['systemroot'] + + def _get_config_protect_skip(self): + """ + Return the configuration protection path set. + """ + misc_settings = self._entropy.ClientSettings()['misc'] + protectskip = misc_settings['configprotectskip'] + + if not const_is_python3(): + protectskip = set(( + const_convert_to_rawstring( + x, from_enctype = etpConst['conf_encoding']) for x in + misc_settings['configprotectskip'])) + + return protectskip + + def _get_config_protect(self, entropy_repository, package_id, mask = False): + """ + Return configuration protection (or mask) metadata for the given + package. + This method should not be used as source for storing metadata into + repositories since the returned objects may not be decoded in utf-8. + Data returned by this method is expected to be used only by internal + functions. + """ + cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] + misc_data = self._settings[cl_id]['misc'] + + if mask: + paths = entropy_repository.retrieveProtectMask(package_id).split() + misc_key = "configprotectmask" + else: + paths = entropy_repository.retrieveProtect(package_id).split() + misc_key = "configprotect" + + root = etpConst['systemroot'] + config = set(("%s%s" % (root, path) for path in paths)) + config.update(misc_data[misc_key]) + + # os.* methods in Python 2.x do not expect unicode strings + # This set of data is only used by _handle_config_protect atm. + if not const_is_python3(): + config = set((const_convert_to_rawstring(x) for x in config)) + + return config + + def _get_config_protect_metadata(self, installed_repository, + installed_package_id): + """ + Get the config_protect+mask metadata object. + Make sure to call this before the package goes away from the + repository. + """ + protect = self._get_config_protect( + installed_repository, installed_package_id) + mask = self._get_config_protect( + installed_repository, installed_package_id, mask = True) + + metadata = { + 'config_protect+mask': (protect, mask) + } + return metadata + + def _handle_config_protect(self, protect, mask, protectskip, + fromfile, tofile, + do_allocation_check = True, + do_quiet = False): + """ + Handle configuration file protection. This method contains the logic + for determining if a file should be protected from overwrite. + """ + protected = False + do_continue = False + in_mask = False + + if tofile in protect: + protected = True + in_mask = True + + elif os.path.dirname(tofile) in protect: + protected = True + in_mask = True + + else: + tofile_testdir = os.path.dirname(tofile) + old_tofile_testdir = None + while tofile_testdir != old_tofile_testdir: + if tofile_testdir in protect: + protected = True + in_mask = True + break + old_tofile_testdir = tofile_testdir + tofile_testdir = os.path.dirname(tofile_testdir) + + if protected: # check if perhaps, file is masked, so unprotected + + if tofile in mask: + protected = False + in_mask = False + + elif os.path.dirname(tofile) in mask: + protected = False + in_mask = False + + else: + tofile_testdir = os.path.dirname(tofile) + old_tofile_testdir = None + while tofile_testdir != old_tofile_testdir: + if tofile_testdir in mask: + protected = False + in_mask = False + break + old_tofile_testdir = tofile_testdir + tofile_testdir = os.path.dirname(tofile_testdir) + + if not os.path.lexists(tofile): + protected = False # file doesn't exist + + # check if it's a text file + if protected: + protected = entropy.tools.istextfile(tofile) + in_mask = protected + + if fromfile is not None: + if protected and os.path.lexists(fromfile) and \ + (not os.path.exists(fromfile)) and os.path.islink(fromfile): + # broken symlink, don't protect + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to handle file protection for: " \ + "%s, broken symlink in package" % ( + tofile, + ) + ) + msg = _("Cannot protect broken symlink") + mytxt = "%s:" % ( + purple(msg), + ) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = brown(" ## ") + ) + self._entropy.output( + tofile, + level = "warning", + header = brown(" ## ") + ) + protected = False + + if not protected: + return in_mask, protected, tofile, do_continue + + ## ## + # file is protected # + ##__________________## + + # check if protection is disabled for this element + if tofile in protectskip: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Skipping config file installation/removal, " \ + "as stated in client.conf: %s" % (tofile,) + ) + if not do_quiet: + mytxt = "%s: %s" % ( + _("Skipping file installation/removal"), + tofile, + ) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = darkred(" ## ") + ) + do_continue = True + return in_mask, protected, tofile, do_continue + + ## ## + # file is protected (2) # + ##______________________## + + prot_status = True + if do_allocation_check: + spm_class = self._entropy.Spm_class() + tofile, prot_status = spm_class.allocate_protected_file(fromfile, + tofile) + + if not prot_status: + # a protected file with the same content + # is already in place, so not going to protect + # the same file twice + protected = False + return in_mask, protected, tofile, do_continue + + ## ## + # file is protected (3) # + ##______________________## + + oldtofile = tofile + if oldtofile.find("._cfg") != -1: + oldtofile = os.path.join(os.path.dirname(oldtofile), + os.path.basename(oldtofile)[10:]) + + if not do_quiet: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Protecting config file: %s" % (oldtofile,) + ) + mytxt = red("%s: %s") % (_("Protecting config file"), oldtofile,) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = darkred(" ## ") + ) + + return in_mask, protected, tofile, do_continue + + def _remove_content_from_system_loop(self, inst_repo, remove_content, + directories, directories_cache, + not_removed_due_to_collisions, + colliding_path_messages, + automerge_metadata, col_protect, + protect, mask, protectskip, + sys_root): + """ + Body of the _remove_content_from_system() method. + """ + info_dirs = self._get_info_directories() + metadata = self.metadata() + + for _pkg_id, item, _ftype in remove_content: + + if not item: + continue # empty element?? + + sys_root_item = sys_root + item + sys_root_item_encoded = sys_root_item + if not const_is_python3(): + # this is coming from the db, and it's pure utf-8 + sys_root_item_encoded = const_convert_to_rawstring( + sys_root_item, + from_enctype = etpConst['conf_raw_encoding']) + + # collision check + if col_protect > 0: + + if inst_repo.isFileAvailable(item) \ + and os.path.isfile(sys_root_item_encoded): + + # in this way we filter out directories + colliding_path_messages.add(sys_root_item) + not_removed_due_to_collisions.add(item) + continue + + protected = False + in_mask = False + + if not metadata['removeconfig']: + + protected_item_test = sys_root_item + (in_mask, protected, _x, + do_continue) = self._handle_config_protect( + protect, mask, protectskip, None, protected_item_test, + do_allocation_check = False, do_quiet = True + ) + + if do_continue: + protected = True + + # when files have not been modified by the user + # and they are inside a config protect directory + # we could even remove them directly + if in_mask: + + oldprot_md5 = automerge_metadata.get(item) + if oldprot_md5: + + try: + in_system_md5 = entropy.tools.md5sum( + protected_item_test) + except (OSError, IOError) as err: + if err.errno != errno.ENOENT: + raise + in_system_md5 = "?" + + if oldprot_md5 == in_system_md5: + prot_msg = _("Removing config file, never modified") + mytxt = "%s: %s" % ( + darkgreen(prot_msg), + blue(item), + ) + self._entropy.output( + mytxt, + importance = 1, + level = "info", + header = red(" ## ") + ) + protected = False + do_continue = False + + # Is file or directory a protected item? + if protected: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['verbose_loglevel_id'], + "[remove] Protecting config file: %s" % (sys_root_item,) + ) + mytxt = "[%s] %s: %s" % ( + red(_("remove")), + brown(_("Protecting config file")), + sys_root_item, + ) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = red(" ## ") + ) + continue + + try: + os.lstat(sys_root_item_encoded) + except OSError as err: + if err.errno in (errno.ENOENT, errno.ENOTDIR): + continue # skip file, does not exist + raise + + except UnicodeEncodeError: + msg = _("This package contains a badly encoded file !!!") + mytxt = brown(msg) + self._entropy.output( + red("QA: ")+mytxt, + importance = 1, + level = "warning", + header = darkred(" ## ") + ) + continue # file has a really bad encoding + + if os.path.isdir(sys_root_item_encoded) and \ + os.path.islink(sys_root_item_encoded): + # S_ISDIR returns False for directory symlinks, + # so using os.path.isdir valid directory symlink + if sys_root_item not in directories_cache: + # collect for Trigger + metadata['affected_directories'].add(item) + directories.add((sys_root_item, "link")) + directories_cache.add(sys_root_item) + continue + + if os.path.isdir(sys_root_item_encoded): + # plain directory + if sys_root_item not in directories_cache: + # collect for Trigger + metadata['affected_directories'].add(item) + directories.add((sys_root_item, "dir")) + directories_cache.add(sys_root_item) + continue + + # files, symlinks or not + # just a file or symlink or broken + # directory symlink (remove now) + + try: + os.remove(sys_root_item_encoded) + except OSError as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "[remove] Unable to remove %s, error: %s" % ( + sys_root_item, err,) + ) + continue + + # collect for Trigger + dir_name = os.path.dirname(item) + metadata['affected_directories'].add(dir_name) + + # account for info files, if any + if dir_name in info_dirs: + for _ext in self._INFO_EXTS: + if item.endswith(_ext): + metadata['affected_infofiles'].add(item) + break + + # add its parent directory + dirobj = const_convert_to_unicode( + os.path.dirname(sys_root_item_encoded)) + if dirobj not in directories_cache: + if os.path.isdir(dirobj) and os.path.islink(dirobj): + directories.add((dirobj, "link")) + elif os.path.isdir(dirobj): + directories.add((dirobj, "dir")) + + directories_cache.add(dirobj) + + def _remove_content_from_system(self, installed_repository, + automerge_metadata = None): + """ + Remove installed package content (files/directories) from live system. + + @keyword automerge_metadata: Entropy "automerge metadata" + @type automerge_metadata: dict + """ + if automerge_metadata is None: + automerge_metadata = {} + + metadata = self.metadata() + sys_root = etpConst['systemroot'] + # load CONFIG_PROTECT and CONFIG_PROTECT_MASK + misc_settings = self._entropy.ClientSettings()['misc'] + col_protect = misc_settings['collisionprotect'] + + # remove files from system + directories = set() + directories_cache = set() + not_removed_due_to_collisions = set() + colliding_path_messages = set() + + protect_mask = metadata['config_protect+mask'] + if protect_mask is not None: + protect, mask = protect_mask + else: + protect, mask = set(), set() + protectskip = self._get_config_protect_skip() + + remove_content = None + try: + # simulate a removecontent list/set object + remove_content = [] + if metadata['removecontent_file'] is not None: + remove_content = Content.FileContentReader( + metadata['removecontent_file']) + + self._remove_content_from_system_loop( + installed_repository, + remove_content, directories, directories_cache, + not_removed_due_to_collisions, colliding_path_messages, + automerge_metadata, col_protect, protect, mask, protectskip, + sys_root) + + finally: + if hasattr(remove_content, "close"): + remove_content.close() + + if colliding_path_messages: + self._entropy.output( + "%s:" % (_("Collision found during removal of"),), + importance = 1, + level = "warning", + header = red(" ## ") + ) + + for path in sorted(colliding_path_messages): + self._entropy.output( + purple(path), + importance = 0, + level = "warning", + header = red(" ## ") + ) + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "Collision found during removal of %s - cannot overwrite" % ( + path,) + ) + + # removing files not removed from removecontent. + # it happened that boot services not removed due to + # collisions got removed from their belonging runlevels + # by postremove step. + # since this is a set, it is a mapped type, so every + # other instance around will feature this update + if not_removed_due_to_collisions: + def _filter(_path): + return _path not in not_removed_due_to_collisions + Content.filter_content_file( + metadata['removecontent_file'], + _filter) + + # now handle directories + directories = sorted(directories, reverse = True) + while True: + taint = False + for directory, dirtype in directories: + mydir = "%s%s" % (sys_root, directory,) + if dirtype == "link": + try: + mylist = os.listdir(mydir) + if not mylist: + try: + os.remove(mydir) + taint = True + except OSError: + pass + except OSError: + pass + elif dirtype == "dir": + try: + mylist = os.listdir(mydir) + if not mylist: + try: + os.rmdir(mydir) + taint = True + except OSError: + pass + except OSError: + pass + + if not taint: + break + + del directories_cache + del directories + + def _spm_remove_package(self, atom): + """ + Call Source Package Manager interface and tell it to remove our + just removed package. + + @return: execution status + @rtype: int + """ + spm = self._entropy.Spm() + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Removing from SPM: %s" % (atom,) + ) + return spm.remove_installed_package(self.metadata()) diff --git a/lib/entropy/client/interfaces/package/actions/action.py b/lib/entropy/client/interfaces/package/actions/action.py new file mode 100644 index 000000000..864627f21 --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/action.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import os + +from entropy.const import etpConst, const_convert_to_unicode +from entropy.i18n import _ +from entropy.output import darkred, blue + +import entropy.dep + +from .. import _content as Content + + +class PackageAction(object): + + """ + Base PackageAction object. Do not instantiate this and its subclasses + directly but rather use the PackageActionFactory. + """ + + _INFO_EXTS = ( + const_convert_to_unicode(".gz"), + const_convert_to_unicode(".bz2") + ) + + # Set a valid action name in subclasses + NAME = None + + def __init__(self, entropy_client, package_match, opts = None): + self._entropy = entropy_client + self._settings = self._entropy.Settings() + self._package_match = package_match + self._package_id, self._repository_id = package_match + if opts is None: + opts = {} + self._opts = opts + self._xterm_header = "" + self._content_files = [] + + def package_id(self): + """ + Return the package identifier of this object. + """ + return self._package_id + + def repository_id(self): + """ + Return the repository identifier of this object. + """ + return self._repository_id + + def atom(self): + """ + Return the package atom string of this object. + """ + repo = self._entropy.open_repository(self._repository_id) + return repo.retrieveAtom(self._package_id) + + def set_xterm_header(self, header): + """ + Set the xterm terminal header text that will prefix the activity title. + + @param header: the xterm header title + @type header: string + """ + self._xterm_header = header + + def setup(self): + """ + Setup the action metadata. There is no need to call this directly, + unless you want to pre-generate the whole PackageAction metadata. + This method will be called by start() anyway. + """ + raise NotImplementedError() + + def start(self): + """ + Execute the action. Return an exit status. + """ + acquired = False + exit_st = 1 + try: + acquired = not self._entropy.wait_resources() + if not acquired: + return 20 + exit_st = self._run() + finally: + if acquired: + self._entropy.unlock_resources() + + if exit_st != 0: + self._entropy.output( + blue(_("An error occured. Action aborted.")), + importance = 2, + level = "error", + header = darkred(" ## ") + ) + return exit_st + + def _run(self): + """ + This method is called by start() and this is where subclasses + must implement their "run()" logic. + This method must return an exit status code (int). + This method must call setup() at the beginning of its execution. + """ + raise NotImplementedError() + + def finalize(self): + """ + Finalize the object, release all its resources. + Subclasses must call this method in their overridden ones. + """ + # remove temporary content files + for content_file in self._content_files: + try: + os.remove(content_file) + except (OSError, IOError): + pass + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + raise NotImplementedError() + + @classmethod + def splitdebug_enabled(cls, entropy_client, pkg_match): + """ + Return whether splitdebug is enabled for package. + """ + settings = entropy_client.Settings() + # this is a SystemSettings.CachingList object + splitdebug = settings['splitdebug'] + splitdebug_mask = settings['splitdebug_mask'] + _pkg_id, pkg_repo = pkg_match + + def _generate_cache(lst_obj): + # compute the package matching then + pkg_matches = set() + for dep in lst_obj: + dep, repo_ids = entropy.dep.dep_get_match_in_repos(dep) + if repo_ids is not None: + if pkg_repo not in repo_ids: + # skip entry, not me + continue + dep_matches, _rc = entropy_client.atom_match( + dep, multi_match=True, multi_repo=True) + pkg_matches |= dep_matches + + # set cache back + lst_obj.set(pkg_matches) + return pkg_matches + + enabled = False + if not splitdebug: + # no entries, consider splitdebug always enabled + enabled = True + else: + # whitelist support + pkg_matches = splitdebug.get() + if pkg_matches is None: + pkg_matches = _generate_cache(splitdebug) + + # determine if it's enabled then + enabled = pkg_match in pkg_matches + + # if it's enabled, check whether it's blacklisted + if enabled: + # blacklist support + pkg_matches = splitdebug_mask.get() + if pkg_matches is None: + # compute the package matching + pkg_matches = _generate_cache(splitdebug_mask) + + enabled = pkg_match not in pkg_matches + + return enabled + + @classmethod + def get_standard_fetch_disk_path(cls, download): + """ + Return standard path where package is going to be downloaded. + "download" argument passed must come from + EntropyRepository.retrieveDownloadURL() + """ + return os.path.join(etpConst['entropypackagesworkdir'], download) + + def __repr__(self): + return "<%s at %s | metadata: %s" % ( + self.__class__.__name__, + hex(id(self)), + self.metadata()) + + def __str__(self): + return repr(self) + + def _get_info_directories(self): + """ + Return a list of `info` directories as declared in the + INFOPATH and INFODIR environment variable. + """ + info_dirs = os.getenv("INFOPATH", "").split(":") + info_dirs += os.getenv("INFODIR", "").split(":") + info_dirs = [const_convert_to_unicode( + os.path.normpath(x)) for x in info_dirs] + info_dirs.sort() + return info_dirs + + def _get_splitdebug_metadata(self): + """ + Return package metadata related to split debug files support. + """ + client_settings = self._entropy.ClientSettings() + misc_data = client_settings['misc'] + splitdebug = misc_data['splitdebug'] + splitdebug_dirs = misc_data['splitdebug_dirs'] + + metadata = { + 'splitdebug': splitdebug, + 'splitdebug_dirs': splitdebug_dirs, + } + return metadata + + def _package_splitdebug_enabled(self, pkg_match): + """ + Determine if splitdebug is enabled for the package being installed + or just fetched. This method should be called only if system-wide + splitdebug setting in client.conf is enabled already. + """ + return self.splitdebug_enabled(self._entropy, pkg_match) + + def _generate_content_file(self, content, package_id = None, + filter_splitdebug = False, + splitdebug = None, + splitdebug_dirs = None): + """ + Generate a file containing the package content metadata. + """ + content_path = None + try: + content_path = Content.generate_content_file( + content, package_id = package_id, + filter_splitdebug = filter_splitdebug, + splitdebug = splitdebug, + splitdebug_dirs = splitdebug_dirs) + return content_path + finally: + if content_path is not None: + self._content_files.append(content_path) + + def _generate_content_safety_file(self, content_safety): + """ + Generate a file containing the package content safety metadata. + """ + content_path = None + try: + content_path = Content.generate_content_safety_file( + content_safety) + return content_path + finally: + if content_path is not None: + self._content_files.append(content_path) + + @classmethod + def _get_url_name(cls, url): + """ + Given a mirror URL, returns a smaller string representing the URL name. + + @param url: URL string + @type url: string + @return: representative URL string + @rtype: string + """ + url_data = entropy.tools.spliturl(url) + url_name = url_data.netloc + url_scheme = url_data.scheme + if not url_scheme: + url_scheme = "unknown" + return "%s://%s" % (url_scheme, url_name,) + + @classmethod + def _get_licenses(cls, entropy_repository, package_id): + """ + Return a set of license identifiers for the given package. + """ + pkg_license = set() + r_license = entropy_repository.retrieveLicense(package_id) + if r_license is not None: + pkg_license |= set(r_license.split()) + return pkg_license diff --git a/lib/entropy/client/interfaces/package/actions/config.py b/lib/entropy/client/interfaces/package/actions/config.py new file mode 100644 index 000000000..4abcbe2a6 --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/config.py @@ -0,0 +1,211 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +from entropy.const import etpConst +from entropy.i18n import _ +from entropy.output import red, darkred, blue, brown + +from .action import PackageAction + + +class _PackageConfigAction(PackageAction): + """ + PackageAction used for package configuration. + """ + + NAME = "config" + + def __init__(self, entropy_client, package_match, opts = None): + """ + Object constructor. + """ + super(_PackageConfigAction, self).__init__( + entropy_client, package_match, opts = opts) + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageConfigAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + return self._meta + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + inst_repo = self._entropy.open_repository(self._repository_id) + + metadata['atom'] = inst_repo.retrieveAtom(self._package_id) + key, slot = inst_repo.retrieveKeySlot(self._package_id) + metadata['key'], metadata['slot'] = key, slot + metadata['version'] = inst_repo.retrieveVersion(self._package_id) + metadata['category'] = inst_repo.retrieveCategory(self._package_id) + metadata['name'] = inst_repo.retrieveName(self._package_id) + metadata['spm_repository'] = inst_repo.retrieveSpmRepository( + self._package_id) + + metadata['accept_license'] = self._get_licenses( + inst_repo, self._package_id) + metadata['phases'] = [] + metadata['phases'].append(self._config) + + self._meta = metadata + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + exit_st = 0 + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st + + def _configure_package(self): + """ + Configure the package. + """ + spm = self._entropy.Spm() + + self._entropy.output( + "SPM: %s" % ( + brown(_("configuration phase")), + ), + importance = 0, + header = red(" ## ") + ) + + try: + spm.execute_package_phase( + self._meta, self._meta, + self.NAME, "configure") + + except spm.PhaseFailure as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Phase execution failed with %s, %d" % ( + err.message, err.code)) + return err.code + + except spm.OutdatedPhaseError as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Source Package Manager is too old: %s" % ( + err)) + + err_msg = "%s: %s" % ( + brown(_("Source Package Manager is too old, please update it")), + err) + self._entropy.output( + err_msg, + importance = 1, + header = darkred(" ## "), + level = "error" + ) + return 1 + + except spm.PhaseError as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Phase execution error: %s" % ( + err)) + return 1 + + return 0 + + def _config(self): + """ + Execute the config phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Configuring package"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + txt = "%s: %s" % ( + blue(_("Configuring package")), + red(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + exit_st = self._configure_package() + if exit_st == 1: + txt = _("An error occured while trying to configure the package") + txt2 = "%s. %s: %s" % ( + red(_("Make sure that your system is healthy")), + blue(_("Error")), + exit_st, + ) + self._entropy.output( + darkred(txt), + importance = 1, + level = "error", + header = red(" ## ") + ) + self._entropy.output( + txt2, + importance = 1, + level = "error", + header = red(" ## ") + ) + + elif exit_st == 2: + txt = _("An error occured while trying to configure the package") + txt2 = "%s. %s: %s" % ( + red(_("It seems that Source Package Manager entry is missing")), + blue(_("Error")), + exit_st, + ) + self._entropy.output( + darkred(txt), + importance = 1, + level = "error", + header = red(" ## ") + ) + self._entropy.output( + txt2, + importance = 1, + level = "error", + header = red(" ## ") + ) + + return exit_st diff --git a/lib/entropy/client/interfaces/package/actions/fetch.py b/lib/entropy/client/interfaces/package/actions/fetch.py new file mode 100644 index 000000000..ec1a414e4 --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/fetch.py @@ -0,0 +1,1146 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import codecs +import errno +import os +import shutil +import stat + +from entropy.const import etpConst, const_debug_write, const_debug_enabled, \ + const_mkstemp +from entropy.client.mirrors import StatusInterface +from entropy.fetchers import UrlFetcher +from entropy.i18n import _ +from entropy.output import red, darkred, blue, purple, darkgreen +from entropy.security import Repository as RepositorySecurity + +import entropy.dep +import entropy.tools + +from .action import PackageAction + + +class _PackageFetchAction(PackageAction): + """ + PackageAction used for package download. + """ + + NAME = "fetch" + + def __init__(self, entropy_client, package_match, opts = None): + """ + Object constructor. + """ + super(_PackageFetchAction, self).__init__( + entropy_client, package_match, opts = opts) + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageFetchAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + return self._meta + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + metadata['fetch_abort_function'] = self._opts.get( + 'fetch_abort_function') + + # NOTE: if you want to implement download-to-dir feature in your + # client, you've found what you were looking for. + # fetch_path is the path where data should be downloaded + # it overrides default path + fetch_path = self._opts.get('fetch_path', None) + if fetch_path is not None: + if entropy.tools.is_valid_path(fetch_path): + metadata['fetch_path'] = fetch_path + + # if splitdebug is enabled, check if it's also enabled + # via package.splitdebug + splitdebug = metadata['splitdebug'] + if splitdebug: + splitdebug = self._package_splitdebug_enabled( + self._package_match) + + repo = self._entropy.open_repository(self._repository_id) + metadata['atom'] = repo.retrieveAtom(self._package_id) + metadata['slot'] = repo.retrieveSlot(self._package_id) + + inst_repo = self._entropy.installed_repository() + metadata['installed_package_id'], _inst_rc = inst_repo.atomMatch( + entropy.dep.dep_getkey(metadata['atom']), + matchSlot = metadata['slot']) + + cl_id = etpConst['system_settings_plugins_ids']['client_plugin'] + edelta_support = self._settings[cl_id]['misc']['edelta_support'] + metadata['edelta_support'] = edelta_support + metadata['checksum'] = repo.retrieveDigest(self._package_id) + sha1, sha256, sha512, gpg = repo.retrieveSignatures( + self._package_id) + signatures = { + 'sha1': sha1, + 'sha256': sha256, + 'sha512': sha512, + 'gpg': gpg, + } + metadata['signatures'] = signatures + extra_download = repo.retrieveExtraDownload(self._package_id) + if not splitdebug: + extra_download = [x for x in extra_download if \ + x['type'] != "debug"] + metadata['extra_download'] = extra_download + metadata['download'] = repo.retrieveDownloadURL( + self._package_id) + + # export main package download path to metadata + # this is actually used by PackageKit backend in order + # to signal downloaded package files + metadata['pkgpath'] = self._get_fetch_disk_path( + metadata['download'], metadata) + + metadata['phases'] = [] + + if not self._repository_id.endswith(etpConst['packagesext']): + + dl_check = self._check_package_path_download( + metadata['download'], None) + dl_fetch = dl_check < 0 + + if not dl_fetch: + for extra_download in metadata['extra_download']: + dl_check = self._check_package_path_download( + extra_download['download'], None) + if dl_check < 0: + # dl_check checked again right below + break + + if dl_check < 0: + metadata['phases'].append(self._fetch) + + metadata['phases'].append(self._checksum) + + def _check_matching_size(download, size): + d_path = self._get_fetch_disk_path(download, metadata) + try: + st = os.stat(d_path) + return size == st.st_size + except OSError as err: + if err.errno != errno.ENOENT: + raise + return False + + matching_size = _check_matching_size(metadata['download'], + repo.retrieveSize(self._package_id)) + if matching_size: + for extra_download in metadata['extra_download']: + matching_size = _check_matching_size( + extra_download['download'], + extra_download['size']) + if not matching_size: + break + + # downloading binary package + # if file exists, first checksum then fetch + if matching_size: + metadata['phases'].reverse() + + self._meta = metadata + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + exit_st = 0 + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st + + def _get_fetch_disk_path(self, download, metadata): + """ + Return proper Entropy package store path. + """ + if 'fetch_path' in metadata: + # only supported by fetch action, multifetch also unsupported + pkg_disk_path = os.path.join(metadata['fetch_path'], + os.path.basename(download)) + else: + pkg_disk_path = self.get_standard_fetch_disk_path(download) + return pkg_disk_path + + def _check_package_path_download(self, download, checksum): + """ + Internal function that verifies if a package tarball is already + available locally and quickly computes its md5. Please note that + stronger crypto hash functions are used during the real package + validation phase. + """ + pkg_path = self._get_fetch_disk_path(download, self._meta) + + if not os.path.isfile(pkg_path): + return -1 + if checksum is None: + return 0 + + if entropy.tools.compare_md5(pkg_path, checksum): + return 0 + return -2 + + def _build_uris_list(self, original_repo, repository_id): + """ + Build a list of possible download URIs for the given repository. + Handle fallback to the original repository (the one originally + containing the package). + """ + avail_data = self._settings['repositories']['available'] + product = self._settings['repositories']['product'] + uris = [] + + plain_packages = avail_data[repository_id]['plain_packages'] + for uri in plain_packages: + expanded_uri = entropy.tools.expand_plain_package_mirror( + uri, product, original_repo) + uris.append(expanded_uri) + + uris.reverse() + uris.extend(avail_data[repository_id]['packages'][::-1]) + + return uris + + def _approve_edelta(self, url, installed_package_id, package_digest): + """ + Approve Entropy package delta support for given url, checking if + a previously fetched package is available. + + @return: edelta URL to download and previously downloaded package path + or None if edelta is not available + @rtype: tuple of strings or None + """ + inst_repo = self._entropy.installed_repository() + download_url = inst_repo.retrieveDownloadURL(installed_package_id) + installed_digest = inst_repo.retrieveDigest(installed_package_id) + installed_fetch_path = self._get_fetch_disk_path( + download_url, self._meta) + + edelta_local_approved = False + try: + edelta_local_approved = entropy.tools.compare_md5( + installed_fetch_path, installed_digest) + except (OSError, IOError) as err: + const_debug_write( + __name__, "_approve_edelta, error: %s" % (err,)) + return + + if edelta_local_approved: + hash_tag = installed_digest + package_digest + edelta_file_name = entropy.tools.generate_entropy_delta_file_name( + os.path.basename(download_url), + os.path.basename(url), + hash_tag) + edelta_url = os.path.join( + os.path.dirname(url), + etpConst['packagesdeltasubdir'], + edelta_file_name) + + return edelta_url, installed_fetch_path + + def _setup_differential_download(self, fetcher, url, resume, + fetch_path, repository, package_id): + """ + Setup differential download in case of URL supporting it. + Internal function. + + @param fetcher: UrlFetcher or MultipleUrlFetcher class + @param url: URL to check differential download against + @type url: string + @param resume: resume support + @type resume: bool + @param fetch_path: path where package file will be saved + @type fetch_path: string + @param repository: repository identifier belonging to package file + @type repository: string + @param package_id: package identifier belonging to repository identifier + @type package_id: int + """ + # no resume? no party? + if not resume: + const_debug_write(__name__, + "_setup_differential_download(%s) %s" % ( + url, "resume disabled")) + return + + if not fetcher.supports_differential_download(url): + # no differential download support + const_debug_write(__name__, + "_setup_differential_download(%s) %s" % ( + url, "no differential download support")) + return + + # this is the fetch path of the file that is going to be downloaded + # not going to overwrite it if a file is located there because + # user would want a resume for sure in first place + if os.path.isfile(fetch_path): + const_debug_write(__name__, + "_setup_differential_download(%s) %s" % ( + url, "fetch path already exists, not overwriting")) + return + + pkg_repo = self._entropy.open_repository(repository) + keyslot = pkg_repo.retrieveKeySlot(package_id) + if keyslot is None: + # fucked up entry, not dealing with it here + const_debug_write(__name__, + "_setup_differential_download(%s) %s" % ( + url, "key, slot fucked up")) + return + + key, slot = keyslot + inst_repo = self._entropy.installed_repository() + pkg_ids = inst_repo.searchKeySlot(key, slot) + if not pkg_ids: + # not installed, nothing to use as diff download + const_debug_write(__name__, + "_setup_differential_download(%s) %s" % ( + url, "no installed packages")) + return + + # grab the highest, we don't know if user was able to mess up + # its installed packages repository + pkg_id = max(pkg_ids) + download_url = inst_repo.retrieveDownloadURL(pkg_id) + installed_fetch_path = self._get_fetch_disk_path( + download_url, self._meta) + + try: + shutil.copyfile(installed_fetch_path, fetch_path) + except (OSError, IOError, shutil.Error) as err: + const_debug_write( + __name__, + "_setup_differential_download(%s), copyfile error: %s" % ( + url, err)) + try: + os.remove(fetch_path) + except OSError: + pass + return + + try: + user = os.stat(installed_fetch_path)[stat.ST_UID] + group = os.stat(installed_fetch_path)[stat.ST_GID] + os.chown(fetch_path, user, group) + except (OSError, IOError) as err: + const_debug_write( + __name__, + "_setup_differential_download(%s), chown error: %s" % ( + url, err)) + return + + try: + shutil.copystat(installed_fetch_path, fetch_path) + except (OSError, IOError, shutil.Error) as err: + const_debug_write( + __name__, + "_setup_differential_download(%s), copystat error: %s" % ( + url, err)) + return + + const_debug_write( + __name__, + "_setup_differential_download(%s) copied to %s" % ( + url, fetch_path)) + + def _try_edelta_fetch(self, url, save_path, resume): + + # no edelta support enabled + if not self._meta.get('edelta_support'): + return 1, 0.0 + # edelta enabled? + if not entropy.tools.is_entropy_delta_available(): + return 1, 0.0 + + # when called by _fetch_file, which is called by _download_package + # which is called by _match_checksum, which is called by + # multi_match_checksum, installed_package_id metadatum is not available + # So, be fault tolerant. + installed_package_id = self._meta['installed_package_id'] + + # fresh install, cannot fetch edelta, edelta only works for installed + # packages, by design. + if installed_package_id == -1: + return 1, 0.0 + + edelta_approve = self._approve_edelta(url, installed_package_id, + self._meta['checksum']) + + if edelta_approve is None: + # edelta not available, give up + return 1, 0.0 + edelta_url, installed_fetch_path = edelta_approve + + # check if edelta file is available online + edelta_save_path = save_path + etpConst['packagesdeltaext'] + + max_tries = 2 + edelta_approved = False + data_transfer = 0 + download_plan = [(edelta_url, edelta_save_path) for _x in \ + range(max_tries)] + + delta_resume = resume + fetch_abort_function = self._meta.get('fetch_abort_function') + fetch_errors = ( + UrlFetcher.TIMEOUT_FETCH_ERROR, + UrlFetcher.GENERIC_FETCH_ERROR, + UrlFetcher.GENERIC_FETCH_WARN, + ) + + for delta_url, delta_save in download_plan: + + delta_fetcher = self._entropy._url_fetcher(delta_url, + delta_save, resume = delta_resume, + abort_check_func = fetch_abort_function) + + try: + # make sure that we don't need to abort already + # doing the check here avoids timeouts + if fetch_abort_function != None: + fetch_abort_function() + + delta_checksum = delta_fetcher.download() + data_transfer = delta_fetcher.get_transfer_rate() + del delta_fetcher + except KeyboardInterrupt: + return -100, data_transfer + + except NameError: + raise + + except Exception: + return -1, data_transfer + + if delta_checksum in fetch_errors: + # make sure this points to hell + delta_resume = False + # retry + continue + + # now check + tmp_save_path = save_path + ".edelta_pkg_tmp" + # yay, we can apply the delta and cook the new package file! + try: + entropy.tools.apply_entropy_delta(installed_fetch_path, + delta_save, tmp_save_path) + except IOError: + # make sure this points to the hell + delta_resume = False + # retry + try: + os.remove(tmp_save_path) + except (OSError, IOError): + pass + continue + + os.rename(tmp_save_path, save_path) + edelta_approved = True + break + + if edelta_approved: + # we can happily return + return 0, data_transfer + # error, give up with the edelta stuff + return 1, data_transfer + + def _fetch_file(self, url, save_path, digest = None, resume = True, + download = None, package_id = None, repository = None): + """ + Internal method. Try to download the package file. + """ + + def do_stfu_rm(xpath): + try: + os.remove(xpath) + except OSError: + pass + + def do_get_md5sum(path): + try: + return entropy.tools.md5sum(path) + except IOError: + return None + except OSError: + return None + + fetch_abort_function = self._meta.get('fetch_abort_function') + filepath_dir = os.path.dirname(save_path) + + if not os.path.isdir(os.path.realpath(filepath_dir)): + try: + os.remove(filepath_dir) + except OSError as err: + const_debug_write(__name__, + "_fetch_file.remove, %s, error: %s" % ( + filepath_dir, err)) + try: + os.makedirs(filepath_dir, 0o755) + except OSError as err: + const_debug_write(__name__, + "_fetch_file.makedirs, %s, error: %s" % ( + filepath_dir, err)) + return -1, 0, False + + exit_st, data_transfer = self._try_edelta_fetch(url, save_path, resume) + if exit_st == 0: + return exit_st, data_transfer, False + elif exit_st < 0: # < 0 errors are unrecoverable + return exit_st, data_transfer, False + # otherwise, just fallback to package download + + existed_before = False + if os.path.isfile(save_path) and os.path.exists(save_path): + existed_before = True + + fetch_intf = self._entropy._url_fetcher( + url, save_path, resume = resume, + abort_check_func = fetch_abort_function) + if (download is not None) and (package_id is not None) and \ + (repository is not None) and (exit_st == 0): + fetch_path = self._get_fetch_disk_path(download, self._meta) + self._setup_differential_download( + self._entropy._url_fetcher, url, + resume, fetch_path, repository, package_id) + + data_transfer = 0 + resumed = False + try: + # make sure that we don't need to abort already + # doing the check here avoids timeouts + if fetch_abort_function != None: + fetch_abort_function() + + fetch_checksum = fetch_intf.download() + data_transfer = fetch_intf.get_transfer_rate() + resumed = fetch_intf.is_resumed() + except KeyboardInterrupt: + return -100, data_transfer, resumed + + except NameError: + raise + + except: + if const_debug_enabled(): + self._entropy.output( + "fetch_file:", + importance = 1, + level = "warning", + header = red(" ## ") + ) + entropy.tools.print_traceback() + if (not existed_before) or (not resume): + do_stfu_rm(save_path) + return -1, data_transfer, resumed + + if fetch_checksum == UrlFetcher.GENERIC_FETCH_ERROR: + # !! not found + # maybe we already have it? + # this handles the case where network is unavailable + # but file is already downloaded + fetch_checksum = do_get_md5sum(save_path) + if (fetch_checksum != digest) or fetch_checksum is None: + return -3, data_transfer, resumed + + elif fetch_checksum == UrlFetcher.TIMEOUT_FETCH_ERROR: + # maybe we already have it? + # this handles the case where network is unavailable + # but file is already downloaded + fetch_checksum = do_get_md5sum(save_path) + if (fetch_checksum != digest) or fetch_checksum is None: + return -4, data_transfer, resumed + + if digest and (fetch_checksum != digest): + # not properly downloaded + if (not existed_before) or (not resume): + do_stfu_rm(save_path) + return -2, data_transfer, resumed + + return 0, data_transfer, resumed + + def _download_package(self, package_id, repository_id, download, save_path, + digest = False, resume = True): + + avail_data = self._settings['repositories']['available'] + excluded_data = self._settings['repositories']['excluded'] + repo = self._entropy.open_repository(repository_id) + # grab original repo, if any and use it if available + # this is done in order to support "equo repo merge" feature + # allowing client-side repository package metadata moves. + original_repo = repo.getInstalledPackageRepository(package_id) + + if (original_repo != repository_id) and ( + original_repo not in avail_data) and ( + original_repo is not None): + # build up a new uris list, at least try, hoping that + # repository is just shadowing original_repo + # for example: original_repo got copied to repository, without + # copying packages, which would be useless. like it happens + # with sabayon-weekly + uris = self._build_uris_list(original_repo, repository_id) + else: + if original_repo in avail_data: + uris = avail_data[original_repo]['packages'][::-1] + if repository_id in avail_data: + uris += avail_data[repository_id]['packages'][::-1] + elif original_repo in excluded_data: + uris = excluded_data[original_repo]['packages'][::-1] + if repository_id in avail_data: + uris += avail_data[repository_id]['packages'][::-1] + else: + uris = avail_data[repository_id]['packages'][::-1] + + remaining = set(uris) + mirror_status = StatusInterface() + + mirrorcount = 0 + for uri in uris: + + if not remaining: + # tried all the mirrors, quitting for error + mirror_status.set_working_mirror(None) + return 3 + + mirror_status.set_working_mirror(uri) + mirrorcount += 1 + mirror_count_txt = "( mirror #%s ) " % (mirrorcount,) + url = uri + "/" + download + + # check if uri is sane + if mirror_status.get_failing_mirror_status(uri) >= 30: + # ohohoh! + # set to 30 for convenience + mirror_status.set_failing_mirror_status(uri, 30) + mytxt = mirror_count_txt + mytxt += blue(" %s: ") % (_("Mirror"),) + mytxt += red(self._get_url_name(uri)) + mytxt += " - %s." % (_("maximum failure threshold reached"),) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = red(" ## ") + ) + + if mirror_status.get_failing_mirror_status(uri) == 30: + # put to 75 then decrement by 4 so we + # won't reach 30 anytime soon ahahaha + mirror_status.add_failing_mirror(uri, 45) + elif mirror_status.get_failing_mirror_status(uri) > 31: + # now decrement each time this point is reached, + # if will be back < 30, then equo will try to use it again + mirror_status.add_failing_mirror(uri, -4) + else: + # put to 0 - reenable mirror, welcome back uri! + mirror_status.set_failing_mirror_status(uri, 0) + + remaining.discard(uri) + continue + + do_resume = resume + timeout_try_count = 50 + + while True: + + txt = mirror_count_txt + txt += blue("%s: ") % (_("Downloading from"),) + txt += red(self._get_url_name(uri)) + self._entropy.output( + txt, + importance = 1, + level = "warning", + header = red(" ## ") + ) + + exit_st, data_transfer, resumed = self._fetch_file( + url, + save_path, + download = download, + package_id = package_id, + repository = repository_id, + digest = digest, + resume = do_resume + ) + + if exit_st == 0: + txt = mirror_count_txt + txt += "%s: " % ( + blue(_("Successfully downloaded from")), + ) + txt += red(self._get_url_name(uri)) + human_bytes = entropy.tools.bytes_into_human( + data_transfer) + txt += " %s %s/%s" % (_("at"), + human_bytes, _("second"),) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + mirror_status.set_working_mirror(None) + return 0 + + elif resumed and (exit_st not in (-3, -4, -100,)): + do_resume = False + continue + + error_message = mirror_count_txt + error_message += blue("%s: %s") % ( + _("Error downloading from"), + red(self._get_url_name(uri)), + ) + + # something bad happened + if exit_st == -1: + error_message += " - %s." % ( + _("file not available on this mirror"),) + + elif exit_st == -2: + mirror_status.add_failing_mirror(uri, 1) + error_message += " - %s." % (_("wrong checksum"),) + + # If file is fetched (with no resume) and its + # complete better to enforce resume to False. + if (data_transfer < 1) and do_resume: + error_message += " %s." % ( + _("Disabling resume"),) + do_resume = False + continue + + elif exit_st == -3: + mirror_status.add_failing_mirror(uri, 3) + error_message += " - %s." % (_("not found"),) + + elif exit_st == -4: # timeout! + timeout_try_count -= 1 + if timeout_try_count > 0: + error_message += " - %s." % ( + _("timeout, retrying on this mirror"),) + else: + error_message += " - %s." % ( + _("timeout, giving up"),) + + elif exit_st == -100: + error_message += " - %s." % ( + _("discarded download"),) + else: + mirror_status.add_failing_mirror(uri, 5) + error_message += " - %s." % (_("unknown reason"),) + + self._entropy.output( + error_message, + importance = 1, + level = "warning", + header = red(" ## ") + ) + + if exit_st == -4: # timeout + if timeout_try_count > 0: + continue + + elif exit_st == -100: # user discarded fetch + mirror_status.set_working_mirror(None) + return 1 + + remaining.discard(uri) + # make sure we don't have nasty issues + if not remaining: + mirror_status.set_working_mirror(None) + return 3 + + break + + mirror_status.set_working_mirror(None) + return 0 + + def _fetch(self): + """ + Execute the package fetch phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Fetching"), + os.path.basename(self._meta['download']), + ) + self._entropy.set_title(xterm_title) + + def _fetch(download, checksum): + txt = "%s: %s" % ( + blue(_("Downloading")), + red(os.path.basename(download)),) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + pkg_disk_path = self._get_fetch_disk_path(download, self._meta) + return self._download_package( + self._package_id, + self._repository_id, + download, + pkg_disk_path, + checksum + ) + + exit_st = _fetch(self._meta['download'], self._meta['checksum']) + if exit_st == 0: + # go ahead with extra_download + for extra_download in self._meta['extra_download']: + exit_st = _fetch(extra_download['download'], + extra_download['md5']) + if exit_st != 0: + break + + if exit_st != 0: + txt = "%s. %s: %s" % ( + red(_("Package cannot be fetched. Try to update repositories")), + blue(_("Error")), + exit_st, + ) + self._entropy.output( + txt, + importance = 1, + level = "error", + header = darkred(" ## ") + ) + + return exit_st + + def _match_checksum(self, package_id, repository, checksum, download, + signatures): + """ + Verify package checksum and return an exit status code. + """ + misc_settings = self._entropy.ClientSettings()['misc'] + enabled_hashes = misc_settings['packagehashes'] + + pkg_disk_path = self._get_fetch_disk_path(download, self._meta) + pkg_disk_path_mtime = pkg_disk_path + etpConst['packagemtimefileext'] + + def do_mtime_validation(): + enc = etpConst['conf_encoding'] + try: + with codecs.open(pkg_disk_path_mtime, + "r", encoding=enc) as mt_f: + stored_mtime = mt_f.read().strip() + except (OSError, IOError) as err: + if err.errno != errno.ENOENT: + raise + return 1 + + try: + cur_mtime = str(os.path.getmtime(pkg_disk_path)) + except (OSError, IOError) as err: + if err.errno != errno.ENOENT: + raise + return 2 + + if cur_mtime == stored_mtime: + return 0 + return 1 + + def do_store_mtime(): + enc = etpConst['conf_encoding'] + try: + with codecs.open(pkg_disk_path_mtime, + "w", encoding=enc) as mt_f: + cur_mtime = str(os.path.getmtime(pkg_disk_path)) + mt_f.write(cur_mtime) + except (OSError, IOError) as err: + if err.errno != errno.ENOENT: + raise + + def do_compare_gpg(pkg_path, hash_val): + + try: + repo_sec = self._entropy.RepositorySecurity() + except RepositorySecurity.GPGServiceNotAvailable: + return None + + # check if we have repository pubkey + try: + if not repo_sec.is_pubkey_available(repository): + return None + except repo_sec.KeyExpired: + # key is expired + return None + + # write gpg signature to disk for verification + tmp_fd, tmp_path = const_mkstemp(prefix="do_compare_gpg") + with os.fdopen(tmp_fd, "w") as tmp_f: + tmp_f.write(hash_val) + + try: + # actually verify + valid, err_msg = repo_sec.verify_file( + repository, pkg_path, tmp_path) + finally: + os.remove(tmp_path) + + if valid: + return True + + if err_msg: + self._entropy.output( + "%s: %s, %s" % ( + darkred(_("Package signature verification error for")), + purple("GPG"), + err_msg, + ), + importance = 0, + level = "error", + header = darkred(" ## ") + ) + return False + + signature_vry_map = { + 'sha1': entropy.tools.compare_sha1, + 'sha256': entropy.tools.compare_sha256, + 'sha512': entropy.tools.compare_sha512, + 'gpg': do_compare_gpg, + } + + def do_signatures_validation(signatures): + # check signatures, if available + if isinstance(signatures, dict): + for hash_type in sorted(signatures): + + hash_val = signatures[hash_type] + # NOTE: workaround bug on unreleased + # entropy versions + if hash_val in signatures: + continue + if hash_val is None: + continue + if hash_type not in enabled_hashes: + self._entropy.output( + "%s %s" % ( + purple(hash_type.upper()), + darkgreen(_("disabled")), + ), + importance = 0, + level = "info", + header = " : " + ) + continue + + cmp_func = signature_vry_map.get(hash_type) + if cmp_func is None: + continue + + self._entropy.output( + "%s: %s" % ( + blue(_("Checking package signature")), + purple(hash_type.upper()), + ), + importance = 0, + level = "info", + header = red(" ## "), + back = True + ) + + valid = cmp_func(pkg_disk_path, hash_val) + if valid is None: + self._entropy.output( + "%s '%s' %s" % ( + darkred(_("Package signature verification")), + purple(hash_type.upper()), + darkred(_("temporarily unavailable")), + ), + importance = 0, + level = "warning", + header = darkred(" ## ") + ) + continue + + if not valid: + self._entropy.output( + "%s: %s %s" % ( + darkred(_("Package signature")), + purple(hash_type.upper()), + darkred(_("does not match the recorded one")), + ), + importance = 0, + level = "error", + header = darkred(" ## ") + ) + return 1 + + self._entropy.output( + "%s %s" % ( + purple(hash_type.upper()), + darkgreen(_("matches")), + ), + importance = 0, + level = "info", + header = " : " + ) + + return 0 + + dlcount = 0 + match = False + max_dlcount = 5 + + while dlcount <= max_dlcount: + + self._entropy.output( + blue(_("Checking package checksum...")), + importance = 0, + level = "info", + header = red(" ## "), + back = True + ) + + dlcheck = self._check_package_path_download( + download, checksum) + if dlcheck == 0: + basef = os.path.basename(download) + self._entropy.output( + "%s: %s" % ( + blue(_("Package checksum matches")), + darkgreen(basef), + ), + importance = 0, + level = "info", + header = red(" ## ") + ) + + # check if package has been already checked + dlcheck = do_mtime_validation() + if dlcheck != 0: + dlcheck = do_signatures_validation(signatures) + + if dlcheck == 0: + do_store_mtime() + match = True + break # file downloaded successfully + + if dlcheck != 0: + dlcount += 1 + mytxt = _("Checksum does not match. Download attempt #%s") % ( + dlcount, + ) + self._entropy.output( + darkred(mytxt), + importance = 0, + level = "warning", + header = darkred(" ## ") + ) + + # Unfortunately, disabling resume makes possible to recover + # from bad download data. trying to resume would do more harm + # than good in the majority of cases. + fetch = self._download_package( + package_id, + repository, + download, + pkg_disk_path, + checksum, + resume = False + ) + + if fetch != 0: + self._entropy.output( + blue(_("Cannot properly fetch package! Quitting.")), + importance = 0, + level = "error", + header = darkred(" ## ") + ) + return fetch + + # package is fetched, let's loop one more time + # to make sure to run all the checksum checks + continue + + if not match: + txt = _("Cannot fetch package or checksum does not match") + txt2 = _("Try to download latest repositories") + for txt in (txt, txt2,): + self._entropy.output( + "%s." % ( + blue(txt), + ), + importance = 0, + level = "info", + header = red(" ## ") + ) + return 1 + + return 0 + + def _checksum(self): + """ + Execute the package checksum validation phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Verifying"), + os.path.basename(self._meta['download']), + ) + self._entropy.set_title(xterm_title) + + exit_st = self._match_checksum(self._package_id, + self._repository_id, self._meta['checksum'], + self._meta['download'], self._meta['signatures']) + if exit_st != 0: + return exit_st + + for extra_download in self._meta['extra_download']: + download = extra_download['download'] + checksum = extra_download['md5'] + signatures = { + 'sha1': extra_download['sha1'], + 'sha256': extra_download['sha256'], + 'sha512': extra_download['sha512'], + 'gpg': extra_download['gpg'], + } + exit_st = self._match_checksum( + self._package_id, + self._repository_id, checksum, + download, signatures) + if exit_st != 0: + return exit_st + + return 0 diff --git a/lib/entropy/client/interfaces/package/actions/install.py b/lib/entropy/client/interfaces/package/actions/install.py new file mode 100644 index 000000000..91f58adc7 --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/install.py @@ -0,0 +1,1780 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import errno +import os +import shutil +import stat +import time + +from entropy.const import etpConst, const_convert_to_unicode, \ + const_convert_to_rawstring, const_is_python3 +from entropy.exceptions import EntropyException +from entropy.i18n import _ +from entropy.output import darkred, red, purple, brown, blue, darkgreen, teal + +import entropy.dep +import entropy.tools + +from ._manage import _PackageInstallRemoveAction + +from .. import _content as Content + + +class _PackageInstallAction(_PackageInstallRemoveAction): + """ + PackageAction used for package installation. + """ + + class InvalidArchitecture(EntropyException): + """ + Raised when a package for another architecture is attempted + to be installed. + """ + + NAME = "install" + + def __init__(self, entropy_client, package_match, opts = None): + """ + Object constructor. + """ + super(_PackageInstallAction, self).__init__( + entropy_client, package_match, opts = opts) + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageInstallAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + inst_repo = self._entropy.installed_repository() + repo = self._entropy.open_repository(self._repository_id) + + misc_settings = self._entropy.ClientSettings()['misc'] + metadata['edelta_support'] = misc_settings['edelta_support'] + is_package_repo = self._repository_id.endswith( + etpConst['packagesext']) + + # if splitdebug is enabled, check if it's also enabled + # via package.splitdebug + if metadata['splitdebug']: + # yeah, this has to affect exported splitdebug setting + # because it is read during package files installation + # Older splitdebug data was in the same package file of + # the actual content. Later on, splitdebug data was moved + # to its own package file that gets downloaded and unpacked + # only if required (if splitdebug is enabled) + metadata['splitdebug'] = self._package_splitdebug_enabled( + self._package_match) + + # fetch abort function + metadata['fetch_abort_function'] = self._opts.get( + 'fetch_abort_function') + + install_source = etpConst['install_sources']['unknown'] + meta_inst_source = self._opts.get('install_source', install_source) + if meta_inst_source in list(etpConst['install_sources'].values()): + install_source = meta_inst_source + metadata['install_source'] = install_source + + metadata['already_protected_config_files'] = {} + metadata['configprotect_data'] = [] + metadata['triggers'] = {} + metadata['atom'] = repo.retrieveAtom(self._package_id) + metadata['slot'] = repo.retrieveSlot(self._package_id) + + ver, tag, rev = repo.getVersioningData(self._package_id) + metadata['version'] = ver + metadata['versiontag'] = tag + metadata['revision'] = rev + + metadata['extra_download'] = [] + metadata['splitdebug_pkgfile'] = True + if not is_package_repo: + metadata['splitdebug_pkgfile'] = False + extra_download = repo.retrieveExtraDownload(self._package_id) + if not metadata['splitdebug']: + extra_download = [x for x in extra_download if \ + x['type'] != "debug"] + metadata['extra_download'] += extra_download + + metadata['category'] = repo.retrieveCategory(self._package_id) + metadata['download'] = repo.retrieveDownloadURL(self._package_id) + metadata['name'] = repo.retrieveName(self._package_id) + metadata['checksum'] = repo.retrieveDigest(self._package_id) + sha1, sha256, sha512, gpg = repo.retrieveSignatures(self._package_id) + signatures = { + 'sha1': sha1, + 'sha256': sha256, + 'sha512': sha512, + 'gpg': gpg, + } + metadata['signatures'] = signatures + metadata['conflicts'] = self._get_package_conflicts( + repo, self._package_id) + + description = repo.retrieveDescription(self._package_id) + if description: + if len(description) > 74: + description = description[:74].strip() + description += "..." + metadata['description'] = description + + # this is set by __install_package() and required by spm_install + # phase + metadata['installed_package_id'] = None + metadata['remove_package_id'] = -1 + + metadata['remove_metaopts'] = { + 'removeconfig': True, + } + metadata['remove_metaopts'].update( + self._opts.get('remove_metaopts', {})) + + metadata['merge_from'] = None + mf = self._opts.get('merge_from') + if mf is not None: + metadata['merge_from'] = const_convert_to_unicode(mf) + metadata['removeconfig'] = self._opts.get('removeconfig', False) + + remove_package_id, _inst_rc = inst_repo.atomMatch( + entropy.dep.dep_getkey(metadata['atom']), + matchSlot = metadata['slot']) + metadata['remove_package_id'] = remove_package_id + + # collects directories whose content has been modified + # this information is then handed to the Trigger + metadata['affected_directories'] = set() + metadata['affected_infofiles'] = set() + + # smartpackage ? + metadata['smartpackage'] = False + # set unpack dir and image dir + if is_package_repo: + + try: + compiled_arch = repo.getSetting("arch") + arch_fine = compiled_arch == etpConst['currentarch'] + except KeyError: + arch_fine = True # sorry, old db, cannot check + + if not arch_fine: + raise self.InvalidArchitecture( + "Package compiled for a different architecture") + + repo_data = self._settings['repositories'] + repo_meta = repo_data['available'][self._repository_id] + metadata['smartpackage'] = repo_meta['smartpackage'] + metadata['pkgpath'] = repo_meta['pkgpath'] + + else: + metadata['pkgpath'] = self.get_standard_fetch_disk_path( + metadata['download']) + + metadata['unpackdir'] = etpConst['entropyunpackdir'] + \ + os.path.sep + self._escape_path(metadata['download']) + + metadata['imagedir'] = metadata['unpackdir'] + os.path.sep + \ + etpConst['entropyimagerelativepath'] + + metadata['pkgdbpath'] = os.path.join(metadata['unpackdir'], + "edb/pkg.db") + + if metadata['remove_package_id'] == -1: + # nothing to remove, fresh install + metadata['removecontent_file'] = None + else: + metadata['removeatom'] = inst_repo.retrieveAtom( + metadata['remove_package_id']) + + # generate content file + content = inst_repo.retrieveContentIter( + metadata['remove_package_id'], + order_by="file", reverse=True) + metadata['removecontent_file'] = \ + self._generate_content_file(content) + + remove_trigger = inst_repo.getTriggerData( + metadata['remove_package_id']) + metadata['triggers']['remove'] = remove_trigger + + remove_trigger['affected_directories'] = \ + metadata['affected_directories'] + remove_trigger['affected_infofiles'] = \ + metadata['affected_infofiles'] + + remove_trigger['spm_repository'] = inst_repo.retrieveSpmRepository( + self._package_id) + remove_trigger.update(splitdebug_metadata) + + remove_trigger['accept_license'] = self._get_licenses( + inst_repo, metadata['remove_package_id']) + + # setup config_protect and config_protect_mask metadata before it's + # too late. + protect = self._get_config_protect_metadata( + inst_repo, metadata['remove_package_id']) + metadata.update(protect) + + metadata['phases'] = [] + if metadata['conflicts']: + metadata['phases'].append(self._remove_conflicts) + + if metadata['merge_from']: + metadata['phases'].append(self._merge) + else: + metadata['phases'].append(self._unpack) + + # preinstall placed before preremove in order + # to respect Spm order + metadata['phases'].append(self._setup) + metadata['phases'].append(self._pre_install) + + metadata['phases'].append(self._install) + if metadata['remove_package_id'] != -1: + metadata['phases'].append(self._pre_remove) + metadata['phases'].append(self._install_clean) + metadata['phases'].append(self._post_remove) + metadata['phases'].append(self._post_remove_install) + + metadata['phases'].append(self._install_spm) + metadata['phases'].append(self._post_install) + metadata['phases'].append(self._cleanup) + + install_trigger = repo.getTriggerData(self._package_id) + metadata['triggers']['install'] = install_trigger + + install_trigger['unpackdir'] = metadata['unpackdir'] + install_trigger['imagedir'] = metadata['imagedir'] + install_trigger['spm_repository'] = repo.retrieveSpmRepository( + self._package_id) + + metadata['accept_license'] = self._get_licenses( + repo, self._package_id) + install_trigger['accept_license'] = metadata['accept_license'] + + install_trigger.update(splitdebug_metadata) + + self._meta = metadata + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + spm_class = self._entropy.Spm_class() + exit_st = spm_class.entropy_install_setup_hook( + self._entropy, self._meta) + if exit_st != 0: + return exit_st + + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st + + def _escape_path(self, path): + """ + Some applications (like ld) don't like ":" in path, others just don't + escape paths at all. So, it's better to avoid to use field separators + in path. + """ + path = path.replace(":", "_") + path = path.replace("~", "_") + return path + + def _get_package_conflicts(self, entropy_repository, package_id): + """ + Return a set of conflict dependencies for the given package. + """ + conflicts = entropy_repository.retrieveConflicts(package_id) + inst_repo = self._entropy.installed_repository() + + found_conflicts = set() + for conflict in conflicts: + inst_package_id, _inst_rc = inst_repo.atomMatch(conflict) + if inst_package_id == -1: + continue + + # check if the package shares the same key and slot + match_data = entropy_repository.retrieveKeySlot(package_id) + installed_match_data = inst_repo.retrieveKeySlot(inst_package_id) + if match_data != installed_match_data: + found_conflicts.add(inst_package_id) + + # auto conflicts support + found_conflicts |= self._entropy._generate_dependency_inverse_conflicts( + (package_id, entropy_repository.name), just_id=True) + + return found_conflicts + + def _remove_conflicts(self): + """ + Execute the package conflicts removal phase. + """ + inst_repo = self._entropy.installed_repository() + confl_package_ids = [x for x in self._meta['conflicts'] if \ + inst_repo.isPackageIdAvailable(x)] + if not confl_package_ids: + return 0 + + # calculate removal dependencies + # system_packages must be False because we should not exclude + # them from the dependency tree in any case. Also, we cannot trigger + # DependenciesNotRemovable() exception, too. + proposed_pkg_ids = self._entropy.get_removal_queue(confl_package_ids, + system_packages = False) + # we don't want to remove the whole inverse dependencies of course, + # but just the conflicting ones, in a proper order + package_ids = [x for x in proposed_pkg_ids if x in confl_package_ids] + # make sure that every package is listed in package_ids before + # proceeding, cannot keep packages behind anyway, and must be fault + # tolerant. Besides, having missing packages here should never happen. + package_ids += [x for x in confl_package_ids if x not in \ + package_ids] + + if not package_ids: + return 0 + + factory = self._entropy.PackageActionFactory() + + for package_id in package_ids: + + pkg = factory.get( + factory.REMOVE_ACTION, + (package_id, inst_repo.name), + opts = self._meta['remove_metaopts']) + pkg.set_xterm_header(self._xterm_header) + + exit_st = pkg.start() + pkg.finalize() + if exit_st != 0: + return exit_st + + return 0 + + def _unpack_package(self, download, package_path, image_dir, pkg_dbpath): + """ + Effectively unpack the package tarballs. + """ + txt = "%s: %s" % ( + blue(_("Unpacking")), + red(os.path.basename(download)), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Unpacking package: %s" % (download,) + ) + + # removed in the meantime? fail. + # this is just a safety measure, but won't do anything + # against races. + if not os.path.isfile(package_path): + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Error, package was removed: %s" % (package_path,) + ) + return 1 + + # make sure image_dir always exists + # pkgs not providing any file would cause image_dir + # to not be created by uncompress_tarball + try: + os.makedirs(image_dir, 0o755) + except OSError as err: + if err.errno != errno.EEXIST: + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "Unable to mkdir: %s, error: %s" % ( + image_dir, repr(err),) + ) + self._entropy.output( + "%s: %s" % (brown(_("Unpack error")), err.errno,), + importance = 1, + level = "error", + header = red(" ## ") + ) + return 1 + + # pkg_dbpath is only non-None for the base package file + # extra package files don't carry any other edb information + if pkg_dbpath is not None: + # extract entropy database from package file + # in order to avoid having to read content data + # from the repository database, which, in future + # is allowed to not provide such info. + pkg_dbdir = os.path.dirname(pkg_dbpath) + if not os.path.isdir(pkg_dbdir): + os.makedirs(pkg_dbdir, 0o755) + # extract edb + dump_exit_st = entropy.tools.dump_entropy_metadata( + package_path, pkg_dbpath) + if not dump_exit_st: + # error during entropy db extraction from package file + # might be because edb entry point is not found or + # because there is not enough space for it + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "Unable to dump edb for: " + pkg_dbpath + ) + self._entropy.output( + brown(_("Unable to find Entropy metadata in package")), + importance = 1, + level = "error", + header = red(" ## ") + ) + return 1 + + try: + exit_st = entropy.tools.uncompress_tarball( + package_path, + extract_path = image_dir, + catch_empty = True + ) + except EOFError as err: + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "EOFError on " + package_path + " " + \ + repr(err) + ) + entropy.tools.print_traceback() + # try again until unpack_tries goes to 0 + exit_st = 1 + except Exception as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Ouch! error while unpacking " + \ + package_path + " " + repr(err) + ) + entropy.tools.print_traceback() + # try again until unpack_tries goes to 0 + exit_st = 1 + + if exit_st != 0: + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "Unable to unpack: %s" % (package_path,) + ) + self._entropy.output( + brown(_("Unable to unpack package")), + importance = 1, + level = "error", + header = red(" ## ") + ) + + return exit_st + + def _fill_image_dir(self, merge_from, image_dir): + """ + Fill the image directory with content from a filesystme path. + """ + repo = self._entropy.open_repository(self._repository_id) + # this is triggered by merge_from pkgmeta metadata + # even if repositories are allowed to not have content + # metadata, in this particular case, it is mandatory + contents = repo.retrieveContentIter( + self._package_id, + order_by = "file") + + for path, ftype in contents: + # convert back to filesystem str + encoded_path = path + path = os.path.join(merge_from, encoded_path[1:]) + topath = os.path.join(image_dir, encoded_path[1:]) + path = const_convert_to_rawstring(path) + topath = const_convert_to_rawstring(topath) + + try: + exist = os.lstat(path) + except OSError: + continue # skip file + + if "dir" == ftype and \ + not stat.S_ISDIR(exist.st_mode) and \ + os.path.isdir(path): + # workaround for directory symlink issues + path = os.path.realpath(path) + + copystat = False + # if our directory is a symlink instead, then copy the symlink + if os.path.islink(path): + tolink = os.readlink(path) + if os.path.islink(topath): + os.remove(topath) + os.symlink(tolink, topath) + elif os.path.isdir(path): + if not os.path.isdir(topath): + os.makedirs(topath) + copystat = True + elif os.path.isfile(path): + if os.path.isfile(topath): + os.remove(topath) # should never happen + shutil.copy2(path, topath) + copystat = True + + if copystat: + user = os.stat(path)[stat.ST_UID] + group = os.stat(path)[stat.ST_GID] + os.chown(topath, user, group) + shutil.copystat(path, topath) + + def _merge(self): + """ + Execute the merge (from) phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Merging"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + txt = "%s: %s" % ( + blue(_("Merging package")), + red(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Merging package: %s" % (self._meta['atom'],) + ) + + self._fill_image_dir(self._meta['merge_from'], + self._meta['imagedir']) + spm_class = self._entropy.Spm_class() + return spm_class.entropy_install_unpack_hook(self._entropy, + self._meta) + + def _unpack(self): + """ + Execute the unpack phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Unpacking"), + self._meta['download'], + ) + self._entropy.set_title(xterm_title) + + unpack_dir = self._meta['unpackdir'] + + if not const_is_python3(): + # unpackdir comes from download metadatum, which is utf-8 + # (conf_encoding) + unpack_dir = const_convert_to_rawstring(unpack_dir, + from_enctype = etpConst['conf_encoding']) + + if os.path.isdir(unpack_dir): + # this, if Python 2.x, must be fed with rawstrings + shutil.rmtree(unpack_dir) + elif os.path.isfile(unpack_dir): + os.remove(unpack_dir) + os.makedirs(unpack_dir) + + exit_st = self._unpack_package( + self._meta['download'], self._meta['pkgpath'], + self._meta['imagedir'], self._meta['pkgdbpath']) + + if exit_st == 0: + for extra_download in self._meta['extra_download']: + download = extra_download['download'] + pkgpath = self.get_standard_fetch_disk_path(download) + exit_st = self._unpack_package(download, pkgpath, + self._meta['imagedir'], None) + if exit_st != 0: + break + + if exit_st != 0: + if exit_st == 512: + errormsg = "%s. %s. %s: 512" % ( + red(_("You are running out of disk space")), + red(_("I bet, you're probably Michele")), + blue(_("Error")), + ) + else: + msg = _("An error occured while trying to unpack the package") + errormsg = "%s. %s. %s: %s" % ( + red(msg), + red(_("Check if your system is healthy")), + blue(_("Error")), + exit_st, + ) + self._entropy.output( + errormsg, + importance = 1, + level = "error", + header = red(" ## ") + ) + return exit_st + + spm_class = self._entropy.Spm_class() + # call Spm unpack hook + return spm_class.entropy_install_unpack_hook(self._entropy, + self._meta) + + def _setup(self): + """ + Execute the setup phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Setup"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + exit_st = 0 + data = self._meta['triggers'].get('install') + action_data = self._meta['triggers'].get('install') + + if data: + trigger = self._entropy.Triggers( + self.NAME, "setup", + data, action_data) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + if exit_st != 0: + return exit_st + + # NOTE: fixup permissions in the image directory + # the setup phase could have created additional users and groups + package_path = self._meta['pkgpath'] + prefix_dir = self._meta['imagedir'] + try: + entropy.tools.apply_tarball_ownership(package_path, prefix_dir) + except IOError as err: + msg = "%s: %s" % ( + brown(_("Error during package files permissions setup")), + err,) + self._entropy.output( + msg, + importance = 1, + level = "error", + header = darkred(" !!! ") + ) + exit_st = 1 + + return exit_st + + def _pre_install(self): + """ + Execute the pre-install phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Pre-install"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + data = self._meta['triggers'].get('install') + action_data = self._meta['triggers'].get('install') + exit_st = 0 + + if data: + trigger = self._entropy.Triggers( + self.NAME, "preinstall", + data, action_data) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _pre_remove(self): + """ + Execute the pre-remove phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Pre-remove"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + data = self._meta['triggers'].get('remove') + action_data = self._meta['triggers'].get('install') + exit_st = 0 + + if data: + trigger = self._entropy.Triggers( + self.NAME, "preremove", data, + action_data) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _install_clean(self): + """ + Cleanup package files not used anymore by newly installed version. + This is part of the atomic install, which overwrites the live fs with + new files and removes old afterwards. + """ + self._entropy.output( + blue(_("Cleaning previously installed application data.")), + importance = 1, + level = "info", + header = red(" ## ") + ) + + self._remove_content_from_system( + self._entropy.installed_repository(), + automerge_metadata = self._meta['already_protected_config_files']) + + return 0 + + def _post_remove(self): + """ + Execute the post-remove phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Post-remove"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + data = self._meta['triggers'].get('remove') + action_data = self._meta['triggers'].get('install') + exit_st = 0 + + if data: + trigger = self._entropy.Triggers( + self.NAME, "postremove", data, + action_data) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _post_remove_install(self): + """ + Execute the post-remove SPM package metadata phase. + """ + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Remove old package (spm data): %s" % (self._meta['removeatom'],) + ) + return self._spm_remove_package(self._meta['removeatom']) + + def _install_spm(self): + """ + Execute the installation of SPM package metadata. + """ + spm = self._entropy.Spm() + + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Installing new SPM entry: %s" % (self._meta['atom'],) + ) + + # this comes from _add_installed_package() + installed_package_id = self._meta['installed_package_id'] + + spm_uid = spm.add_installed_package(self._meta) + inst_repo = self._entropy.installed_repository() + if spm_uid != -1: + inst_repo.insertSpmUid(installed_package_id, spm_uid) + + return 0 + + def _post_install(self): + """ + Execute the post-install phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Post-install"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + data = self._meta['triggers'].get('install') + action_data = self._meta['triggers'].get('install') + exit_st = 0 + + if data: + trigger = self._entropy.Triggers( + self.NAME, "postinstall", + data, action_data) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _cleanup(self): + """ + Execute the cleanup phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Cleaning"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + txt = "%s: %s" % ( + blue(_("Cleaning")), + red(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + # shutil.rmtree wants raw strings, otherwise it will explode + unpack_dir = const_convert_to_rawstring(self._meta['unpackdir']) + + # best-effort below. + try: + shutil.rmtree(unpack_dir, True) + except shutil.Error as err: + self._entropy.logger.log( + "[Package]", etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to cleanup directory %s," \ + " error: %s" % (unpack_dir, err,)) + try: + os.rmdir(unpack_dir) + except OSError: + pass + + return 0 + + def _filter_out_files_installed_on_diff_path(self, content_file, + installed_content): + """ + Use case: if a package provided files in /lib then, a new version + of that package moved the same files under /lib64, we need to check + if both directory paths solve to the same inode and if so, + add to our set that we're going to return. + """ + sys_root = self._get_system_root() + second_pass_removal = set() + + if not installed_content: + # nothing to filter, no-op + return + def _main_filter(_path): + item_dir = os.path.dirname("%s%s" % ( + sys_root, _path,)) + item = os.path.join( + os.path.realpath(item_dir), + os.path.basename(_path)) + if item in installed_content: + second_pass_removal.add(item) + return False + return True + + # first pass, remove direct matches, schedule a second pass + # list of files + Content.filter_content_file(content_file, _main_filter) + + if not second_pass_removal: + # done then + return + + # second pass, drop remaining files + # unfortunately, this is the only way to work it out + # with iterators + def _filter(_path): + return _path not in second_pass_removal + Content.filter_content_file(content_file, _filter) + + def _add_installed_package(self, items_installed, items_not_installed): + """ + For internal use only. + Copy package from repository to installed packages one. + """ + def _merge_removecontent(inst_repo, repo, _package_id): + # NOTE: this could be a source of memory consumption + # but generally, the difference between two contents + # is really small + content_diff = list(inst_repo.contentDiff( + self._meta['remove_package_id'], + repo, + _package_id, + extended=True)) + + if content_diff: + + # reverse-order compare + def _cmp_func(_path, _spath): + if _path > _spath: + return -1 + elif _path == _spath: + return 0 + return 1 + + # must be sorted, and in reverse order + # or the merge step won't work + content_diff.sort(reverse=True) + + Content.merge_content_file( + self._meta['removecontent_file'], + content_diff, _cmp_func) + + inst_repo = self._entropy.installed_repository() + smart_pkg = self._meta['smartpackage'] + repo = self._entropy.open_repository(self._repository_id) + + splitdebug, splitdebug_dirs = ( + self._meta['splitdebug'], + self._meta['splitdebug_dirs']) + + if smart_pkg or self._meta['merge_from']: + + data = repo.getPackageData(self._package_id, + content_insert_formatted = True, + get_changelog = False, get_content = False, + get_content_safety = False) + + content = repo.retrieveContentIter( + self._package_id) + content_file = self._generate_content_file( + content, package_id = self._package_id, + filter_splitdebug = True, + splitdebug = splitdebug, + splitdebug_dirs = splitdebug_dirs) + + content_safety = repo.retrieveContentSafetyIter( + self._package_id) + content_safety_file = self._generate_content_safety_file( + content_safety) + + if self._meta['remove_package_id'] != -1 and \ + self._meta['removecontent_file'] is not None: + _merge_removecontent( + inst_repo, repo, self._package_id) + + else: + + # normal repositories + data = repo.getPackageData(self._package_id, + get_content = False, get_changelog = False) + + # indexing_override = False : no need to index tables + # xcache = False : no need to use on-disk cache + # skipChecks = False : creating missing tables is unwanted, + # and also no foreign keys update + # readOnly = True: no need to open in write mode + pkg_repo = self._entropy.open_generic_repository( + self._meta['pkgdbpath'], skip_checks = True, + indexing_override = False, read_only = True, + xcache = False) + + # it is safe to consider that package dbs coming from repos + # contain only one entry + pkg_package_id = sorted(pkg_repo.listAllPackageIds(), + reverse = True)[0] + content = pkg_repo.retrieveContentIter( + pkg_package_id) + content_file = self._generate_content_file( + content, package_id = self._package_id, + filter_splitdebug = True, + splitdebug = splitdebug, + splitdebug_dirs = splitdebug_dirs) + + # setup content safety metadata, get from package + content_safety = pkg_repo.retrieveContentSafetyIter( + pkg_package_id) + content_safety_file = self._generate_content_safety_file( + content_safety) + + if self._meta['remove_package_id'] != -1 and \ + self._meta['removecontent_file'] is not None: + _merge_removecontent(inst_repo, pkg_repo, pkg_package_id) + + pkg_repo.close() + + # items_installed is useful to avoid the removal of installed + # files by __remove_package just because + # there's a difference in the directory path, perhaps, + # which is not handled correctly by + # EntropyRepository.contentDiff for obvious reasons + # (think about stuff in /usr/lib and /usr/lib64, + # where the latter is just a symlink to the former) + # -- + # fix removecontent, need to check if we just installed files + # that resolves at the same directory path (different symlink) + if self._meta['removecontent_file'] is not None: + self._filter_out_files_installed_on_diff_path( + self._meta['removecontent_file'], + items_installed) + + # filter out files not installed from content metadata + # these include splitdebug files, when splitdebug is + # disabled. + if items_not_installed: + def _filter(_path): + return _path not in items_not_installed + Content.filter_content_file( + content_file, _filter) + + # this is needed to make postinstall trigger work properly + self._meta['triggers']['install']['affected_directories'] = \ + self._meta['affected_directories'] + self._meta['triggers']['install']['affected_infofiles'] = \ + self._meta['affected_infofiles'] + + # always set data['injected'] to False + # installed packages database SHOULD never have more + # than one package for scope (key+slot) + data['injected'] = False + # spm counter will be set in self._install_package_into_spm_database() + data['counter'] = -1 + # branch must be always set properly, it could happen it's not + # when installing packages through their .tbz2s + data['branch'] = self._settings['repositories']['branch'] + # there is no need to store needed paths into db + if "needed_paths" in data: + del data['needed_paths'] + # there is no need to store changelog data into db + if "changelog" in data: + del data['changelog'] + # we don't want it to be added now, we want to add install source + # info too. + if "original_repository" in data: + del data['original_repository'] + # rewrite extra_download metadata with the currently provided, + # and accepted extra_download items (in case of splitdebug being + # disable, we're not going to add those entries, for example) + data['extra_download'] = self._meta['extra_download'] + + data['content'] = None + data['content_safety'] = None + try: + # now we are ready to craft a 'content' iter object + data['content'] = Content.FileContentReader( + content_file) + data['content_safety'] = Content.FileContentSafetyReader( + content_safety_file) + package_id = inst_repo.handlePackage( + data, revision = data['revision'], + formattedContent = True) + finally: + if data['content'] is not None: + try: + data['content'].close() + data['content'] = None + except (OSError, IOError): + data['content'] = None + if data['content_safety'] is not None: + try: + data['content_safety'].close() + data['content_safety'] = None + except (OSError, IOError): + data['content_safety'] = None + + # update datecreation + ctime = time.time() + inst_repo.setCreationDate(package_id, str(ctime)) + + # add idpk to the installedtable + inst_repo.dropInstalledPackageFromStore(package_id) + inst_repo.storeInstalledPackage(package_id, + self._repository_id, self._meta['install_source']) + + automerge_data = self._meta.get('configprotect_data') + if automerge_data: + inst_repo.insertAutomergefiles(package_id, automerge_data) + + inst_repo.commit() + + # replace current empty "content" metadata info + # content metadata is required by + # _spm_install_package() -> Spm.add_installed_package() + # in case of injected packages (SPM metadata might be + # incomplete). + self._meta['triggers']['install']['content'] = \ + Content.FileContentReader(content_file) + + return package_id + + def _install_package(self): + """ + Execute the package installation code. + """ + # clear on-disk cache + self._entropy.clear_cache() + + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Installing package: %s" % (self._meta['atom'],) + ) + + inst_repo = self._entropy.installed_repository() + + if self._meta['remove_package_id'] != -1: + am_files = inst_repo.retrieveAutomergefiles( + self._meta['remove_package_id'], + get_dict = True) + self._meta['already_protected_config_files'] = am_files + + # items_*installed will be filled by _move_image_to_system + # then passed to _add_installed_package() + items_installed = set() + items_not_installed = set() + exit_st = self._move_image_to_system( + items_installed, items_not_installed) + if exit_st != 0: + return exit_st + + txt = "%s: %s" % ( + blue(_("Updating installed packages repository")), + teal(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + self._meta['installed_package_id'] = self._add_installed_package( + items_installed, items_not_installed) + + return 0 + + def _install(self): + """ + Execute the install phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Installing"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + txt = "%s: %s" % ( + blue(_("Installing package")), + red(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + self._entropy.output( + "[%s]" % ( + purple(self._meta['description']), + ), + importance = 1, + level = "info", + header = red(" ## ") + ) + + if self._meta['splitdebug']: + if self._meta.get('splitdebug_pkgfile'): + txt = "[%s]" % ( + teal(_("unsupported splitdebug usage (package files)")),) + level = "warning" + else: + txt = "[%s]" % ( + teal(_("<3 debug files installation enabled <3")),) + level = "info" + self._entropy.output( + txt, + importance = 1, + level = level, + header = red(" ## ") + ) + + exit_st = self._install_package() + if exit_st != 0: + txt = "%s. %s. %s: %s" % ( + red(_("An error occured while trying to install the package")), + red(_("Check if your system is healthy")), + blue(_("Error")), + exit_st, + ) + self._entropy.output( + txt, + importance = 1, + level = "error", + header = red(" ## ") + ) + return exit_st + + def _handle_install_collision_protect(self, tofile, todbfile): + """ + Handle files collition protection for the install phase. + """ + inst_repo = self._entropy.installed_repository() + avail = inst_repo.isFileAvailable( + const_convert_to_unicode(todbfile), get_id = True) + + if (self._meta['remove_package_id'] not in avail) and avail: + mytxt = darkred(_("Collision found during install for")) + mytxt += " %s - %s" % ( + blue(tofile), + darkred(_("cannot overwrite")), + ) + self._entropy.output( + red("QA: ")+mytxt, + importance = 1, + level = "warning", + header = darkred(" ## ") + ) + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Collision found during install " \ + "for %s - cannot overwrite" % (tofile,) + ) + return False + + return True + + def _move_image_to_system(self, items_installed, items_not_installed): + """ + Internal method that moves the package image directory to the live + filesystem. + """ + metadata = self.metadata() + repo = self._entropy.open_repository(self._repository_id) + protect = self._get_config_protect(repo, self._package_id) + mask = self._get_config_protect(repo, self._package_id, + mask = True) + protectskip = self._get_config_protect_skip() + + # support for unit testing settings + sys_root = self._get_system_root() + sys_set_plg_id = \ + etpConst['system_settings_plugins_ids']['client_plugin'] + misc_data = self._settings[sys_set_plg_id]['misc'] + col_protect = misc_data['collisionprotect'] + splitdebug, splitdebug_dirs = metadata['splitdebug'], \ + metadata['splitdebug_dirs'] + info_dirs = self._get_info_directories() + + # setup image_dir properly + image_dir = metadata['imagedir'][:] + if not const_is_python3(): + # image_dir comes from unpackdir, which comes from download + # metadatum, which is utf-8 (conf_encoding) + image_dir = const_convert_to_rawstring(image_dir, + from_enctype = etpConst['conf_encoding']) + movefile = entropy.tools.movefile + + def workout_subdir(currentdir, subdir): + + imagepath_dir = os.path.join(currentdir, subdir) + rel_imagepath_dir = imagepath_dir[len(image_dir):] + rootdir = sys_root + rel_imagepath_dir + + # splitdebug (.debug files) support + # If splitdebug is not enabled, do not create splitdebug directories + # and move on instead (return) + if not splitdebug: + for split_dir in splitdebug_dirs: + if rootdir.startswith(split_dir): + # also drop item from content metadata. In this way + # SPM has in sync information on what the package + # content really is. + # --- + # we should really use unicode + # strings for items_not_installed + unicode_rootdir = const_convert_to_unicode(rootdir) + items_not_installed.add(unicode_rootdir) + return + + # handle broken symlinks + if os.path.islink(rootdir) and not os.path.exists(rootdir): + # broken symlink + os.remove(rootdir) + + # if our directory is a file on the live system + elif os.path.isfile(rootdir): # really weird...! + + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! %s is a file when it should be " \ + "a directory !! Removing in 20 seconds..." % (rootdir,) + ) + mytxt = darkred(_("%s is a file when should be a " \ + "directory !! Removing in 20 seconds...") % (rootdir,)) + + self._entropy.output( + red("QA: ")+mytxt, + importance = 1, + level = "warning", + header = red(" !!! ") + ) + os.remove(rootdir) + + # if our directory is a symlink instead, then copy the symlink + if os.path.islink(imagepath_dir): + + # if our live system features a directory instead of + # a symlink, we should consider removing the directory + if not os.path.islink(rootdir) and os.path.isdir(rootdir): + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! %s is a directory when it should be " \ + "a symlink !! Removing in 20 seconds..." % ( + rootdir,) + ) + mytxt = "%s: %s" % ( + _("directory expected, symlink found"), + rootdir, + ) + mytxt2 = _("Removing in 20 seconds !!") + for txt in (mytxt, mytxt2,): + self._entropy.output( + darkred("QA: ") + darkred(txt), + importance = 1, + level = "warning", + header = red(" !!! ") + ) + + # fucking kill it in any case! + # rootdir must die! die die die die! + # /me brings chainsaw + try: + shutil.rmtree(rootdir, True) + except (shutil.Error, OSError,) as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to rm %s " \ + "directory ! [workout_subdir/1]: %s" % ( + rootdir, err, + ) + ) + + tolink = os.readlink(imagepath_dir) + live_tolink = None + if os.path.islink(rootdir): + live_tolink = os.readlink(rootdir) + + if tolink != live_tolink: + _symfail = False + if os.path.lexists(rootdir): + # at this point, it must be a file + try: + os.remove(rootdir) + except OSError as err: + _symfail = True + # must be atomic, too bad if it fails + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to remove %s " \ + "file ! [workout_file/0]: %s" % ( + rootdir, err, + ) + ) + msg = _("Cannot remove symlink") + mytxt = "%s: %s => %s" % ( + purple(msg), + blue(rootdir), + repr(err), + ) + self._entropy.output( + mytxt, + importance = 1, + level = "warning", + header = brown(" ## ") + ) + if not _symfail: + os.symlink(tolink, rootdir) + + elif not os.path.isdir(rootdir): + # directory not found, we need to create it + + try: + # really force a simple mkdir first of all + os.mkdir(rootdir) + except OSError: + os.makedirs(rootdir) + + + if not os.path.islink(rootdir): + + # symlink doesn't need permissions, also + # until os.walk ends they might be broken + user = os.stat(imagepath_dir)[stat.ST_UID] + group = os.stat(imagepath_dir)[stat.ST_GID] + try: + os.chown(rootdir, user, group) + shutil.copystat(imagepath_dir, rootdir) + except (OSError, IOError) as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Error during workdir setup " \ + "%s, %s, errno: %s" % ( + rootdir, + err, + err.errno, + ) + ) + # skip some errors because we may have + # unwritable directories + if err.errno not in ( + errno.EPERM, errno.ENOENT, + errno.ENOTDIR): + mytxt = "%s: %s, %s, %s" % ( + brown("Error during workdir setup"), + purple(rootdir), err, + err.errno + ) + self._entropy.output( + mytxt, + importance = 1, + level = "error", + header = darkred(" !!! ") + ) + return 4 + + item_dir, item_base = os.path.split(rootdir) + item_dir = os.path.realpath(item_dir) + item_inst = os.path.join(item_dir, item_base) + item_inst = const_convert_to_unicode(item_inst) + items_installed.add(item_inst) + + + def workout_file(currentdir, item): + + fromfile = os.path.join(currentdir, item) + rel_fromfile = fromfile[len(image_dir):] + rel_fromfile_dir = os.path.dirname(rel_fromfile) + tofile = sys_root + rel_fromfile + + rel_fromfile_dir_utf = const_convert_to_unicode( + rel_fromfile_dir) + metadata['affected_directories'].add( + rel_fromfile_dir_utf) + + # account for info files, if any + if rel_fromfile_dir_utf in info_dirs: + rel_fromfile_utf = const_convert_to_unicode( + rel_fromfile) + for _ext in self._INFO_EXTS: + if rel_fromfile_utf.endswith(_ext): + metadata['affected_infofiles'].add( + rel_fromfile_utf) + break + + # splitdebug (.debug files) support + # If splitdebug is not enabled, do not create + # splitdebug directories and move on instead (return) + if not splitdebug: + for split_dir in splitdebug_dirs: + if tofile.startswith(split_dir): + # also drop item from content metadata. In this way + # SPM has in sync information on what the package + # content really is. + # --- + # we should really use unicode + # strings for items_not_installed + unicode_tofile = const_convert_to_unicode(tofile) + items_not_installed.add(unicode_tofile) + return 0 + + if col_protect > 1: + todbfile = fromfile[len(image_dir):] + myrc = self._handle_install_collision_protect(tofile, + todbfile) + if not myrc: + return 0 + + prot_old_tofile = tofile[len(sys_root):] + # configprotect_data is passed to insertAutomergefiles() + # which always expects unicode data. + # revert back to unicode (we previously called encode on + # image_dir (which is passed to os.walk, which generates + # raw strings) + prot_old_tofile = const_convert_to_unicode(prot_old_tofile) + + pre_tofile = tofile[:] + (in_mask, protected, + tofile, do_return) = self._handle_config_protect( + protect, mask, protectskip, fromfile, tofile) + + # collect new config automerge data + if in_mask and os.path.exists(fromfile): + try: + prot_md5 = const_convert_to_unicode( + entropy.tools.md5sum(fromfile)) + metadata['configprotect_data'].append( + (prot_old_tofile, prot_md5,)) + except (IOError,) as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to get md5 of %s " \ + "file ! [workout_file/1]: %s" % ( + fromfile, err, + ) + ) + + # check if it's really necessary to protect file + if protected: + + # second task + # prot_old_tofile is always unicode, it must be, see above + oldprot_md5 = metadata['already_protected_config_files'].get( + prot_old_tofile) + + if oldprot_md5: + + try: + in_system_md5 = entropy.tools.md5sum(pre_tofile) + except (OSError, IOError) as err: + if err.errno != errno.ENOENT: + raise + in_system_md5 = "?" + + if oldprot_md5 == in_system_md5: + # we can merge it, files, even if + # contains changes have not been modified + # by the user + msg = _("Automerging config file, never modified") + mytxt = "%s: %s" % ( + darkgreen(msg), + blue(pre_tofile), + ) + self._entropy.output( + mytxt, + importance = 1, + level = "info", + header = red(" ## ") + ) + protected = False + do_return = False + tofile = pre_tofile + + if do_return: + return 0 + + try: + from_r_path = os.path.realpath(fromfile) + except RuntimeError: + # circular symlink, fuck! + # really weird...! + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! %s is a circular symlink !!!" % (fromfile,) + ) + mytxt = "%s: %s" % ( + _("Circular symlink issue"), + const_convert_to_unicode(fromfile), + ) + self._entropy.output( + darkred("QA: ") + darkred(mytxt), + importance = 1, + level = "warning", + header = red(" !!! ") + ) + from_r_path = fromfile + + try: + to_r_path = os.path.realpath(tofile) + except RuntimeError: + # circular symlink, fuck! + # really weird...! + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! %s is a circular symlink !!!" % (tofile,) + ) + mytxt = "%s: %s" % ( + _("Circular symlink issue"), + const_convert_to_unicode(tofile), + ) + self._entropy.output( + darkred("QA: ") + darkred(mytxt), + importance = 1, + level = "warning", + header = red(" !!! ") + ) + to_r_path = tofile + + if from_r_path == to_r_path and os.path.islink(tofile): + # there is a serious issue here, better removing tofile, + # happened to someone. + + try: + # try to cope... + os.remove(tofile) + except (OSError, IOError,) as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to cope to oddity of %s " \ + "file ! [workout_file/2]: %s" % ( + tofile, err, + ) + ) + + # if our file is a dir on the live system + if os.path.isdir(tofile) and not os.path.islink(tofile): + + # really weird...! + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! %s is a directory when it should " \ + "be a file !! Removing in 20 seconds..." % (tofile,) + ) + + mytxt = "%s: %s" % ( + _("file expected, directory found"), + const_convert_to_unicode(tofile), + ) + mytxt2 = _("Removing in 20 seconds !!") + for txt in (mytxt, mytxt2,): + self._entropy.output( + darkred("QA: ") + darkred(txt), + importance = 1, + level = "warning", + header = red(" !!! ") + ) + + try: + shutil.rmtree(tofile, True) + except (shutil.Error, IOError,) as err: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Failed to cope to oddity of %s " \ + "file ! [workout_file/3]: %s" % ( + tofile, err, + ) + ) + + # moving file using the raw format + try: + done = movefile(fromfile, tofile, src_basedir = image_dir) + except (IOError,) as err: + # try to move forward, sometimes packages might be + # fucked up and contain broken things + if err.errno not in (errno.ENOENT, errno.EACCES,): + raise + + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Error during file move" \ + " to system: %s => %s | IGNORED: %s" % ( + const_convert_to_unicode(fromfile), + const_convert_to_unicode(tofile), + err, + ) + ) + done = True + + if not done: + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "WARNING!!! Error during file move" \ + " to system: %s => %s" % (fromfile, tofile,) + ) + mytxt = "%s: %s => %s, %s" % ( + _("File move error"), + const_convert_to_unicode(fromfile), + const_convert_to_unicode(tofile), + _("please report"), + ) + self._entropy.output( + red("QA: ")+darkred(mytxt), + importance = 1, + level = "warning", + header = red(" !!! ") + ) + return 4 + + item_dir = os.path.realpath(os.path.dirname(tofile)) + item_inst = os.path.join(item_dir, os.path.basename(tofile)) + item_inst = const_convert_to_unicode(item_inst) + items_installed.add(item_inst) + + if protected and \ + os.getenv("ENTROPY_CLIENT_ENABLE_OLD_FILEUPDATES"): + # add to disk cache + file_updates = self._entropy.PackageFileUpdates() + file_updates.add(tofile, quiet = True) + + return 0 + + # merge data into system + for currentdir, subdirs, files in os.walk(image_dir): + + # create subdirs + for subdir in subdirs: + workout_subdir(currentdir, subdir) + + for item in files: + move_st = workout_file(currentdir, item) + if move_st != 0: + return move_st + + return 0 + + diff --git a/lib/entropy/client/interfaces/package/actions/multifetch.py b/lib/entropy/client/interfaces/package/actions/multifetch.py new file mode 100644 index 000000000..671797232 --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/multifetch.py @@ -0,0 +1,757 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import errno +import os + +from entropy.const import etpConst, const_setup_perms +from entropy.client.mirrors import StatusInterface +from entropy.fetchers import UrlFetcher +from entropy.output import blue, darkblue, bold, red, darkred, brown, darkgreen +from entropy.i18n import _, ngettext + +import entropy.dep +import entropy.tools + + +from .fetch import _PackageFetchAction + + +class _PackageMultiFetchAction(_PackageFetchAction): + """ + PackageAction used for package source code download. + + As opposed to the other PackageAction classes, this class + expects a list of package matches instead of just one. + """ + + NAME = "multi_fetch" + + def __init__(self, entropy_client, package_matches, opts = None): + """ + Object constructor. + """ + super(_PackageMultiFetchAction, self).__init__( + entropy_client, (None, None), opts = opts) + + self._package_matches = package_matches + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageMultiFetchAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + return self._meta + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + metadata['fetch_abort_function'] = self._opts.get( + 'fetch_abort_function') + + misc_settings = self._entropy.ClientSettings()['misc'] + metadata['edelta_support'] = misc_settings['edelta_support'] + + metadata['matches'] = self._package_matches + metadata['atoms'] = [] + metadata['repository_atoms'] = {} + + temp_fetch_list = [] + temp_checksum_list = [] + temp_already_downloaded_count = 0 + + def _setup_download(download, size, package_id, repository_id, digest, + signatures): + + obj = (package_id, repository_id, download, digest, signatures) + temp_checksum_list.append(obj) + + down_path = self.get_standard_fetch_disk_path(download) + try: + down_st = os.lstat(down_path) + st_size = down_st.st_size + except OSError as err: + if err.errno != errno.ENOENT: + raise + st_size = None + + if st_size is not None: + if st_size == size: + return 1 + else: + obj = (package_id, repository_id, download, digest, signatures) + temp_fetch_list.append(obj) + + return 0 + + for package_id, repository_id in self._package_matches: + + if repository_id.endswith(etpConst['packagesext']): + continue + + repo = self._entropy.open_repository(repository_id) + atom = repo.retrieveAtom(package_id) + + metadata['atoms'].append(atom) + if repository_id not in metadata['repository_atoms']: + metadata['repository_atoms'][repository_id] = set() + metadata['repository_atoms'][repository_id].add(atom) + + download = repo.retrieveDownloadURL(package_id) + digest = repo.retrieveDigest(package_id) + sha1, sha256, sha512, gpg = repo.retrieveSignatures(package_id) + size = repo.retrieveSize(package_id) + signatures = { + 'sha1': sha1, + 'sha256': sha256, + 'sha512': sha512, + 'gpg': gpg, + } + temp_already_downloaded_count += _setup_download( + download, size, package_id, repository_id, digest, signatures) + + extra_downloads = repo.retrieveExtraDownload(package_id) + + splitdebug = metadata['splitdebug'] + # if splitdebug is enabled, check if it's also enabled + # via package.splitdebug + if splitdebug: + splitdebug = self._package_splitdebug_enabled( + (package_id, repository_id)) + + if not splitdebug: + extra_downloads = [ + x for x in extra_downloads if x['type'] != "debug"] + + for extra_download in extra_downloads: + download = extra_download['download'] + size = extra_download['size'] + digest = extra_download['md5'] + signatures = { + 'sha1': extra_download['sha1'], + 'sha256': extra_download['sha256'], + 'sha512': extra_download['sha512'], + 'gpg': extra_download['gpg'], + } + temp_already_downloaded_count += _setup_download( + download, size, package_id, repository_id, + digest, signatures) + + metadata['multi_fetch_list'] = temp_fetch_list + metadata['multi_checksum_list'] = temp_checksum_list + + metadata['phases'] = [] + if metadata['multi_fetch_list']: + metadata['phases'].append(self._fetch) + + if metadata['multi_checksum_list']: + metadata['phases'].append(self._checksum) + + if temp_already_downloaded_count == len(temp_checksum_list): + metadata['phases'].reverse() + + self._meta = metadata + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + exit_st = 0 + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st + + def _try_edelta_multifetch(self, url_data, resume): + """ + Attempt to download and use the edelta file. + """ + # no edelta support enabled + if not self._meta.get('edelta_support'): + return [], 0.0, False + + if not entropy.tools.is_entropy_delta_available(): + return [], 0.0, False + + url_path_list = [] + url_data_map = {} + url_data_map_idx = 0 + inst_repo = self._entropy.installed_repository() + for pkg_id, repository_id, url, dest_path, cksum in url_data: + + repo = self._entropy.open_repository(repository_id) + if cksum is None: + # cannot setup edelta without checksum, get from repository + cksum = repo.retrieveDigest(pkg_id) + if cksum is None: + # still nothing + continue + + key_slot = repo.retrieveKeySlotAggregated(pkg_id) + if key_slot is None: + # wtf corrupted entry, skip + continue + + installed_package_id, _inst_rc = inst_repo.atomMatch(key_slot) + if installed_package_id == -1: + # package is not installed + continue + + edelta_approve = self._approve_edelta(url, installed_package_id, + cksum) + if edelta_approve is None: + # no edelta support + continue + edelta_url, installed_fetch_path = edelta_approve + + edelta_save_path = dest_path + etpConst['packagesdeltaext'] + key = (edelta_url, edelta_save_path) + url_path_list.append(key) + url_data_map_idx += 1 + url_data_map[url_data_map_idx] = ( + pkg_id, repository_id, url, + dest_path, cksum, edelta_url, + edelta_save_path, installed_fetch_path) + + if not url_path_list: + # no martini, no party! + return [], 0.0, False + + fetch_abort_function = self._meta.get('fetch_abort_function') + fetch_intf = self._entropy._multiple_url_fetcher(url_path_list, + resume = resume, abort_check_func = fetch_abort_function, + url_fetcher_class = self._entropy._url_fetcher) + try: + # make sure that we don't need to abort already + # doing the check here avoids timeouts + if fetch_abort_function != None: + fetch_abort_function() + + data = fetch_intf.download() + except KeyboardInterrupt: + return [], 0.0, True + data_transfer = fetch_intf.get_transfer_rate() + + fetch_errors = ( + UrlFetcher.TIMEOUT_FETCH_ERROR, + UrlFetcher.GENERIC_FETCH_ERROR, + UrlFetcher.GENERIC_FETCH_WARN, + ) + + valid_idxs = [] + for url_data_map_idx, cksum in tuple(data.items()): + + if cksum in fetch_errors: + # download failed + continue + + (pkg_id, repository_id, url, dest_path, + orig_cksum, edelta_url, edelta_save_path, + installed_fetch_path) = url_data_map[url_data_map_idx] + + # now check + tmp_dest_path = dest_path + ".edelta_pkg_tmp" + # yay, we can apply the delta and cook the new package file! + try: + entropy.tools.apply_entropy_delta(installed_fetch_path, + edelta_save_path, tmp_dest_path) + except IOError: + # give up with this edelta + try: + os.remove(tmp_dest_path) + except (OSError, IOError): + pass + continue + + os.rename(tmp_dest_path, dest_path) + valid_idxs.append(url_data_map_idx) + + fetched_url_data = [] + for url_data_map_idx in valid_idxs: + (pkg_id, repository_id, url, dest_path, + orig_cksum, edelta_url, edelta_save_path, + installed_fetch_path) = url_data_map[url_data_map_idx] + + try: + valid = entropy.tools.compare_md5(dest_path, orig_cksum) + except (IOError, OSError): + valid = False + + if valid: + url_data_item = (pkg_id, repository_id, url, + dest_path, orig_cksum) + fetched_url_data.append(url_data_item) + + return fetched_url_data, data_transfer, False + + def _fetch_files(self, url_data_list, resume = True): + """ + Effectively fetch the package files. + """ + + def _generate_checksum_map(url_data): + ck_map = {} + ck_map_id = 0 + for _pkg_id, _repository_id, _url, _dest_path, cksum in url_data: + ck_map_id += 1 + if cksum is not None: + ck_map[ck_map_id] = cksum + return ck_map + + fetch_abort_function = self._meta.get('fetch_abort_function') + # avoid tainting data pointed by url_data_list + url_data = url_data_list[:] + diff_map = {} + + # setup directories + for pkg_id, repository_id, url, dest_path, _cksum in url_data: + dest_dir = os.path.dirname(dest_path) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir, 0o775) + const_setup_perms(dest_dir, etpConst['entropygid']) + + checksum_map = _generate_checksum_map(url_data) + fetched_url_data, data_transfer, abort = self._try_edelta_multifetch( + url_data, resume) + if abort: + return -100, {}, 0 + + for url_data_item in fetched_url_data: + url_data.remove(url_data_item) + + # some packages haven't been downloaded using edelta + if url_data: + + url_path_list = [] + for pkg_id, repository_id, url, dest_path, _cksum in url_data: + url_path_list.append((url, dest_path,)) + self._setup_differential_download( + self._entropy._multiple_url_fetcher, url, resume, dest_path, + repository_id, pkg_id) + + # load class + fetch_intf = self._entropy._multiple_url_fetcher( + url_path_list, resume = resume, + abort_check_func = fetch_abort_function, + url_fetcher_class = self._entropy._url_fetcher, + checksum = True) + try: + # make sure that we don't need to abort already + # doing the check here avoids timeouts + if fetch_abort_function != None: + fetch_abort_function() + + data = fetch_intf.download() + except KeyboardInterrupt: + return -100, {}, 0 + + data_transfer = fetch_intf.get_transfer_rate() + checksum_map = _generate_checksum_map(url_data) + for ck_id in checksum_map: + orig_checksum = checksum_map.get(ck_id) + if orig_checksum != data.get(ck_id): + diff_map[url_path_list[ck_id-1][0]] = orig_checksum + + if diff_map: + defval = -1 + for key, val in tuple(diff_map.items()): + if val == UrlFetcher.GENERIC_FETCH_WARN: + diff_map[key] = -2 + elif val == UrlFetcher.TIMEOUT_FETCH_ERROR: + diff_map[key] = -4 + elif val == UrlFetcher.GENERIC_FETCH_ERROR: + diff_map[key] = -3 + elif val == -100: + defval = -100 + + return defval, diff_map, data_transfer + + return 0, diff_map, data_transfer + + def _download_packages(self, download_list): + """ + Internal function. Download packages. + """ + avail_data = self._settings['repositories']['available'] + excluded_data = self._settings['repositories']['excluded'] + + repo_uris = {} + for pkg_id, repository_id, fname, cksum, _signatures in download_list: + repo = self._entropy.open_repository(repository_id) + + # grab original repo, if any and use it if available + # this is done in order to support "equo repo merge" feature + # allowing client-side repository package metadata moves. + original_repo = repo.getInstalledPackageRepository(pkg_id) + + if (original_repo != repository_id) and ( + original_repo not in avail_data) and ( + original_repo is not None): + # build up a new uris list, at least try, hoping that + # repository is just shadowing original_repo + # for example: original_repo got copied to repository, without + # copying packages, which would be useless. like it happens + # with sabayon-weekly + uris = self._build_uris_list(original_repo, repository_id) + + else: + if original_repo in avail_data: + uris = avail_data[original_repo]['packages'][::-1] + uris += avail_data[repository_id]['packages'][::-1] + elif original_repo in excluded_data: + uris = excluded_data[original_repo]['packages'][::-1] + uris += avail_data[repository_id]['packages'][::-1] + else: + uris = avail_data[repository_id]['packages'][::-1] + + obj = repo_uris.setdefault(repository_id, []) + # append at the beginning + new_ones = [x for x in uris if x not in obj][::-1] + for new_obj in new_ones: + obj.insert(0, new_obj) + + remaining = repo_uris.copy() + mirror_status = StatusInterface() + + def get_best_mirror(repository_id): + try: + return remaining[repository_id][0] + except IndexError: + return None + + def update_download_list(down_list, failed_down): + newlist = [] + for pkg_id, repository_id, fname, cksum, signatures in down_list: + p_uri = get_best_mirror(repository_id) + p_uri = os.path.join(p_uri, fname) + if p_uri not in failed_down: + continue + newlist.append( + (pkg_id, repository_id, fname, cksum, signatures) + ) + return newlist + + # return True: for failing, return False: for fine + def mirror_fail_check(repository_id, best_mirror): + # check if uri is sane + if not mirror_status.get_failing_mirror_status(best_mirror) >= 30: + return False + + # set to 30 for convenience + mirror_status.set_failing_mirror_status(best_mirror, 30) + + mirrorcount = repo_uris[repository_id].index(best_mirror) + 1 + txt = "( mirror #%s ) %s %s - %s" % ( + mirrorcount, + blue(_("Mirror")), + red(self._get_url_name(best_mirror)), + _("maximum failure threshold reached"), + ) + self._entropy.output( + txt, + importance = 1, + level = "warning", + header = red(" ## ") + ) + + if mirror_status.get_failing_mirror_status(best_mirror) == 30: + mirror_status.add_failing_mirror(best_mirror, 45) + elif mirror_status.get_failing_mirror_status(best_mirror) > 31: + mirror_status.add_failing_mirror(best_mirror, -4) + else: + mirror_status.set_failing_mirror_status(best_mirror, 0) + + try: + remaining[repository_id].remove(best_mirror) + except ValueError: + # ignore + pass + return True + + def show_download_summary(down_list): + for _pkg_id, repository_id, fname, _cksum, _signatures in down_list: + best_mirror = get_best_mirror(repository_id) + mirrorcount = repo_uris[repository_id].index(best_mirror) + 1 + basef = os.path.basename(fname) + + txt = "( mirror #%s ) [%s] %s %s" % ( + mirrorcount, + brown(basef), + blue("@"), + red(self._get_url_name(best_mirror)), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + def show_successful_download(down_list, data_transfer): + for _pkg_id, repository_id, fname, _cksum, _signatures in down_list: + best_mirror = get_best_mirror(repository_id) + mirrorcount = repo_uris[repository_id].index(best_mirror) + 1 + basef = os.path.basename(fname) + + txt = "( mirror #%s ) [%s] %s %s %s" % ( + mirrorcount, + brown(basef), + darkred(_("success")), + blue("@"), + red(self._get_url_name(best_mirror)), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + txt = " %s: %s%s%s" % ( + blue(_("Aggregated transfer rate")), + bold(entropy.tools.bytes_into_human(data_transfer)), + darkred("/"), + darkblue(_("second")), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + def show_download_error(down_list, p_exit_st): + for _pkg_id, repository_id, _fname, _cksum, _signs in down_list: + best_mirror = get_best_mirror(repository_id) + mirrorcount = repo_uris[repository_id].index(best_mirror) + 1 + + txt = "( mirror #%s ) %s: %s" % ( + mirrorcount, + blue(_("Error downloading from")), + red(self._get_url_name(best_mirror)), + ) + if p_exit_st == -1: + txt += " - %s." % ( + _("data not available on this mirror"),) + elif p_exit_st == -2: + mirror_status.add_failing_mirror(best_mirror, 1) + txt += " - %s." % (_("wrong checksum"),) + + elif p_exit_st == -3: + txt += " - %s." % (_("not found"),) + + elif p_exit_st == -4: # timeout! + txt += " - %s." % (_("timeout error"),) + + elif p_exit_st == -100: + txt += " - %s." % (_("discarded download"),) + + else: + mirror_status.add_failing_mirror(best_mirror, 5) + txt += " - %s." % (_("unknown reason"),) + + self._entropy.output( + txt, + importance = 1, + level = "warning", + header = red(" ## ") + ) + + def remove_failing_mirrors(repos): + for repository_id in repos: + get_best_mirror(repository_id) + if remaining[repository_id]: + remaining[repository_id].pop(0) + + def check_remaining_mirror_failure(repos): + return [x for x in repos if not remaining.get(x)] + + d_list = download_list[:] + + while True: + do_resume = True + timeout_try_count = 50 + + while True: + fetch_files_list = [] + + for pkg_id, repository_id, fname, cksum, _signs in d_list: + best_mirror = get_best_mirror(repository_id) + # set working mirror, dont care if its None + mirror_status.set_working_mirror(best_mirror) + if best_mirror is not None: + mirror_fail_check(repository_id, best_mirror) + best_mirror = get_best_mirror(repository_id) + + if best_mirror is None: + # at least one package failed to download + # properly, give up with everything + return 3, d_list + + myuri = os.path.join(best_mirror, fname) + pkg_path = self.get_standard_fetch_disk_path(fname) + fetch_files_list.append( + (pkg_id, repository_id, myuri, pkg_path, cksum,) + ) + + show_download_summary(d_list) + (exit_st, failed_downloads, + data_transfer) = self._fetch_files( + fetch_files_list, resume = do_resume) + + if exit_st == 0: + show_successful_download( + d_list, data_transfer) + return 0, [] + + d_list = update_download_list( + d_list, failed_downloads) + + if exit_st not in (-3, -4, -100,) and failed_downloads and \ + do_resume: + # disable resume + do_resume = False + continue + + show_download_error(d_list, exit_st) + + if exit_st == -4: # timeout + timeout_try_count -= 1 + if timeout_try_count > 0: + continue + + elif exit_st == -100: # user discarded fetch + return 1, [] + + myrepos = set([x[1] for x in d_list]) + remove_failing_mirrors(myrepos) + + # make sure we don't have nasty issues + remaining_failure = check_remaining_mirror_failure( + myrepos) + + if remaining_failure: + return 3, d_list + + break + + return 0, [] + + def _fetch(self): + """ + Execute the fetch phase. + """ + m_fetch_len = len(self._meta['multi_fetch_list']) / 2 + xterm_title = "%s: %s %s" % ( + _("Multi Fetching"), + m_fetch_len, + ngettext("package", "packages", m_fetch_len), + ) + self._entropy.set_title(xterm_title) + + m_fetch_len = len(self._meta['multi_fetch_list']) + txt = "%s: %s %s" % ( + blue(_("Downloading")), + darkred("%s" % (m_fetch_len,)), + ngettext("archive", "archives", m_fetch_len), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + exit_st, err_list = self._download_packages( + self._meta['multi_fetch_list']) + if exit_st == 0: + return 0 + + txt = _("Some packages cannot be fetched") + txt2 = _("Try to update your repositories and retry") + for txt in (txt, txt2,): + self._entropy.output( + "%s." % ( + darkred(txt), + ), + importance = 0, + level = "info", + header = red(" ## ") + ) + + self._entropy.output( + "%s: %s" % ( + brown(_("Error")), + exit_st, + ), + importance = 0, + level = "info", + header = red(" ## ") + ) + + for _pkg_id, repo, fname, cksum, _signatures in err_list: + self._entropy.output( + "[%s|%s] %s" % ( + blue(repo), + darkgreen(cksum), + darkred(fname), + ), + importance = 1, + level = "error", + header = darkred(" # ") + ) + + return exit_st + + def _checksum(self): + """ + Execute the checksum verification phase. + """ + m_len = len(self._meta['multi_checksum_list']) + xterm_title = "%s: %s %s" % ( + _("Multi Verification"), + m_len, + ngettext("package", "packages", m_len), + ) + self._entropy.set_title(xterm_title) + + exit_st = 0 + ck_list = self._meta['multi_checksum_list'] + + for pkg_id, repository_id, download, digest, signatures in ck_list: + exit_st = self._match_checksum( + pkg_id, repository_id, digest, download, signatures) + if exit_st != 0: + break + + return exit_st diff --git a/lib/entropy/client/interfaces/package/actions/remove.py b/lib/entropy/client/interfaces/package/actions/remove.py new file mode 100644 index 000000000..a8d58eb7b --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/remove.py @@ -0,0 +1,286 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +from entropy.const import etpConst +from entropy.exceptions import SPMError +from entropy.i18n import _ +from entropy.output import blue, red + +import entropy.dep + +from ._manage import _PackageInstallRemoveAction + + +class _PackageRemoveAction(_PackageInstallRemoveAction): + """ + PackageAction used for package removal. + """ + + NAME = "remove" + + def __init__(self, entropy_client, package_match, opts = None): + """ + Object constructor. + """ + super(_PackageRemoveAction, self).__init__( + entropy_client, package_match, opts = opts) + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageRemoveAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + inst_repo = self._entropy.open_repository(self._repository_id) + metadata['configprotect_data'] = [] + metadata['triggers'] = {} + metadata['atom'] = inst_repo.retrieveAtom(self._package_id) + # removeatom metadata key used by Spm.remove_installed_package() + metadata['removeatom'] = metadata['atom'] + metadata['slot'] = inst_repo.retrieveSlot(self._package_id) + metadata['versiontag'] = inst_repo.retrieveTag(self._package_id) + metadata['removeconfig'] = self._opts.get('removeconfig', False) + + content = inst_repo.retrieveContentIter( + self._package_id, order_by="file", reverse=True) + metadata['removecontent_file'] = self._generate_content_file( + content) + + # collects directories whose content has been modified + # this information is then handed to the Trigger + metadata['affected_directories'] = set() + metadata['affected_infofiles'] = set() + + trigger = inst_repo.getTriggerData(self._package_id) + metadata['triggers']['remove'] = trigger + + trigger['affected_directories'] = metadata['affected_directories'] + trigger['affected_infofiles'] = metadata['affected_infofiles'] + trigger['spm_repository'] = inst_repo.retrieveSpmRepository( + self._package_id) + + trigger['accept_license'] = self._get_licenses( + inst_repo, self._package_id) + trigger.update(splitdebug_metadata) + + # setup config_protect and config_protect+mask metadata before it's + # too late. + protect = self._get_config_protect_metadata(inst_repo, self._package_id) + metadata.update(protect) + + metadata['phases'] = [ + self._pre_remove, + self._remove, + self._post_remove, + self._post_remove_remove, + ] + self._meta = metadata + + def _pre_remove(self): + """ + Run the pre-remove phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Pre-remove"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + exit_st = 0 + data = self._meta['triggers']['remove'] + if not data: + return exit_st + + trigger = self._entropy.Triggers( + self.NAME, "preremove", data, None) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _remove(self): + """ + Run the remove phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Removing"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + self._entropy.output( + "%s: %s" % ( + blue(_("Removing")), + red(self._meta['atom']), + ), + importance = 1, + level = "info", + header = red(" ## ") + ) + + self._entropy.clear_cache() + + self._entropy.logger.log("[Package]", + etpConst['logging']['normal_loglevel_id'], + "Removing package: %s" % (self._meta['atom'],)) + + txt = "%s: %s" % ( + blue(_("Removing from Entropy")), + red(self._meta['atom']), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + inst_repo = self._entropy.open_repository(self._repository_id) + automerge_metadata = inst_repo.retrieveAutomergefiles( + self._package_id, get_dict = True) + + inst_repo.removePackage(self._package_id) + # commit changes, to avoid users pressing CTRL+C and still having + # all the db entries in, so we need to commit at every iteration + inst_repo.commit() + + self._remove_content_from_system( + inst_repo, automerge_metadata = automerge_metadata) + + return 0 + + def _post_remove(self): + """ + Run the first post-remove phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Post-remove"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + exit_st = 0 + data = self._meta['triggers']['remove'] + if not data: + return exit_st + + trigger = self._entropy.Triggers( + self.NAME, "postremove", data, None) + ack = trigger.prepare() + if ack: + exit_st = trigger.run() + trigger.kill() + + return exit_st + + def _spm_update_package_uid(self, installed_package_id, spm_atom): + """ + Update Source Package Manager <-> Entropy package identifiers coupling. + Entropy can handle multiple packages in the same scope from a SPM POV + (see the "package tag" feature to provide linux kernel module packages + for different kernel versions). This method just reassigns a new SPM + unique package identifier to Entropy. + + @param installed_package_id: Entropy package identifier bound to + given spm_atom + @type installed_package_id: int + @param spm_atom: SPM package atom + @type spm_atom: string + @return: execution status + @rtype: int + """ + spm = self._entropy.Spm() + + try: + spm_uid = spm.assign_uid_to_installed_package(spm_atom) + except (SPMError, KeyError,): + # installed package not available, we must ignore it + self._entropy.logger.log( + "[Package]", + etpConst['logging']['normal_loglevel_id'], + "Spm uid not available for Spm package: %s (pkg not avail?)" % ( + spm_atom, + ) + ) + spm_uid = -1 + + if spm_uid != -1: + inst_repo = self._entropy.open_repository(self._repository_id) + inst_repo.insertSpmUid(installed_package_id, spm_uid) + + def _post_remove_remove(self): + """ + Post-remove phase of package remove action, this step removes SPM + package entries if there are no other Entropy-tagged packages installed. + """ + # remove pkg + # -- now it's possible to remove SPM package entry. + # if another package with the same atom is installed in + # Entropy db, do not call SPM at all because it would cause + # to get that package removed from there resulting in matching + # inconsistencies. + # -- of course, we need to drop versiontag before being able to look + # for other pkgs with same atom but different tag (which is an + # entropy-only metadatum) + + atom = self._meta['atom'] + inst_repo = self._entropy.open_repository(self._repository_id) + test_atom = entropy.dep.remove_tag(atom) + installed_package_ids = inst_repo.getPackageIds(test_atom) + + spm = self._entropy.Spm() + spm_atom = spm.convert_from_entropy_package_name(atom) + + if not installed_package_ids: + exit_st = self._spm_remove_package(spm_atom) + if exit_st != 0: + return exit_st + + for installed_package_id in installed_package_ids: + # we have one installed, we need to update SPM uid + self._spm_update_package_uid(installed_package_id, spm_atom) + + return 0 + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + exit_st = 0 + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st diff --git a/lib/entropy/client/interfaces/package/actions/source.py b/lib/entropy/client/interfaces/package/actions/source.py new file mode 100644 index 000000000..1da64ecae --- /dev/null +++ b/lib/entropy/client/interfaces/package/actions/source.py @@ -0,0 +1,301 @@ +# -*- coding: utf-8 -*- +""" + + @author: Fabio Erculiani + @contact: lxnay@sabayon.org + @copyright: Fabio Erculiani + @license: GPL-2 + + B{Entropy Package Manager Client Package Interface}. + +""" +import os +import shutil + +from entropy.const import etpConst, const_setup_perms +from entropy.i18n import _ +from entropy.output import blue, red, brown + +import entropy.dep +import entropy.tools + + +from .fetch import _PackageFetchAction + + +class _PackageSourceAction(_PackageFetchAction): + """ + PackageAction used for package source code download. + """ + + NAME = "source" + + def __init__(self, entropy_client, package_match, opts = None): + """ + Object constructor. + """ + super(_PackageSourceAction, self).__init__( + entropy_client, package_match, opts = opts) + self._meta = None + + def finalize(self): + """ + Finalize the object, release all its resources. + """ + super(_PackageSourceAction, self).finalize() + if self._meta is not None: + meta = self._meta + self._meta = None + meta.clear() + + def metadata(self): + """ + Return the package metadata dict object for manipulation. + """ + return self._meta + + def setup(self): + """ + Setup the PackageAction. + """ + if self._meta is not None: + # already configured + return + + metadata = {} + splitdebug_metadata = self._get_splitdebug_metadata() + metadata.update(splitdebug_metadata) + + metadata['fetch_abort_function'] = self._opts.get( + 'fetch_abort_function') + + # NOTE: if you want to implement download-to-dir feature in your + # client, you've found what you were looking for. + # fetch_path is the path where data should be downloaded + # it overrides default path + fetch_path = self._opts.get('fetch_path', None) + if fetch_path is not None: + if entropy.tools.is_valid_path(fetch_path): + metadata['fetch_path'] = fetch_path + + # if splitdebug is enabled, check if it's also enabled + # via package.splitdebug + splitdebug = metadata['splitdebug'] + if splitdebug: + splitdebug = self._package_splitdebug_enabled( + self._package_match) + + repo = self._entropy.open_repository(self._repository_id) + metadata['atom'] = repo.retrieveAtom(self._package_id) + metadata['slot'] = repo.retrieveSlot(self._package_id) + + inst_repo = self._entropy.installed_repository() + metadata['installed_package_id'], _inst_rc = inst_repo.atomMatch( + entropy.dep.dep_getkey(metadata['atom']), + matchSlot = metadata['slot']) + + metadata['edelta_support'] = False + metadata['extra_download'] = tuple() + metadata['download'] = repo.retrieveSources( + self._package_id, extended = True) + # fake path, don't use + metadata['pkgpath'] = etpConst['entropypackagesworkdir'] + + metadata['phases'] = [] + + if not metadata['download']: + metadata['phases'].append(self._fetch_not_available) + return + + metadata['phases'].append(self._fetch) + + # create sources destination directory + unpack_dir = os.path.join( + etpConst['entropyunpackdir'], + "sources", metadata['atom']) + metadata['unpackdir'] = unpack_dir + + self._meta = metadata + + def _run(self): + """ + Execute the action. Return an exit status. + """ + self.setup() + + unpack_dir = self._meta['unpackdir'] + + if not self._meta.get('fetch_path'): + try: + if os.path.lexists(unpack_dir): + if os.path.isfile(unpack_dir): + os.remove(unpack_dir) + elif os.path.isdir(unpack_dir): + shutil.rmtree(unpack_dir, True) + if not os.path.lexists(unpack_dir): + os.makedirs(unpack_dir, 0o755) + const_setup_perms(unpack_dir, etpConst['entropygid'], + recursion = False, uid = etpConst['uid']) + + except (OSError, IOError) as err: + self._entropy.output( + "%s: %s" % ( + blue(_("Fetch path setup error")), + err, + ), + importance = 1, + level = "info", + header = red(" ## ") + ) + + return 1 + + exit_st = 0 + for method in self._meta['phases']: + exit_st = method() + if exit_st != 0: + break + return exit_st + + def _fetch_not_available(self): + """ + Execute the fetch not available phase. + """ + self._entropy.output( + blue(_("Source code not available.")), + importance = 1, + level = "info", + header = red(" ## ") + ) + return 0 + + def _fetch_source(self, url, dest_file): + """ + Fetch the source code tarball(s). + """ + self._entropy.output( + "%s: %s" % ( + blue(_("Downloading")), + brown(url), + ), + importance = 1, + level = "info", + header = red(" ## ") + ) + + exit_st, data_transfer, _resumed = self._fetch_file( + url, dest_file, digest = None, resume = False) + + if exit_st == 0: + human_bytes = entropy.tools.bytes_into_human(data_transfer) + txt = "%s: %s %s %s/%s" % ( + blue(_("Successfully downloaded from")), + red(self._get_url_name(url)), + _("at"), + human_bytes, + _("second"), + ) + self._entropy.output( + txt, + importance = 1, + level = "info", + header = red(" ## ") + ) + + self._entropy.output( + "%s: %s" % ( + blue(_("Local path")), + brown(dest_file), + ), + importance = 1, + level = "info", + header = red(" # ") + ) + return exit_st + + error_message = "%s: %s" % ( + blue(_("Error downloading from")), + red(self._get_url_name(url)), + ) + if exit_st == -1: + error_message += " - %s." % ( + _("file not available on this mirror"), + ) + elif exit_st == -3: + error_message += " - not found." + elif exit_st == -100: + error_message += " - %s." % (_("discarded download"),) + else: + error_message += " - %s: %s" % ( + _("unknown reason"), exit_st, + ) + + self._entropy.output( + error_message, + importance = 1, + level = "warning", + header = red(" ## ") + ) + return exit_st + + def _fetch(self): + """ + Execute the source fetch phase. + """ + xterm_title = "%s %s: %s" % ( + self._xterm_header, + _("Fetching sources"), + self._meta['atom'], + ) + self._entropy.set_title(xterm_title) + + down_data = self._meta['download'] + down_keys = list(down_data.keys()) + d_cache = set() + exit_st = 0 + key_cache = [os.path.basename(x) for x in down_keys] + + for key in sorted(down_keys): + + key_name = os.path.basename(key) + if key_name in d_cache: + continue + # first fine wins + + keyboard_interrupt = False + for url in down_data[key]: + + file_name = os.path.basename(url) + if self._meta.get('fetch_path'): + dest_file = os.path.join( + self._meta['fetch_path'], + file_name) + else: + dest_file = os.path.join(self._meta['unpackdir'], + file_name) + + try: + exit_st = self._fetch_source(url, dest_file) + except KeyboardInterrupt: + keyboard_interrupt = True + break + + if exit_st == -100: + keyboard_interrupt = True + break + + if exit_st == 0: + d_cache.add(key_name) + break + + if keyboard_interrupt: + exit_st = 1 + break + + key_cache.remove(key_name) + if exit_st != 0 and key_name not in key_cache: + break + + exit_st = 0 + + return exit_st