[entropy.client.interfaces.package] split Package class, create new API

The Package has been split into multiple PackageAction based classes.
The new interface uses a factory (PackageActionFactory) to build
the PackageAction objects on behalf of the caller.

The new factory-based interface is clean, modular and faster than
the old Package spaghetti class.

Backward compatibility has been preserved, as long as caller doesn't
peek into internal structures and metadata. The old interface has
been deprecated and a new Entropy Client method called PackageActionFactory()
has been added.

Please migrate your code to the new API, the old API will be dropped in 1
solar year.
This commit is contained in:
Fabio Erculiani
2013-11-23 13:00:53 +01:00
parent 44444b43d8
commit 453481204d
12 changed files with 5716 additions and 4968 deletions
+12 -5
View File
@@ -9,6 +9,8 @@
B{Entropy Package Manager Client Instance Loaders Interface}.
"""
import warnings
from entropy.spm.plugins.factory import get_default_instance as get_spm, \
get_default_class as get_spm_default_class
@@ -25,12 +27,15 @@ class LoadersMixin:
# during complete system upgrades
from entropy.client.interfaces.trigger import Trigger
from entropy.client.interfaces.repository import Repository
from entropy.client.interfaces.package import Package
from entropy.client.interfaces.package import \
PackageActionFactory, PackageActionFactoryWrapper
from entropy.client.interfaces.sets import Sets
from entropy.client.misc import ConfigurationUpdates
from entropy.client.services.interfaces import \
ClientWebServiceFactory, RepositoryWebServiceFactory
self.__package_loader = Package
self.__package_factory = PackageActionFactory
self.__package_loader = PackageActionFactoryWrapper
self.__repository_loader = Repository
self.__trigger_loader = Trigger
self.__sets_loader = Sets
@@ -141,14 +146,16 @@ class LoadersMixin:
def Package(self):
"""
Load Entropy Package instance object
@return:
@rtype: entropy.client.interfaces.package.Package
"""
warnings.warn("deprecated, use PackageActionFactory",
DeprecationWarning)
return self.__package_loader(self)
def PackageActionFactory(self):
"""
Load Entropy PackageActionFactory instance object
"""
return self.__package_factory(self)
def ConfigurationUpdates(self):
"""
File diff suppressed because it is too large Load Diff
@@ -10,9 +10,11 @@
"""
import codecs
import errno
import sys
import os
from entropy.const import etpConst
from entropy.const import etpConst, const_mkstemp
import entropy.tools
@@ -312,3 +314,189 @@ class FileContentSafetyReader(object):
if self._file is not None:
self._file.close()
self._file = None
def generate_content_safety_file(content_safety):
"""
Generate a file containing the "content_safety" metadata,
reading by content_safety list or iterator. Each item
of "content_safety" must contain (path, sha256, mtime).
Each item shall be written to file, one per line,
in the following form: "<mtime>|<sha256>|<path>".
The order of the element in "content_safety" will be kept.
"""
tmp_dir = os.path.join(
etpConst['entropyunpackdir'],
"__generate_content_safety_file_f")
try:
os.makedirs(tmp_dir, 0o755)
except OSError as err:
if err.errno != errno.EEXIST:
raise
tmp_fd, tmp_path = None, None
generated = False
try:
tmp_fd, tmp_path = const_mkstemp(
prefix="PackageContentSafety",
dir=tmp_dir)
with FileContentSafetyWriter(tmp_fd) as tmp_f:
for path, sha256, mtime in content_safety:
tmp_f.write(path, sha256, mtime)
generated = True
return tmp_path
finally:
if tmp_fd is not None:
try:
os.close(tmp_fd)
except OSError:
pass
if tmp_path is not None and not generated:
try:
os.remove(tmp_path)
except (OSError, IOError):
pass
def generate_content_file(content, package_id = None,
filter_splitdebug = False,
splitdebug = None,
splitdebug_dirs = None):
"""
Generate a file containing the "content" metadata,
reading by content list or iterator. Each item
of "content" must contain (path, ftype).
Each item shall be written to file, one per line,
in the following form: "[<package_id>|]<ftype>|<path>".
The order of the element in "content" will be kept.
"""
tmp_dir = os.path.join(
etpConst['entropyunpackdir'],
"__generate_content_file_f")
try:
os.makedirs(tmp_dir, 0o755)
except OSError as err:
if err.errno != errno.EEXIST:
raise
tmp_fd, tmp_path = None, None
generated = False
try:
tmp_fd, tmp_path = const_mkstemp(
prefix="PackageContent",
dir=tmp_dir)
with FileContentWriter(tmp_fd) as tmp_f:
for path, ftype in content:
if filter_splitdebug and not splitdebug:
# if filter_splitdebug is enabled, this
# code filters out all the paths starting
# with splitdebug_dirs, if splitdebug is
# disabled for package.
_skip = False
for split_dir in splitdebug_dirs:
if path.startswith(split_dir):
_skip = True
break
if _skip:
continue
tmp_f.write(package_id, path, ftype)
generated = True
return tmp_path
finally:
if tmp_fd is not None:
try:
os.close(tmp_fd)
except OSError:
pass
if tmp_path is not None and not generated:
try:
os.remove(tmp_path)
except (OSError, IOError):
pass
def merge_content_file(content_file, sorted_content,
cmp_func):
"""
Given a sorted content_file content and a sorted list of
content (sorted_content), apply the "merge" step of a merge
sort algorithm. In other words, add the sorted_content to
content_file keeping content_file content ordered.
It is of couse O(n+m) where n = lines in content_file and
m = sorted_content length.
"""
tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX
sorted_ptr = 0
_sorted_path = None
_sorted_ftype = None
_package_id = 0 # will be filled
try:
with FileContentWriter(tmp_content_file) as tmp_w:
with FileContentReader(content_file) as tmp_r:
for _package_id, _path, _ftype in tmp_r:
while True:
try:
_sorted_path, _sorted_ftype = \
sorted_content[sorted_ptr]
except IndexError:
_sorted_path = None
_sorted_ftype = None
if _sorted_path is None:
tmp_w.write(_package_id, _path, _ftype)
break
cmp_outcome = cmp_func(_path, _sorted_path)
if cmp_outcome < 0:
tmp_w.write(_package_id, _path, _ftype)
break
# always privilege _ftype over _sorted_ftype
# _sorted_ftype might be invalid
tmp_w.write(
_package_id, _sorted_path, _ftype)
sorted_ptr += 1
if cmp_outcome == 0:
# write only one
break
# add the remainder
if sorted_ptr < len(sorted_content):
_sorted_rem = sorted_content[sorted_ptr:]
for _sorted_path, _sorted_ftype in _sorted_rem:
tmp_w.write(
_package_id, _sorted_path, _sorted_ftype)
os.rename(tmp_content_file, content_file)
finally:
try:
os.remove(tmp_content_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def filter_content_file(content_file, filter_func):
"""
This method rewrites the content of content_file by applying
a filter to the path elements.
"""
tmp_content_file = content_file + FileContentWriter.TMP_SUFFIX
try:
with FileContentWriter(tmp_content_file) as tmp_w:
with FileContentReader(content_file) as tmp_r:
for _package_id, _path, _ftype in tmp_r:
if filter_func(_path):
tmp_w.write(_package_id, _path, _ftype)
os.rename(tmp_content_file, content_file)
finally:
try:
os.remove(tmp_content_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise
@@ -0,0 +1,590 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
import errno
import os
from entropy.const import etpConst, const_convert_to_unicode, \
const_convert_to_rawstring, const_is_python3
from entropy.i18n import _
from entropy.output import red, purple, brown, darkred, blue, darkgreen
import entropy.tools
from .. import _content as Content
from .action import PackageAction
class _PackageInstallRemoveAction(PackageAction):
"""
Abstract class that exposes shared functions between install
and remove PackageAction classes.
"""
def metadata(self):
"""
Return the package metadata dict object for manipulation.
"""
return self._meta
def setup(self):
"""
Overridden from PackageAction.
"""
raise NotImplementedError()
def _run(self):
"""
Overridden from PackageAction.
"""
raise NotImplementedError()
def _get_system_root(self):
"""
Return the path to the system root directory.
"""
metadata = self.metadata()
return metadata.get('unittest_root', "") + etpConst['systemroot']
def _get_config_protect_skip(self):
"""
Return the configuration protection path set.
"""
misc_settings = self._entropy.ClientSettings()['misc']
protectskip = misc_settings['configprotectskip']
if not const_is_python3():
protectskip = set((
const_convert_to_rawstring(
x, from_enctype = etpConst['conf_encoding']) for x in
misc_settings['configprotectskip']))
return protectskip
def _get_config_protect(self, entropy_repository, package_id, mask = False):
"""
Return configuration protection (or mask) metadata for the given
package.
This method should not be used as source for storing metadata into
repositories since the returned objects may not be decoded in utf-8.
Data returned by this method is expected to be used only by internal
functions.
"""
cl_id = etpConst['system_settings_plugins_ids']['client_plugin']
misc_data = self._settings[cl_id]['misc']
if mask:
paths = entropy_repository.retrieveProtectMask(package_id).split()
misc_key = "configprotectmask"
else:
paths = entropy_repository.retrieveProtect(package_id).split()
misc_key = "configprotect"
root = etpConst['systemroot']
config = set(("%s%s" % (root, path) for path in paths))
config.update(misc_data[misc_key])
# os.* methods in Python 2.x do not expect unicode strings
# This set of data is only used by _handle_config_protect atm.
if not const_is_python3():
config = set((const_convert_to_rawstring(x) for x in config))
return config
def _get_config_protect_metadata(self, installed_repository,
installed_package_id):
"""
Get the config_protect+mask metadata object.
Make sure to call this before the package goes away from the
repository.
"""
protect = self._get_config_protect(
installed_repository, installed_package_id)
mask = self._get_config_protect(
installed_repository, installed_package_id, mask = True)
metadata = {
'config_protect+mask': (protect, mask)
}
return metadata
def _handle_config_protect(self, protect, mask, protectskip,
fromfile, tofile,
do_allocation_check = True,
do_quiet = False):
"""
Handle configuration file protection. This method contains the logic
for determining if a file should be protected from overwrite.
"""
protected = False
do_continue = False
in_mask = False
if tofile in protect:
protected = True
in_mask = True
elif os.path.dirname(tofile) in protect:
protected = True
in_mask = True
else:
tofile_testdir = os.path.dirname(tofile)
old_tofile_testdir = None
while tofile_testdir != old_tofile_testdir:
if tofile_testdir in protect:
protected = True
in_mask = True
break
old_tofile_testdir = tofile_testdir
tofile_testdir = os.path.dirname(tofile_testdir)
if protected: # check if perhaps, file is masked, so unprotected
if tofile in mask:
protected = False
in_mask = False
elif os.path.dirname(tofile) in mask:
protected = False
in_mask = False
else:
tofile_testdir = os.path.dirname(tofile)
old_tofile_testdir = None
while tofile_testdir != old_tofile_testdir:
if tofile_testdir in mask:
protected = False
in_mask = False
break
old_tofile_testdir = tofile_testdir
tofile_testdir = os.path.dirname(tofile_testdir)
if not os.path.lexists(tofile):
protected = False # file doesn't exist
# check if it's a text file
if protected:
protected = entropy.tools.istextfile(tofile)
in_mask = protected
if fromfile is not None:
if protected and os.path.lexists(fromfile) and \
(not os.path.exists(fromfile)) and os.path.islink(fromfile):
# broken symlink, don't protect
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to handle file protection for: " \
"%s, broken symlink in package" % (
tofile,
)
)
msg = _("Cannot protect broken symlink")
mytxt = "%s:" % (
purple(msg),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = brown(" ## ")
)
self._entropy.output(
tofile,
level = "warning",
header = brown(" ## ")
)
protected = False
if not protected:
return in_mask, protected, tofile, do_continue
## ##
# file is protected #
##__________________##
# check if protection is disabled for this element
if tofile in protectskip:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Skipping config file installation/removal, " \
"as stated in client.conf: %s" % (tofile,)
)
if not do_quiet:
mytxt = "%s: %s" % (
_("Skipping file installation/removal"),
tofile,
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
do_continue = True
return in_mask, protected, tofile, do_continue
## ##
# file is protected (2) #
##______________________##
prot_status = True
if do_allocation_check:
spm_class = self._entropy.Spm_class()
tofile, prot_status = spm_class.allocate_protected_file(fromfile,
tofile)
if not prot_status:
# a protected file with the same content
# is already in place, so not going to protect
# the same file twice
protected = False
return in_mask, protected, tofile, do_continue
## ##
# file is protected (3) #
##______________________##
oldtofile = tofile
if oldtofile.find("._cfg") != -1:
oldtofile = os.path.join(os.path.dirname(oldtofile),
os.path.basename(oldtofile)[10:])
if not do_quiet:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Protecting config file: %s" % (oldtofile,)
)
mytxt = red("%s: %s") % (_("Protecting config file"), oldtofile,)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
return in_mask, protected, tofile, do_continue
def _remove_content_from_system_loop(self, inst_repo, remove_content,
directories, directories_cache,
not_removed_due_to_collisions,
colliding_path_messages,
automerge_metadata, col_protect,
protect, mask, protectskip,
sys_root):
"""
Body of the _remove_content_from_system() method.
"""
info_dirs = self._get_info_directories()
metadata = self.metadata()
for _pkg_id, item, _ftype in remove_content:
if not item:
continue # empty element??
sys_root_item = sys_root + item
sys_root_item_encoded = sys_root_item
if not const_is_python3():
# this is coming from the db, and it's pure utf-8
sys_root_item_encoded = const_convert_to_rawstring(
sys_root_item,
from_enctype = etpConst['conf_raw_encoding'])
# collision check
if col_protect > 0:
if inst_repo.isFileAvailable(item) \
and os.path.isfile(sys_root_item_encoded):
# in this way we filter out directories
colliding_path_messages.add(sys_root_item)
not_removed_due_to_collisions.add(item)
continue
protected = False
in_mask = False
if not metadata['removeconfig']:
protected_item_test = sys_root_item
(in_mask, protected, _x,
do_continue) = self._handle_config_protect(
protect, mask, protectskip, None, protected_item_test,
do_allocation_check = False, do_quiet = True
)
if do_continue:
protected = True
# when files have not been modified by the user
# and they are inside a config protect directory
# we could even remove them directly
if in_mask:
oldprot_md5 = automerge_metadata.get(item)
if oldprot_md5:
try:
in_system_md5 = entropy.tools.md5sum(
protected_item_test)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
in_system_md5 = "?"
if oldprot_md5 == in_system_md5:
prot_msg = _("Removing config file, never modified")
mytxt = "%s: %s" % (
darkgreen(prot_msg),
blue(item),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
protected = False
do_continue = False
# Is file or directory a protected item?
if protected:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['verbose_loglevel_id'],
"[remove] Protecting config file: %s" % (sys_root_item,)
)
mytxt = "[%s] %s: %s" % (
red(_("remove")),
brown(_("Protecting config file")),
sys_root_item,
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
continue
try:
os.lstat(sys_root_item_encoded)
except OSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
continue # skip file, does not exist
raise
except UnicodeEncodeError:
msg = _("This package contains a badly encoded file !!!")
mytxt = brown(msg)
self._entropy.output(
red("QA: ")+mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
continue # file has a really bad encoding
if os.path.isdir(sys_root_item_encoded) and \
os.path.islink(sys_root_item_encoded):
# S_ISDIR returns False for directory symlinks,
# so using os.path.isdir valid directory symlink
if sys_root_item not in directories_cache:
# collect for Trigger
metadata['affected_directories'].add(item)
directories.add((sys_root_item, "link"))
directories_cache.add(sys_root_item)
continue
if os.path.isdir(sys_root_item_encoded):
# plain directory
if sys_root_item not in directories_cache:
# collect for Trigger
metadata['affected_directories'].add(item)
directories.add((sys_root_item, "dir"))
directories_cache.add(sys_root_item)
continue
# files, symlinks or not
# just a file or symlink or broken
# directory symlink (remove now)
try:
os.remove(sys_root_item_encoded)
except OSError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"[remove] Unable to remove %s, error: %s" % (
sys_root_item, err,)
)
continue
# collect for Trigger
dir_name = os.path.dirname(item)
metadata['affected_directories'].add(dir_name)
# account for info files, if any
if dir_name in info_dirs:
for _ext in self._INFO_EXTS:
if item.endswith(_ext):
metadata['affected_infofiles'].add(item)
break
# add its parent directory
dirobj = const_convert_to_unicode(
os.path.dirname(sys_root_item_encoded))
if dirobj not in directories_cache:
if os.path.isdir(dirobj) and os.path.islink(dirobj):
directories.add((dirobj, "link"))
elif os.path.isdir(dirobj):
directories.add((dirobj, "dir"))
directories_cache.add(dirobj)
def _remove_content_from_system(self, installed_repository,
automerge_metadata = None):
"""
Remove installed package content (files/directories) from live system.
@keyword automerge_metadata: Entropy "automerge metadata"
@type automerge_metadata: dict
"""
if automerge_metadata is None:
automerge_metadata = {}
metadata = self.metadata()
sys_root = etpConst['systemroot']
# load CONFIG_PROTECT and CONFIG_PROTECT_MASK
misc_settings = self._entropy.ClientSettings()['misc']
col_protect = misc_settings['collisionprotect']
# remove files from system
directories = set()
directories_cache = set()
not_removed_due_to_collisions = set()
colliding_path_messages = set()
protect_mask = metadata['config_protect+mask']
if protect_mask is not None:
protect, mask = protect_mask
else:
protect, mask = set(), set()
protectskip = self._get_config_protect_skip()
remove_content = None
try:
# simulate a removecontent list/set object
remove_content = []
if metadata['removecontent_file'] is not None:
remove_content = Content.FileContentReader(
metadata['removecontent_file'])
self._remove_content_from_system_loop(
installed_repository,
remove_content, directories, directories_cache,
not_removed_due_to_collisions, colliding_path_messages,
automerge_metadata, col_protect, protect, mask, protectskip,
sys_root)
finally:
if hasattr(remove_content, "close"):
remove_content.close()
if colliding_path_messages:
self._entropy.output(
"%s:" % (_("Collision found during removal of"),),
importance = 1,
level = "warning",
header = red(" ## ")
)
for path in sorted(colliding_path_messages):
self._entropy.output(
purple(path),
importance = 0,
level = "warning",
header = red(" ## ")
)
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"Collision found during removal of %s - cannot overwrite" % (
path,)
)
# removing files not removed from removecontent.
# it happened that boot services not removed due to
# collisions got removed from their belonging runlevels
# by postremove step.
# since this is a set, it is a mapped type, so every
# other instance around will feature this update
if not_removed_due_to_collisions:
def _filter(_path):
return _path not in not_removed_due_to_collisions
Content.filter_content_file(
metadata['removecontent_file'],
_filter)
# now handle directories
directories = sorted(directories, reverse = True)
while True:
taint = False
for directory, dirtype in directories:
mydir = "%s%s" % (sys_root, directory,)
if dirtype == "link":
try:
mylist = os.listdir(mydir)
if not mylist:
try:
os.remove(mydir)
taint = True
except OSError:
pass
except OSError:
pass
elif dirtype == "dir":
try:
mylist = os.listdir(mydir)
if not mylist:
try:
os.rmdir(mydir)
taint = True
except OSError:
pass
except OSError:
pass
if not taint:
break
del directories_cache
del directories
def _spm_remove_package(self, atom):
"""
Call Source Package Manager interface and tell it to remove our
just removed package.
@return: execution status
@rtype: int
"""
spm = self._entropy.Spm()
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Removing from SPM: %s" % (atom,)
)
return spm.remove_installed_package(self.metadata())
@@ -0,0 +1,300 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
import os
from entropy.const import etpConst, const_convert_to_unicode
from entropy.i18n import _
from entropy.output import darkred, blue
import entropy.dep
from .. import _content as Content
class PackageAction(object):
"""
Base PackageAction object. Do not instantiate this and its subclasses
directly but rather use the PackageActionFactory.
"""
_INFO_EXTS = (
const_convert_to_unicode(".gz"),
const_convert_to_unicode(".bz2")
)
# Set a valid action name in subclasses
NAME = None
def __init__(self, entropy_client, package_match, opts = None):
self._entropy = entropy_client
self._settings = self._entropy.Settings()
self._package_match = package_match
self._package_id, self._repository_id = package_match
if opts is None:
opts = {}
self._opts = opts
self._xterm_header = ""
self._content_files = []
def package_id(self):
"""
Return the package identifier of this object.
"""
return self._package_id
def repository_id(self):
"""
Return the repository identifier of this object.
"""
return self._repository_id
def atom(self):
"""
Return the package atom string of this object.
"""
repo = self._entropy.open_repository(self._repository_id)
return repo.retrieveAtom(self._package_id)
def set_xterm_header(self, header):
"""
Set the xterm terminal header text that will prefix the activity title.
@param header: the xterm header title
@type header: string
"""
self._xterm_header = header
def setup(self):
"""
Setup the action metadata. There is no need to call this directly,
unless you want to pre-generate the whole PackageAction metadata.
This method will be called by start() anyway.
"""
raise NotImplementedError()
def start(self):
"""
Execute the action. Return an exit status.
"""
acquired = False
exit_st = 1
try:
acquired = not self._entropy.wait_resources()
if not acquired:
return 20
exit_st = self._run()
finally:
if acquired:
self._entropy.unlock_resources()
if exit_st != 0:
self._entropy.output(
blue(_("An error occured. Action aborted.")),
importance = 2,
level = "error",
header = darkred(" ## ")
)
return exit_st
def _run(self):
"""
This method is called by start() and this is where subclasses
must implement their "run()" logic.
This method must return an exit status code (int).
This method must call setup() at the beginning of its execution.
"""
raise NotImplementedError()
def finalize(self):
"""
Finalize the object, release all its resources.
Subclasses must call this method in their overridden ones.
"""
# remove temporary content files
for content_file in self._content_files:
try:
os.remove(content_file)
except (OSError, IOError):
pass
def metadata(self):
"""
Return the package metadata dict object for manipulation.
"""
raise NotImplementedError()
@classmethod
def splitdebug_enabled(cls, entropy_client, pkg_match):
"""
Return whether splitdebug is enabled for package.
"""
settings = entropy_client.Settings()
# this is a SystemSettings.CachingList object
splitdebug = settings['splitdebug']
splitdebug_mask = settings['splitdebug_mask']
_pkg_id, pkg_repo = pkg_match
def _generate_cache(lst_obj):
# compute the package matching then
pkg_matches = set()
for dep in lst_obj:
dep, repo_ids = entropy.dep.dep_get_match_in_repos(dep)
if repo_ids is not None:
if pkg_repo not in repo_ids:
# skip entry, not me
continue
dep_matches, _rc = entropy_client.atom_match(
dep, multi_match=True, multi_repo=True)
pkg_matches |= dep_matches
# set cache back
lst_obj.set(pkg_matches)
return pkg_matches
enabled = False
if not splitdebug:
# no entries, consider splitdebug always enabled
enabled = True
else:
# whitelist support
pkg_matches = splitdebug.get()
if pkg_matches is None:
pkg_matches = _generate_cache(splitdebug)
# determine if it's enabled then
enabled = pkg_match in pkg_matches
# if it's enabled, check whether it's blacklisted
if enabled:
# blacklist support
pkg_matches = splitdebug_mask.get()
if pkg_matches is None:
# compute the package matching
pkg_matches = _generate_cache(splitdebug_mask)
enabled = pkg_match not in pkg_matches
return enabled
@classmethod
def get_standard_fetch_disk_path(cls, download):
"""
Return standard path where package is going to be downloaded.
"download" argument passed must come from
EntropyRepository.retrieveDownloadURL()
"""
return os.path.join(etpConst['entropypackagesworkdir'], download)
def __repr__(self):
return "<%s at %s | metadata: %s" % (
self.__class__.__name__,
hex(id(self)),
self.metadata())
def __str__(self):
return repr(self)
def _get_info_directories(self):
"""
Return a list of `info` directories as declared in the
INFOPATH and INFODIR environment variable.
"""
info_dirs = os.getenv("INFOPATH", "").split(":")
info_dirs += os.getenv("INFODIR", "").split(":")
info_dirs = [const_convert_to_unicode(
os.path.normpath(x)) for x in info_dirs]
info_dirs.sort()
return info_dirs
def _get_splitdebug_metadata(self):
"""
Return package metadata related to split debug files support.
"""
client_settings = self._entropy.ClientSettings()
misc_data = client_settings['misc']
splitdebug = misc_data['splitdebug']
splitdebug_dirs = misc_data['splitdebug_dirs']
metadata = {
'splitdebug': splitdebug,
'splitdebug_dirs': splitdebug_dirs,
}
return metadata
def _package_splitdebug_enabled(self, pkg_match):
"""
Determine if splitdebug is enabled for the package being installed
or just fetched. This method should be called only if system-wide
splitdebug setting in client.conf is enabled already.
"""
return self.splitdebug_enabled(self._entropy, pkg_match)
def _generate_content_file(self, content, package_id = None,
filter_splitdebug = False,
splitdebug = None,
splitdebug_dirs = None):
"""
Generate a file containing the package content metadata.
"""
content_path = None
try:
content_path = Content.generate_content_file(
content, package_id = package_id,
filter_splitdebug = filter_splitdebug,
splitdebug = splitdebug,
splitdebug_dirs = splitdebug_dirs)
return content_path
finally:
if content_path is not None:
self._content_files.append(content_path)
def _generate_content_safety_file(self, content_safety):
"""
Generate a file containing the package content safety metadata.
"""
content_path = None
try:
content_path = Content.generate_content_safety_file(
content_safety)
return content_path
finally:
if content_path is not None:
self._content_files.append(content_path)
@classmethod
def _get_url_name(cls, url):
"""
Given a mirror URL, returns a smaller string representing the URL name.
@param url: URL string
@type url: string
@return: representative URL string
@rtype: string
"""
url_data = entropy.tools.spliturl(url)
url_name = url_data.netloc
url_scheme = url_data.scheme
if not url_scheme:
url_scheme = "unknown"
return "%s://%s" % (url_scheme, url_name,)
@classmethod
def _get_licenses(cls, entropy_repository, package_id):
"""
Return a set of license identifiers for the given package.
"""
pkg_license = set()
r_license = entropy_repository.retrieveLicense(package_id)
if r_license is not None:
pkg_license |= set(r_license.split())
return pkg_license
@@ -0,0 +1,211 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
from entropy.const import etpConst
from entropy.i18n import _
from entropy.output import red, darkred, blue, brown
from .action import PackageAction
class _PackageConfigAction(PackageAction):
"""
PackageAction used for package configuration.
"""
NAME = "config"
def __init__(self, entropy_client, package_match, opts = None):
"""
Object constructor.
"""
super(_PackageConfigAction, self).__init__(
entropy_client, package_match, opts = opts)
self._meta = None
def finalize(self):
"""
Finalize the object, release all its resources.
"""
super(_PackageConfigAction, self).finalize()
if self._meta is not None:
meta = self._meta
self._meta = None
meta.clear()
def metadata(self):
"""
Return the package metadata dict object for manipulation.
"""
return self._meta
def setup(self):
"""
Setup the PackageAction.
"""
if self._meta is not None:
# already configured
return
metadata = {}
splitdebug_metadata = self._get_splitdebug_metadata()
metadata.update(splitdebug_metadata)
inst_repo = self._entropy.open_repository(self._repository_id)
metadata['atom'] = inst_repo.retrieveAtom(self._package_id)
key, slot = inst_repo.retrieveKeySlot(self._package_id)
metadata['key'], metadata['slot'] = key, slot
metadata['version'] = inst_repo.retrieveVersion(self._package_id)
metadata['category'] = inst_repo.retrieveCategory(self._package_id)
metadata['name'] = inst_repo.retrieveName(self._package_id)
metadata['spm_repository'] = inst_repo.retrieveSpmRepository(
self._package_id)
metadata['accept_license'] = self._get_licenses(
inst_repo, self._package_id)
metadata['phases'] = []
metadata['phases'].append(self._config)
self._meta = metadata
def _run(self):
"""
Execute the action. Return an exit status.
"""
self.setup()
exit_st = 0
for method in self._meta['phases']:
exit_st = method()
if exit_st != 0:
break
return exit_st
def _configure_package(self):
"""
Configure the package.
"""
spm = self._entropy.Spm()
self._entropy.output(
"SPM: %s" % (
brown(_("configuration phase")),
),
importance = 0,
header = red(" ## ")
)
try:
spm.execute_package_phase(
self._meta, self._meta,
self.NAME, "configure")
except spm.PhaseFailure as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Phase execution failed with %s, %d" % (
err.message, err.code))
return err.code
except spm.OutdatedPhaseError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager is too old: %s" % (
err))
err_msg = "%s: %s" % (
brown(_("Source Package Manager is too old, please update it")),
err)
self._entropy.output(
err_msg,
importance = 1,
header = darkred(" ## "),
level = "error"
)
return 1
except spm.PhaseError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Phase execution error: %s" % (
err))
return 1
return 0
def _config(self):
"""
Execute the config phase.
"""
xterm_title = "%s %s: %s" % (
self._xterm_header,
_("Configuring package"),
self._meta['atom'],
)
self._entropy.set_title(xterm_title)
txt = "%s: %s" % (
blue(_("Configuring package")),
red(self._meta['atom']),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
exit_st = self._configure_package()
if exit_st == 1:
txt = _("An error occured while trying to configure the package")
txt2 = "%s. %s: %s" % (
red(_("Make sure that your system is healthy")),
blue(_("Error")),
exit_st,
)
self._entropy.output(
darkred(txt),
importance = 1,
level = "error",
header = red(" ## ")
)
self._entropy.output(
txt2,
importance = 1,
level = "error",
header = red(" ## ")
)
elif exit_st == 2:
txt = _("An error occured while trying to configure the package")
txt2 = "%s. %s: %s" % (
red(_("It seems that Source Package Manager entry is missing")),
blue(_("Error")),
exit_st,
)
self._entropy.output(
darkred(txt),
importance = 1,
level = "error",
header = red(" ## ")
)
self._entropy.output(
txt2,
importance = 1,
level = "error",
header = red(" ## ")
)
return exit_st
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,757 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
import errno
import os
from entropy.const import etpConst, const_setup_perms
from entropy.client.mirrors import StatusInterface
from entropy.fetchers import UrlFetcher
from entropy.output import blue, darkblue, bold, red, darkred, brown, darkgreen
from entropy.i18n import _, ngettext
import entropy.dep
import entropy.tools
from .fetch import _PackageFetchAction
class _PackageMultiFetchAction(_PackageFetchAction):
"""
PackageAction used for package source code download.
As opposed to the other PackageAction classes, this class
expects a list of package matches instead of just one.
"""
NAME = "multi_fetch"
def __init__(self, entropy_client, package_matches, opts = None):
"""
Object constructor.
"""
super(_PackageMultiFetchAction, self).__init__(
entropy_client, (None, None), opts = opts)
self._package_matches = package_matches
self._meta = None
def finalize(self):
"""
Finalize the object, release all its resources.
"""
super(_PackageMultiFetchAction, self).finalize()
if self._meta is not None:
meta = self._meta
self._meta = None
meta.clear()
def metadata(self):
"""
Return the package metadata dict object for manipulation.
"""
return self._meta
def setup(self):
"""
Setup the PackageAction.
"""
if self._meta is not None:
# already configured
return
metadata = {}
splitdebug_metadata = self._get_splitdebug_metadata()
metadata.update(splitdebug_metadata)
metadata['fetch_abort_function'] = self._opts.get(
'fetch_abort_function')
misc_settings = self._entropy.ClientSettings()['misc']
metadata['edelta_support'] = misc_settings['edelta_support']
metadata['matches'] = self._package_matches
metadata['atoms'] = []
metadata['repository_atoms'] = {}
temp_fetch_list = []
temp_checksum_list = []
temp_already_downloaded_count = 0
def _setup_download(download, size, package_id, repository_id, digest,
signatures):
obj = (package_id, repository_id, download, digest, signatures)
temp_checksum_list.append(obj)
down_path = self.get_standard_fetch_disk_path(download)
try:
down_st = os.lstat(down_path)
st_size = down_st.st_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
st_size = None
if st_size is not None:
if st_size == size:
return 1
else:
obj = (package_id, repository_id, download, digest, signatures)
temp_fetch_list.append(obj)
return 0
for package_id, repository_id in self._package_matches:
if repository_id.endswith(etpConst['packagesext']):
continue
repo = self._entropy.open_repository(repository_id)
atom = repo.retrieveAtom(package_id)
metadata['atoms'].append(atom)
if repository_id not in metadata['repository_atoms']:
metadata['repository_atoms'][repository_id] = set()
metadata['repository_atoms'][repository_id].add(atom)
download = repo.retrieveDownloadURL(package_id)
digest = repo.retrieveDigest(package_id)
sha1, sha256, sha512, gpg = repo.retrieveSignatures(package_id)
size = repo.retrieveSize(package_id)
signatures = {
'sha1': sha1,
'sha256': sha256,
'sha512': sha512,
'gpg': gpg,
}
temp_already_downloaded_count += _setup_download(
download, size, package_id, repository_id, digest, signatures)
extra_downloads = repo.retrieveExtraDownload(package_id)
splitdebug = metadata['splitdebug']
# if splitdebug is enabled, check if it's also enabled
# via package.splitdebug
if splitdebug:
splitdebug = self._package_splitdebug_enabled(
(package_id, repository_id))
if not splitdebug:
extra_downloads = [
x for x in extra_downloads if x['type'] != "debug"]
for extra_download in extra_downloads:
download = extra_download['download']
size = extra_download['size']
digest = extra_download['md5']
signatures = {
'sha1': extra_download['sha1'],
'sha256': extra_download['sha256'],
'sha512': extra_download['sha512'],
'gpg': extra_download['gpg'],
}
temp_already_downloaded_count += _setup_download(
download, size, package_id, repository_id,
digest, signatures)
metadata['multi_fetch_list'] = temp_fetch_list
metadata['multi_checksum_list'] = temp_checksum_list
metadata['phases'] = []
if metadata['multi_fetch_list']:
metadata['phases'].append(self._fetch)
if metadata['multi_checksum_list']:
metadata['phases'].append(self._checksum)
if temp_already_downloaded_count == len(temp_checksum_list):
metadata['phases'].reverse()
self._meta = metadata
def _run(self):
"""
Execute the action. Return an exit status.
"""
self.setup()
exit_st = 0
for method in self._meta['phases']:
exit_st = method()
if exit_st != 0:
break
return exit_st
def _try_edelta_multifetch(self, url_data, resume):
"""
Attempt to download and use the edelta file.
"""
# no edelta support enabled
if not self._meta.get('edelta_support'):
return [], 0.0, False
if not entropy.tools.is_entropy_delta_available():
return [], 0.0, False
url_path_list = []
url_data_map = {}
url_data_map_idx = 0
inst_repo = self._entropy.installed_repository()
for pkg_id, repository_id, url, dest_path, cksum in url_data:
repo = self._entropy.open_repository(repository_id)
if cksum is None:
# cannot setup edelta without checksum, get from repository
cksum = repo.retrieveDigest(pkg_id)
if cksum is None:
# still nothing
continue
key_slot = repo.retrieveKeySlotAggregated(pkg_id)
if key_slot is None:
# wtf corrupted entry, skip
continue
installed_package_id, _inst_rc = inst_repo.atomMatch(key_slot)
if installed_package_id == -1:
# package is not installed
continue
edelta_approve = self._approve_edelta(url, installed_package_id,
cksum)
if edelta_approve is None:
# no edelta support
continue
edelta_url, installed_fetch_path = edelta_approve
edelta_save_path = dest_path + etpConst['packagesdeltaext']
key = (edelta_url, edelta_save_path)
url_path_list.append(key)
url_data_map_idx += 1
url_data_map[url_data_map_idx] = (
pkg_id, repository_id, url,
dest_path, cksum, edelta_url,
edelta_save_path, installed_fetch_path)
if not url_path_list:
# no martini, no party!
return [], 0.0, False
fetch_abort_function = self._meta.get('fetch_abort_function')
fetch_intf = self._entropy._multiple_url_fetcher(url_path_list,
resume = resume, abort_check_func = fetch_abort_function,
url_fetcher_class = self._entropy._url_fetcher)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
data = fetch_intf.download()
except KeyboardInterrupt:
return [], 0.0, True
data_transfer = fetch_intf.get_transfer_rate()
fetch_errors = (
UrlFetcher.TIMEOUT_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_WARN,
)
valid_idxs = []
for url_data_map_idx, cksum in tuple(data.items()):
if cksum in fetch_errors:
# download failed
continue
(pkg_id, repository_id, url, dest_path,
orig_cksum, edelta_url, edelta_save_path,
installed_fetch_path) = url_data_map[url_data_map_idx]
# now check
tmp_dest_path = dest_path + ".edelta_pkg_tmp"
# yay, we can apply the delta and cook the new package file!
try:
entropy.tools.apply_entropy_delta(installed_fetch_path,
edelta_save_path, tmp_dest_path)
except IOError:
# give up with this edelta
try:
os.remove(tmp_dest_path)
except (OSError, IOError):
pass
continue
os.rename(tmp_dest_path, dest_path)
valid_idxs.append(url_data_map_idx)
fetched_url_data = []
for url_data_map_idx in valid_idxs:
(pkg_id, repository_id, url, dest_path,
orig_cksum, edelta_url, edelta_save_path,
installed_fetch_path) = url_data_map[url_data_map_idx]
try:
valid = entropy.tools.compare_md5(dest_path, orig_cksum)
except (IOError, OSError):
valid = False
if valid:
url_data_item = (pkg_id, repository_id, url,
dest_path, orig_cksum)
fetched_url_data.append(url_data_item)
return fetched_url_data, data_transfer, False
def _fetch_files(self, url_data_list, resume = True):
"""
Effectively fetch the package files.
"""
def _generate_checksum_map(url_data):
ck_map = {}
ck_map_id = 0
for _pkg_id, _repository_id, _url, _dest_path, cksum in url_data:
ck_map_id += 1
if cksum is not None:
ck_map[ck_map_id] = cksum
return ck_map
fetch_abort_function = self._meta.get('fetch_abort_function')
# avoid tainting data pointed by url_data_list
url_data = url_data_list[:]
diff_map = {}
# setup directories
for pkg_id, repository_id, url, dest_path, _cksum in url_data:
dest_dir = os.path.dirname(dest_path)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir, 0o775)
const_setup_perms(dest_dir, etpConst['entropygid'])
checksum_map = _generate_checksum_map(url_data)
fetched_url_data, data_transfer, abort = self._try_edelta_multifetch(
url_data, resume)
if abort:
return -100, {}, 0
for url_data_item in fetched_url_data:
url_data.remove(url_data_item)
# some packages haven't been downloaded using edelta
if url_data:
url_path_list = []
for pkg_id, repository_id, url, dest_path, _cksum in url_data:
url_path_list.append((url, dest_path,))
self._setup_differential_download(
self._entropy._multiple_url_fetcher, url, resume, dest_path,
repository_id, pkg_id)
# load class
fetch_intf = self._entropy._multiple_url_fetcher(
url_path_list, resume = resume,
abort_check_func = fetch_abort_function,
url_fetcher_class = self._entropy._url_fetcher,
checksum = True)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
data = fetch_intf.download()
except KeyboardInterrupt:
return -100, {}, 0
data_transfer = fetch_intf.get_transfer_rate()
checksum_map = _generate_checksum_map(url_data)
for ck_id in checksum_map:
orig_checksum = checksum_map.get(ck_id)
if orig_checksum != data.get(ck_id):
diff_map[url_path_list[ck_id-1][0]] = orig_checksum
if diff_map:
defval = -1
for key, val in tuple(diff_map.items()):
if val == UrlFetcher.GENERIC_FETCH_WARN:
diff_map[key] = -2
elif val == UrlFetcher.TIMEOUT_FETCH_ERROR:
diff_map[key] = -4
elif val == UrlFetcher.GENERIC_FETCH_ERROR:
diff_map[key] = -3
elif val == -100:
defval = -100
return defval, diff_map, data_transfer
return 0, diff_map, data_transfer
def _download_packages(self, download_list):
"""
Internal function. Download packages.
"""
avail_data = self._settings['repositories']['available']
excluded_data = self._settings['repositories']['excluded']
repo_uris = {}
for pkg_id, repository_id, fname, cksum, _signatures in download_list:
repo = self._entropy.open_repository(repository_id)
# grab original repo, if any and use it if available
# this is done in order to support "equo repo merge" feature
# allowing client-side repository package metadata moves.
original_repo = repo.getInstalledPackageRepository(pkg_id)
if (original_repo != repository_id) and (
original_repo not in avail_data) and (
original_repo is not None):
# build up a new uris list, at least try, hoping that
# repository is just shadowing original_repo
# for example: original_repo got copied to repository, without
# copying packages, which would be useless. like it happens
# with sabayon-weekly
uris = self._build_uris_list(original_repo, repository_id)
else:
if original_repo in avail_data:
uris = avail_data[original_repo]['packages'][::-1]
uris += avail_data[repository_id]['packages'][::-1]
elif original_repo in excluded_data:
uris = excluded_data[original_repo]['packages'][::-1]
uris += avail_data[repository_id]['packages'][::-1]
else:
uris = avail_data[repository_id]['packages'][::-1]
obj = repo_uris.setdefault(repository_id, [])
# append at the beginning
new_ones = [x for x in uris if x not in obj][::-1]
for new_obj in new_ones:
obj.insert(0, new_obj)
remaining = repo_uris.copy()
mirror_status = StatusInterface()
def get_best_mirror(repository_id):
try:
return remaining[repository_id][0]
except IndexError:
return None
def update_download_list(down_list, failed_down):
newlist = []
for pkg_id, repository_id, fname, cksum, signatures in down_list:
p_uri = get_best_mirror(repository_id)
p_uri = os.path.join(p_uri, fname)
if p_uri not in failed_down:
continue
newlist.append(
(pkg_id, repository_id, fname, cksum, signatures)
)
return newlist
# return True: for failing, return False: for fine
def mirror_fail_check(repository_id, best_mirror):
# check if uri is sane
if not mirror_status.get_failing_mirror_status(best_mirror) >= 30:
return False
# set to 30 for convenience
mirror_status.set_failing_mirror_status(best_mirror, 30)
mirrorcount = repo_uris[repository_id].index(best_mirror) + 1
txt = "( mirror #%s ) %s %s - %s" % (
mirrorcount,
blue(_("Mirror")),
red(self._get_url_name(best_mirror)),
_("maximum failure threshold reached"),
)
self._entropy.output(
txt,
importance = 1,
level = "warning",
header = red(" ## ")
)
if mirror_status.get_failing_mirror_status(best_mirror) == 30:
mirror_status.add_failing_mirror(best_mirror, 45)
elif mirror_status.get_failing_mirror_status(best_mirror) > 31:
mirror_status.add_failing_mirror(best_mirror, -4)
else:
mirror_status.set_failing_mirror_status(best_mirror, 0)
try:
remaining[repository_id].remove(best_mirror)
except ValueError:
# ignore
pass
return True
def show_download_summary(down_list):
for _pkg_id, repository_id, fname, _cksum, _signatures in down_list:
best_mirror = get_best_mirror(repository_id)
mirrorcount = repo_uris[repository_id].index(best_mirror) + 1
basef = os.path.basename(fname)
txt = "( mirror #%s ) [%s] %s %s" % (
mirrorcount,
brown(basef),
blue("@"),
red(self._get_url_name(best_mirror)),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
def show_successful_download(down_list, data_transfer):
for _pkg_id, repository_id, fname, _cksum, _signatures in down_list:
best_mirror = get_best_mirror(repository_id)
mirrorcount = repo_uris[repository_id].index(best_mirror) + 1
basef = os.path.basename(fname)
txt = "( mirror #%s ) [%s] %s %s %s" % (
mirrorcount,
brown(basef),
darkred(_("success")),
blue("@"),
red(self._get_url_name(best_mirror)),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
txt = " %s: %s%s%s" % (
blue(_("Aggregated transfer rate")),
bold(entropy.tools.bytes_into_human(data_transfer)),
darkred("/"),
darkblue(_("second")),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
def show_download_error(down_list, p_exit_st):
for _pkg_id, repository_id, _fname, _cksum, _signs in down_list:
best_mirror = get_best_mirror(repository_id)
mirrorcount = repo_uris[repository_id].index(best_mirror) + 1
txt = "( mirror #%s ) %s: %s" % (
mirrorcount,
blue(_("Error downloading from")),
red(self._get_url_name(best_mirror)),
)
if p_exit_st == -1:
txt += " - %s." % (
_("data not available on this mirror"),)
elif p_exit_st == -2:
mirror_status.add_failing_mirror(best_mirror, 1)
txt += " - %s." % (_("wrong checksum"),)
elif p_exit_st == -3:
txt += " - %s." % (_("not found"),)
elif p_exit_st == -4: # timeout!
txt += " - %s." % (_("timeout error"),)
elif p_exit_st == -100:
txt += " - %s." % (_("discarded download"),)
else:
mirror_status.add_failing_mirror(best_mirror, 5)
txt += " - %s." % (_("unknown reason"),)
self._entropy.output(
txt,
importance = 1,
level = "warning",
header = red(" ## ")
)
def remove_failing_mirrors(repos):
for repository_id in repos:
get_best_mirror(repository_id)
if remaining[repository_id]:
remaining[repository_id].pop(0)
def check_remaining_mirror_failure(repos):
return [x for x in repos if not remaining.get(x)]
d_list = download_list[:]
while True:
do_resume = True
timeout_try_count = 50
while True:
fetch_files_list = []
for pkg_id, repository_id, fname, cksum, _signs in d_list:
best_mirror = get_best_mirror(repository_id)
# set working mirror, dont care if its None
mirror_status.set_working_mirror(best_mirror)
if best_mirror is not None:
mirror_fail_check(repository_id, best_mirror)
best_mirror = get_best_mirror(repository_id)
if best_mirror is None:
# at least one package failed to download
# properly, give up with everything
return 3, d_list
myuri = os.path.join(best_mirror, fname)
pkg_path = self.get_standard_fetch_disk_path(fname)
fetch_files_list.append(
(pkg_id, repository_id, myuri, pkg_path, cksum,)
)
show_download_summary(d_list)
(exit_st, failed_downloads,
data_transfer) = self._fetch_files(
fetch_files_list, resume = do_resume)
if exit_st == 0:
show_successful_download(
d_list, data_transfer)
return 0, []
d_list = update_download_list(
d_list, failed_downloads)
if exit_st not in (-3, -4, -100,) and failed_downloads and \
do_resume:
# disable resume
do_resume = False
continue
show_download_error(d_list, exit_st)
if exit_st == -4: # timeout
timeout_try_count -= 1
if timeout_try_count > 0:
continue
elif exit_st == -100: # user discarded fetch
return 1, []
myrepos = set([x[1] for x in d_list])
remove_failing_mirrors(myrepos)
# make sure we don't have nasty issues
remaining_failure = check_remaining_mirror_failure(
myrepos)
if remaining_failure:
return 3, d_list
break
return 0, []
def _fetch(self):
"""
Execute the fetch phase.
"""
m_fetch_len = len(self._meta['multi_fetch_list']) / 2
xterm_title = "%s: %s %s" % (
_("Multi Fetching"),
m_fetch_len,
ngettext("package", "packages", m_fetch_len),
)
self._entropy.set_title(xterm_title)
m_fetch_len = len(self._meta['multi_fetch_list'])
txt = "%s: %s %s" % (
blue(_("Downloading")),
darkred("%s" % (m_fetch_len,)),
ngettext("archive", "archives", m_fetch_len),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
exit_st, err_list = self._download_packages(
self._meta['multi_fetch_list'])
if exit_st == 0:
return 0
txt = _("Some packages cannot be fetched")
txt2 = _("Try to update your repositories and retry")
for txt in (txt, txt2,):
self._entropy.output(
"%s." % (
darkred(txt),
),
importance = 0,
level = "info",
header = red(" ## ")
)
self._entropy.output(
"%s: %s" % (
brown(_("Error")),
exit_st,
),
importance = 0,
level = "info",
header = red(" ## ")
)
for _pkg_id, repo, fname, cksum, _signatures in err_list:
self._entropy.output(
"[%s|%s] %s" % (
blue(repo),
darkgreen(cksum),
darkred(fname),
),
importance = 1,
level = "error",
header = darkred(" # ")
)
return exit_st
def _checksum(self):
"""
Execute the checksum verification phase.
"""
m_len = len(self._meta['multi_checksum_list'])
xterm_title = "%s: %s %s" % (
_("Multi Verification"),
m_len,
ngettext("package", "packages", m_len),
)
self._entropy.set_title(xterm_title)
exit_st = 0
ck_list = self._meta['multi_checksum_list']
for pkg_id, repository_id, download, digest, signatures in ck_list:
exit_st = self._match_checksum(
pkg_id, repository_id, digest, download, signatures)
if exit_st != 0:
break
return exit_st
@@ -0,0 +1,286 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
from entropy.const import etpConst
from entropy.exceptions import SPMError
from entropy.i18n import _
from entropy.output import blue, red
import entropy.dep
from ._manage import _PackageInstallRemoveAction
class _PackageRemoveAction(_PackageInstallRemoveAction):
"""
PackageAction used for package removal.
"""
NAME = "remove"
def __init__(self, entropy_client, package_match, opts = None):
"""
Object constructor.
"""
super(_PackageRemoveAction, self).__init__(
entropy_client, package_match, opts = opts)
self._meta = None
def finalize(self):
"""
Finalize the object, release all its resources.
"""
super(_PackageRemoveAction, self).finalize()
if self._meta is not None:
meta = self._meta
self._meta = None
meta.clear()
def setup(self):
"""
Setup the PackageAction.
"""
if self._meta is not None:
# already configured
return
metadata = {}
splitdebug_metadata = self._get_splitdebug_metadata()
metadata.update(splitdebug_metadata)
inst_repo = self._entropy.open_repository(self._repository_id)
metadata['configprotect_data'] = []
metadata['triggers'] = {}
metadata['atom'] = inst_repo.retrieveAtom(self._package_id)
# removeatom metadata key used by Spm.remove_installed_package()
metadata['removeatom'] = metadata['atom']
metadata['slot'] = inst_repo.retrieveSlot(self._package_id)
metadata['versiontag'] = inst_repo.retrieveTag(self._package_id)
metadata['removeconfig'] = self._opts.get('removeconfig', False)
content = inst_repo.retrieveContentIter(
self._package_id, order_by="file", reverse=True)
metadata['removecontent_file'] = self._generate_content_file(
content)
# collects directories whose content has been modified
# this information is then handed to the Trigger
metadata['affected_directories'] = set()
metadata['affected_infofiles'] = set()
trigger = inst_repo.getTriggerData(self._package_id)
metadata['triggers']['remove'] = trigger
trigger['affected_directories'] = metadata['affected_directories']
trigger['affected_infofiles'] = metadata['affected_infofiles']
trigger['spm_repository'] = inst_repo.retrieveSpmRepository(
self._package_id)
trigger['accept_license'] = self._get_licenses(
inst_repo, self._package_id)
trigger.update(splitdebug_metadata)
# setup config_protect and config_protect+mask metadata before it's
# too late.
protect = self._get_config_protect_metadata(inst_repo, self._package_id)
metadata.update(protect)
metadata['phases'] = [
self._pre_remove,
self._remove,
self._post_remove,
self._post_remove_remove,
]
self._meta = metadata
def _pre_remove(self):
"""
Run the pre-remove phase.
"""
xterm_title = "%s %s: %s" % (
self._xterm_header,
_("Pre-remove"),
self._meta['atom'],
)
self._entropy.set_title(xterm_title)
exit_st = 0
data = self._meta['triggers']['remove']
if not data:
return exit_st
trigger = self._entropy.Triggers(
self.NAME, "preremove", data, None)
ack = trigger.prepare()
if ack:
exit_st = trigger.run()
trigger.kill()
return exit_st
def _remove(self):
"""
Run the remove phase.
"""
xterm_title = "%s %s: %s" % (
self._xterm_header,
_("Removing"),
self._meta['atom'],
)
self._entropy.set_title(xterm_title)
self._entropy.output(
"%s: %s" % (
blue(_("Removing")),
red(self._meta['atom']),
),
importance = 1,
level = "info",
header = red(" ## ")
)
self._entropy.clear_cache()
self._entropy.logger.log("[Package]",
etpConst['logging']['normal_loglevel_id'],
"Removing package: %s" % (self._meta['atom'],))
txt = "%s: %s" % (
blue(_("Removing from Entropy")),
red(self._meta['atom']),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
inst_repo = self._entropy.open_repository(self._repository_id)
automerge_metadata = inst_repo.retrieveAutomergefiles(
self._package_id, get_dict = True)
inst_repo.removePackage(self._package_id)
# commit changes, to avoid users pressing CTRL+C and still having
# all the db entries in, so we need to commit at every iteration
inst_repo.commit()
self._remove_content_from_system(
inst_repo, automerge_metadata = automerge_metadata)
return 0
def _post_remove(self):
"""
Run the first post-remove phase.
"""
xterm_title = "%s %s: %s" % (
self._xterm_header,
_("Post-remove"),
self._meta['atom'],
)
self._entropy.set_title(xterm_title)
exit_st = 0
data = self._meta['triggers']['remove']
if not data:
return exit_st
trigger = self._entropy.Triggers(
self.NAME, "postremove", data, None)
ack = trigger.prepare()
if ack:
exit_st = trigger.run()
trigger.kill()
return exit_st
def _spm_update_package_uid(self, installed_package_id, spm_atom):
"""
Update Source Package Manager <-> Entropy package identifiers coupling.
Entropy can handle multiple packages in the same scope from a SPM POV
(see the "package tag" feature to provide linux kernel module packages
for different kernel versions). This method just reassigns a new SPM
unique package identifier to Entropy.
@param installed_package_id: Entropy package identifier bound to
given spm_atom
@type installed_package_id: int
@param spm_atom: SPM package atom
@type spm_atom: string
@return: execution status
@rtype: int
"""
spm = self._entropy.Spm()
try:
spm_uid = spm.assign_uid_to_installed_package(spm_atom)
except (SPMError, KeyError,):
# installed package not available, we must ignore it
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Spm uid not available for Spm package: %s (pkg not avail?)" % (
spm_atom,
)
)
spm_uid = -1
if spm_uid != -1:
inst_repo = self._entropy.open_repository(self._repository_id)
inst_repo.insertSpmUid(installed_package_id, spm_uid)
def _post_remove_remove(self):
"""
Post-remove phase of package remove action, this step removes SPM
package entries if there are no other Entropy-tagged packages installed.
"""
# remove pkg
# -- now it's possible to remove SPM package entry.
# if another package with the same atom is installed in
# Entropy db, do not call SPM at all because it would cause
# to get that package removed from there resulting in matching
# inconsistencies.
# -- of course, we need to drop versiontag before being able to look
# for other pkgs with same atom but different tag (which is an
# entropy-only metadatum)
atom = self._meta['atom']
inst_repo = self._entropy.open_repository(self._repository_id)
test_atom = entropy.dep.remove_tag(atom)
installed_package_ids = inst_repo.getPackageIds(test_atom)
spm = self._entropy.Spm()
spm_atom = spm.convert_from_entropy_package_name(atom)
if not installed_package_ids:
exit_st = self._spm_remove_package(spm_atom)
if exit_st != 0:
return exit_st
for installed_package_id in installed_package_ids:
# we have one installed, we need to update SPM uid
self._spm_update_package_uid(installed_package_id, spm_atom)
return 0
def _run(self):
"""
Execute the action. Return an exit status.
"""
self.setup()
exit_st = 0
for method in self._meta['phases']:
exit_st = method()
if exit_st != 0:
break
return exit_st
@@ -0,0 +1,301 @@
# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
import os
import shutil
from entropy.const import etpConst, const_setup_perms
from entropy.i18n import _
from entropy.output import blue, red, brown
import entropy.dep
import entropy.tools
from .fetch import _PackageFetchAction
class _PackageSourceAction(_PackageFetchAction):
"""
PackageAction used for package source code download.
"""
NAME = "source"
def __init__(self, entropy_client, package_match, opts = None):
"""
Object constructor.
"""
super(_PackageSourceAction, self).__init__(
entropy_client, package_match, opts = opts)
self._meta = None
def finalize(self):
"""
Finalize the object, release all its resources.
"""
super(_PackageSourceAction, self).finalize()
if self._meta is not None:
meta = self._meta
self._meta = None
meta.clear()
def metadata(self):
"""
Return the package metadata dict object for manipulation.
"""
return self._meta
def setup(self):
"""
Setup the PackageAction.
"""
if self._meta is not None:
# already configured
return
metadata = {}
splitdebug_metadata = self._get_splitdebug_metadata()
metadata.update(splitdebug_metadata)
metadata['fetch_abort_function'] = self._opts.get(
'fetch_abort_function')
# NOTE: if you want to implement download-to-dir feature in your
# client, you've found what you were looking for.
# fetch_path is the path where data should be downloaded
# it overrides default path
fetch_path = self._opts.get('fetch_path', None)
if fetch_path is not None:
if entropy.tools.is_valid_path(fetch_path):
metadata['fetch_path'] = fetch_path
# if splitdebug is enabled, check if it's also enabled
# via package.splitdebug
splitdebug = metadata['splitdebug']
if splitdebug:
splitdebug = self._package_splitdebug_enabled(
self._package_match)
repo = self._entropy.open_repository(self._repository_id)
metadata['atom'] = repo.retrieveAtom(self._package_id)
metadata['slot'] = repo.retrieveSlot(self._package_id)
inst_repo = self._entropy.installed_repository()
metadata['installed_package_id'], _inst_rc = inst_repo.atomMatch(
entropy.dep.dep_getkey(metadata['atom']),
matchSlot = metadata['slot'])
metadata['edelta_support'] = False
metadata['extra_download'] = tuple()
metadata['download'] = repo.retrieveSources(
self._package_id, extended = True)
# fake path, don't use
metadata['pkgpath'] = etpConst['entropypackagesworkdir']
metadata['phases'] = []
if not metadata['download']:
metadata['phases'].append(self._fetch_not_available)
return
metadata['phases'].append(self._fetch)
# create sources destination directory
unpack_dir = os.path.join(
etpConst['entropyunpackdir'],
"sources", metadata['atom'])
metadata['unpackdir'] = unpack_dir
self._meta = metadata
def _run(self):
"""
Execute the action. Return an exit status.
"""
self.setup()
unpack_dir = self._meta['unpackdir']
if not self._meta.get('fetch_path'):
try:
if os.path.lexists(unpack_dir):
if os.path.isfile(unpack_dir):
os.remove(unpack_dir)
elif os.path.isdir(unpack_dir):
shutil.rmtree(unpack_dir, True)
if not os.path.lexists(unpack_dir):
os.makedirs(unpack_dir, 0o755)
const_setup_perms(unpack_dir, etpConst['entropygid'],
recursion = False, uid = etpConst['uid'])
except (OSError, IOError) as err:
self._entropy.output(
"%s: %s" % (
blue(_("Fetch path setup error")),
err,
),
importance = 1,
level = "info",
header = red(" ## ")
)
return 1
exit_st = 0
for method in self._meta['phases']:
exit_st = method()
if exit_st != 0:
break
return exit_st
def _fetch_not_available(self):
"""
Execute the fetch not available phase.
"""
self._entropy.output(
blue(_("Source code not available.")),
importance = 1,
level = "info",
header = red(" ## ")
)
return 0
def _fetch_source(self, url, dest_file):
"""
Fetch the source code tarball(s).
"""
self._entropy.output(
"%s: %s" % (
blue(_("Downloading")),
brown(url),
),
importance = 1,
level = "info",
header = red(" ## ")
)
exit_st, data_transfer, _resumed = self._fetch_file(
url, dest_file, digest = None, resume = False)
if exit_st == 0:
human_bytes = entropy.tools.bytes_into_human(data_transfer)
txt = "%s: %s %s %s/%s" % (
blue(_("Successfully downloaded from")),
red(self._get_url_name(url)),
_("at"),
human_bytes,
_("second"),
)
self._entropy.output(
txt,
importance = 1,
level = "info",
header = red(" ## ")
)
self._entropy.output(
"%s: %s" % (
blue(_("Local path")),
brown(dest_file),
),
importance = 1,
level = "info",
header = red(" # ")
)
return exit_st
error_message = "%s: %s" % (
blue(_("Error downloading from")),
red(self._get_url_name(url)),
)
if exit_st == -1:
error_message += " - %s." % (
_("file not available on this mirror"),
)
elif exit_st == -3:
error_message += " - not found."
elif exit_st == -100:
error_message += " - %s." % (_("discarded download"),)
else:
error_message += " - %s: %s" % (
_("unknown reason"), exit_st,
)
self._entropy.output(
error_message,
importance = 1,
level = "warning",
header = red(" ## ")
)
return exit_st
def _fetch(self):
"""
Execute the source fetch phase.
"""
xterm_title = "%s %s: %s" % (
self._xterm_header,
_("Fetching sources"),
self._meta['atom'],
)
self._entropy.set_title(xterm_title)
down_data = self._meta['download']
down_keys = list(down_data.keys())
d_cache = set()
exit_st = 0
key_cache = [os.path.basename(x) for x in down_keys]
for key in sorted(down_keys):
key_name = os.path.basename(key)
if key_name in d_cache:
continue
# first fine wins
keyboard_interrupt = False
for url in down_data[key]:
file_name = os.path.basename(url)
if self._meta.get('fetch_path'):
dest_file = os.path.join(
self._meta['fetch_path'],
file_name)
else:
dest_file = os.path.join(self._meta['unpackdir'],
file_name)
try:
exit_st = self._fetch_source(url, dest_file)
except KeyboardInterrupt:
keyboard_interrupt = True
break
if exit_st == -100:
keyboard_interrupt = True
break
if exit_st == 0:
d_cache.add(key_name)
break
if keyboard_interrupt:
exit_st = 1
break
key_cache.remove(key_name)
if exit_st != 0 and key_name not in key_cache:
break
exit_st = 0
return exit_st