1658 lines
61 KiB
Python
1658 lines
61 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
|
|
@author: Fabio Erculiani <lxnay@sabayon.org>
|
|
@contact: lxnay@sabayon.org
|
|
@copyright: Fabio Erculiani
|
|
@license: GPL-2
|
|
|
|
B{Entropy Framework QA module}.
|
|
|
|
This module contains various Quality Assurance routines used by Entropy.
|
|
|
|
B{QAInterface} is the main class for QA routines used by Entropy Server
|
|
and Entropy Client such as binary packages health check, dependency
|
|
test, broken or missing library tes.
|
|
|
|
B{ErrorReportInterface} is the HTTP POST based class for Entropy Client
|
|
exceptions (errors) submission.
|
|
|
|
"""
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import tempfile
|
|
import stat
|
|
import codecs
|
|
|
|
from entropy.output import TextInterface
|
|
from entropy.misc import Lifo
|
|
from entropy.const import etpConst, etpSys, const_debug_write, \
|
|
const_debug_write, const_convert_to_rawstring, const_is_python3
|
|
from entropy.output import blue, darkgreen, red, darkred, bold, purple, brown, \
|
|
teal
|
|
from entropy.exceptions import PermissionDenied, SystemDatabaseError
|
|
from entropy.i18n import _
|
|
from entropy.core import EntropyPluginStore
|
|
from entropy.core.settings.base import SystemSettings
|
|
from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase
|
|
|
|
import entropy.tools
|
|
|
|
class QAEntropyRepositoryPlugin(EntropyRepositoryPlugin):
|
|
|
|
def __init__(self, qa_interface, metadata = None):
|
|
"""
|
|
Entropy QA-side repository EntropyRepository Plugin class.
|
|
This class will be instantiated and automatically added to
|
|
EntropyRepository instances generated by Entropy QA.
|
|
|
|
@param qa_interface: Entropy Client interface instance
|
|
@type qa_interface: entropy.qa.QAInterface class
|
|
@param metadata: any dict form metadata map (key => value)
|
|
@type metadata: dict
|
|
"""
|
|
EntropyRepositoryPlugin.__init__(self)
|
|
self._qa = qa_interface
|
|
if metadata is None:
|
|
self._metadata = {}
|
|
else:
|
|
self._metadata = metadata
|
|
|
|
def get_metadata(self):
|
|
return self._metadata
|
|
|
|
def get_id(self):
|
|
return "__qa__"
|
|
|
|
def add_plugin_hook(self, entropy_repository_instance):
|
|
const_debug_write(__name__,
|
|
"QAEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
|
|
self,)
|
|
)
|
|
|
|
out_intf = self._metadata.get('output_interface')
|
|
if out_intf is not None:
|
|
entropy_repository_instance.output = out_intf.output
|
|
entropy_repository_instance.ask_question = out_intf.ask_question
|
|
|
|
return 0
|
|
|
|
class QAInterfacePlugin:
|
|
"""
|
|
Inherit this class to create a QAInterface tests Plugin.
|
|
You need to implement
|
|
|
|
"""
|
|
def get_tests(self):
|
|
"""
|
|
Return a list of callable functions that will be used by QAInterface,
|
|
for tests execution.
|
|
Note: the callable functions signature must be:
|
|
bool function(package_path)
|
|
|
|
@return: list of callable objects
|
|
@rtype: list
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
def get_id(self):
|
|
"""
|
|
Return an identifier string for this Plugin.
|
|
|
|
@return: identifier string
|
|
@rtype: string
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
class QAInterface(TextInterface, EntropyPluginStore):
|
|
|
|
"""
|
|
Entropy QA interface. This class contains all the Entropy
|
|
QA routines used by Entropy Server and Entropy Client.
|
|
|
|
An instance of QAInterface can be easily retrieved from
|
|
entropy.client.interfaces.Client or entropy.server.interfaces.Server
|
|
through an exposed QA() method.
|
|
This is anyway a stand-alone class.
|
|
|
|
@todo: remove non-QA methods
|
|
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""
|
|
QAInterface constructor.
|
|
"""
|
|
EntropyPluginStore.__init__(self)
|
|
self._settings = SystemSettings()
|
|
|
|
def add_plugin(self, plugin):
|
|
"""
|
|
Add a QAInterface plugin to the testing list.
|
|
|
|
@param plugin: QAInterfacePlugin based instance
|
|
@type plugin: QAInterface based instance
|
|
@raise AttributeError: if plugin is not a QAInterfacePlugin instance
|
|
"""
|
|
if not isinstance(plugin, QAInterfacePlugin):
|
|
raise AttributeError("Specify a QAInterfacePlugin based class")
|
|
return EntropyPluginStore.add_plugin(self, plugin.get_id(), plugin)
|
|
|
|
def test_reverse_dependencies_linking(self, entropy_client,
|
|
package_matches):
|
|
"""
|
|
Scan for broken shared objects linking.
|
|
Note: this only works for packages actually installed on the running
|
|
system.
|
|
It is used by Entropy Server during packages injection into database
|
|
to warn about potentially broken packages.
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client based
|
|
instance object
|
|
@param package_matches: list of entropy package matches tuples
|
|
(package id, repo id)
|
|
@type package_matches: list
|
|
@return: True if any breakage is found, otherwise False
|
|
@rtype: bool
|
|
"""
|
|
|
|
scan_msg = blue(_("Searching for broken reverse dependencies"))
|
|
self.output(
|
|
"%s..." % (scan_msg,),
|
|
importance = 1,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
broken = False
|
|
|
|
count = 0
|
|
maxcount = len(package_matches)
|
|
for pkg_id, pkg_repo in package_matches:
|
|
count += 1
|
|
dbconn = entropy_client.open_repository(pkg_repo)
|
|
atom = dbconn.retrieveAtom(pkg_id)
|
|
scan_msg = "%s, %s:" % (
|
|
blue(_("scanning for broken reverse dependencies")),
|
|
darkgreen(atom),
|
|
)
|
|
self.output(
|
|
"[repo:%s] %s" % (
|
|
darkgreen(pkg_repo),
|
|
scan_msg,
|
|
),
|
|
importance = 1,
|
|
level = "info",
|
|
header = blue(" @@ "),
|
|
back = True,
|
|
count = (count, maxcount,)
|
|
)
|
|
mydepends = dbconn.retrieveReverseDependencies(pkg_id)
|
|
if not mydepends:
|
|
continue
|
|
for mydepend in mydepends:
|
|
myatom = dbconn.retrieveAtom(mydepend)
|
|
self.output(
|
|
"[repo:%s] %s => %s" % (
|
|
darkgreen(pkg_repo),
|
|
darkgreen(atom),
|
|
darkred(myatom),
|
|
),
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" @@ "),
|
|
back = True,
|
|
count = (count, maxcount,)
|
|
)
|
|
mycontent = dbconn.retrieveContent(mydepend)
|
|
mybreakages = self._content_test(mycontent)
|
|
if not mybreakages:
|
|
continue
|
|
broken = True
|
|
self.output(
|
|
"[repo:%s] %s %s => %s" % (
|
|
darkgreen(pkg_repo),
|
|
darkgreen(atom),
|
|
darkred(myatom),
|
|
bold(_("broken libraries detected")),
|
|
),
|
|
importance = 1,
|
|
level = "warning",
|
|
header = purple(" @@ "),
|
|
count = (count, maxcount,)
|
|
)
|
|
for mylib in mybreakages:
|
|
self.output(
|
|
"%s %s:" % (
|
|
darkgreen(mylib),
|
|
red(_("needs")),
|
|
),
|
|
importance = 1,
|
|
level = "warning",
|
|
header = brown(" ## ")
|
|
)
|
|
for needed in mybreakages[mylib]:
|
|
self.output(
|
|
"%s" % (
|
|
red(needed),
|
|
),
|
|
importance = 1,
|
|
level = "warning",
|
|
header = purple(" # ")
|
|
)
|
|
return broken
|
|
|
|
def test_missing_dependencies(self, entropy_client, package_matches,
|
|
self_check = False, black_list = None):
|
|
"""
|
|
Scan given package matches looking for missing dependencies (checking
|
|
ELF metadata).
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client based
|
|
instance object
|
|
@param package_matches: list of entropy package matches tuples
|
|
(package id, repo id)
|
|
@type package_matches: list
|
|
@keyword self_check: also introspect inside the complaining package
|
|
(to avoid reporting false positives when circular dependencies
|
|
occur)
|
|
@type self_check: bool
|
|
@keyword black_list: list of dependencies already blacklisted.
|
|
@type black_list: set
|
|
@return: dict of missing dependencies to add, key is package match,
|
|
value is a dict, with library name + ELF class as key, and
|
|
potential missing deps as value
|
|
@rtype: dict
|
|
"""
|
|
|
|
if black_list is None:
|
|
black_list = set()
|
|
|
|
scan_msg = blue(_("Searching for missing Runtime dependencies"))
|
|
self.output(
|
|
"%s..." % (scan_msg,),
|
|
importance = 1,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
scan_msg = blue(_("scanning"))
|
|
count = 0
|
|
maxcount = len(package_matches)
|
|
missing_map = {}
|
|
repos = sorted(entropy_client.repositories())
|
|
|
|
def _warn_soname(soname, elf_class):
|
|
# try to resolve soname
|
|
for needed_repo in repos:
|
|
needed_dbconn = entropy_client.open_repository(needed_repo)
|
|
pkg_ids = needed_dbconn.resolveNeeded(soname,
|
|
elfclass = elf_class)
|
|
if pkg_ids:
|
|
pkg_atoms = sorted((
|
|
needed_dbconn.retrieveKeySlotAggregated(x) for x in \
|
|
pkg_ids))
|
|
pkg_atoms_string = ', '.join(pkg_atoms)
|
|
else:
|
|
pkg_atoms_string = _("no packages")
|
|
self.output(
|
|
"[%s:%s] %s" % (
|
|
brown(needed_repo),
|
|
teal(soname),
|
|
purple(pkg_atoms_string),
|
|
),
|
|
importance = 0,
|
|
level = "info",
|
|
header = brown(" # ")
|
|
)
|
|
|
|
for package_id, repo in package_matches:
|
|
count += 1
|
|
dbconn = entropy_client.open_repository(repo)
|
|
atom = dbconn.retrieveAtom(package_id)
|
|
self.output(
|
|
"[%s] %s: %s" % (
|
|
darkgreen(repo),
|
|
scan_msg,
|
|
darkgreen(atom),
|
|
),
|
|
importance = 1,
|
|
level = "info",
|
|
header = blue(" @@ "),
|
|
back = True,
|
|
count = (count, maxcount,)
|
|
)
|
|
|
|
missing_extended, missing = self._get_missing_rdepends(
|
|
entropy_client, (package_id, repo),
|
|
self_check = self_check)
|
|
|
|
if not missing:
|
|
continue
|
|
|
|
old_missing = missing.copy()
|
|
missing -= black_list
|
|
for item in list(missing_extended.keys()):
|
|
missing_extended[item] -= black_list
|
|
if not missing_extended[item]:
|
|
del missing_extended[item]
|
|
|
|
if missing != old_missing:
|
|
# print big warning to make dev aware at least
|
|
old_missing -= missing
|
|
if old_missing:
|
|
self.output(
|
|
"[%s] %s: %s %s:" % (
|
|
darkgreen(repo),
|
|
darkred("package"),
|
|
darkgreen(atom),
|
|
darkred(_("blacklisted dependencies !!!")),
|
|
),
|
|
importance = 1,
|
|
level = "warning",
|
|
header = bold(" @@ "),
|
|
count = (count, maxcount,)
|
|
)
|
|
for dep in sorted(old_missing):
|
|
self.output(
|
|
"%s" % (bold(dep),),
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # ")
|
|
)
|
|
|
|
if not missing:
|
|
continue
|
|
if not missing_extended:
|
|
continue
|
|
|
|
missing_map[(package_id, repo)] = missing_extended
|
|
|
|
count = 0
|
|
for package_id, repo in package_matches:
|
|
count += 1
|
|
dbconn = entropy_client.open_repository(repo)
|
|
atom = dbconn.retrieveAtom(package_id)
|
|
|
|
# check for untracked missing sonames (using less reliable
|
|
# ldd check, but just warn)
|
|
missing_sonames = self._get_unresolved_sonames(entropy_client,
|
|
(package_id, repo))
|
|
if missing_sonames:
|
|
self.output(
|
|
"[repo:%s] %s: %s %s:" % (
|
|
darkgreen(repo),
|
|
blue("package"),
|
|
darkgreen(atom),
|
|
blue(_("is potentially missing these dependencies")),
|
|
),
|
|
importance = 1,
|
|
level = "info",
|
|
header = red(" @@ "),
|
|
count = (count, maxcount,)
|
|
)
|
|
for executable, sonames in missing_sonames.items():
|
|
elf_class = entropy.tools.read_elf_class(executable)
|
|
self.output(
|
|
"%s (elf class: %s):" % (
|
|
brown(executable),
|
|
teal(str(elf_class)),
|
|
),
|
|
importance = 0,
|
|
level = "info",
|
|
header = purple(" ## ")
|
|
)
|
|
for soname in sonames:
|
|
_warn_soname(soname, elf_class)
|
|
|
|
return missing_map
|
|
|
|
def test_missing_runtime_libraries(self, entropy_client, package_matches,
|
|
base_repository_id = None, excluded_libraries = None, silent = False):
|
|
"""
|
|
Use collected packages ELF metadata (retrieveNeeded(),
|
|
resolveNeeded()) to look for potentially missing
|
|
shared libraries. This is very handy in case of library breakages
|
|
across multiple server-side repositories.
|
|
For example: you bump libfoo, which provides new library, the SPM forces
|
|
you to rebuild foouser, which uses libfoo. You put both into a testing
|
|
repository but then you only move foouser to the base repository without
|
|
realizing the potential breakage users could run into.
|
|
However, since there can be false positives, this routine cannot block
|
|
you from doing this mistakes.
|
|
Please note that the base repository is the first listed in server.conf
|
|
and will always be considered as self-contained, meaning that all the
|
|
dependencies and sonames must be available within the same.
|
|
The code first tries to resolve the soname inside the same repository,
|
|
then falls back to other ones, if any.
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client based
|
|
instance object
|
|
@param package_matches: list of Entropy package matches
|
|
@type package_matches: list
|
|
@return: list (set) of tuples of length 2 of missing sonames
|
|
[(soname, elfclass), ...]
|
|
@keyword base_repository_id: repository identifier supposed to be
|
|
used as main (or base) repository
|
|
@type base_repository_id: string
|
|
@keyword excluded_libraries: list of libraries that should not be
|
|
verified (perhaps, libGL.so.1 in Gentoo/Sabayon)
|
|
@type excluded_libraries: set
|
|
@keyword silent: no output is print to stdout
|
|
@type silent: bool
|
|
@rtype: set
|
|
"""
|
|
missing_sonames = set()
|
|
if excluded_libraries is None:
|
|
excluded_libraries = set()
|
|
else:
|
|
excluded_libraries = set(excluded_libraries)
|
|
|
|
# update content taken from brokenlinksmask.conf
|
|
excluded_libraries.update(self._settings['broken_links_mask'])
|
|
|
|
def _resolve_needed_content_fallback(repo, pkg_id, library):
|
|
# at this point, perhaps the library is self contained in the
|
|
# package itself, and in non-standard path. so, let's see if
|
|
# it's in the content metadata before giving up.
|
|
# Since this is expensive, try to first match it in
|
|
# pre-hashed metadata (via resolveNeeded())
|
|
file_names = set((os.path.basename(x) for x in \
|
|
repo.retrieveContent(pkg_id)))
|
|
# NOTE: we assume that ELF class is the same as the one requested,
|
|
# because we're not allowed to poll the live system.
|
|
# perhaps in future we could collect more metadata.
|
|
return library in file_names
|
|
|
|
def _resolve_needed(repo, pkg_id, library, elfclass, multi_repo):
|
|
|
|
resolved_needed = repo.resolveNeeded(library,
|
|
elfclass = elfclass)
|
|
if resolved_needed:
|
|
return True
|
|
|
|
if not resolved_needed and not multi_repo:
|
|
# sorry, can't find it, this is the base repo already
|
|
return _resolve_needed_content_fallback(repo, pkg_id, library)
|
|
|
|
for repo_id in entropy_client.repositories():
|
|
if repo_id == repo.repository_id():
|
|
# already searched here
|
|
continue
|
|
other_repo = entropy_client.open_repository(repo_id)
|
|
resolved_needed = other_repo.resolveNeeded(library,
|
|
elfclass = elfclass)
|
|
if resolved_needed:
|
|
# found !
|
|
return True
|
|
|
|
return _resolve_needed_content_fallback(repo, pkg_id, library)
|
|
|
|
def _look_for_available_soname(current_repo_id, library, elfclass):
|
|
where = frozenset()
|
|
for repo_id in entropy_client.repositories():
|
|
if repo_id == current_repo_id:
|
|
# already searched here
|
|
continue
|
|
other_repo = entropy_client.open_repository(repo_id)
|
|
resolved_needed = other_repo.resolveNeeded(library,
|
|
elfclass = elfclass, extended = True)
|
|
if not resolved_needed:
|
|
continue
|
|
where = [(x, repo_id, y) for x, y in resolved_needed]
|
|
break
|
|
return where
|
|
|
|
count = 0
|
|
maxcount = len(package_matches)
|
|
for package_id, repository_id in package_matches:
|
|
|
|
count += 1
|
|
repo = entropy_client.open_repository(repository_id)
|
|
atom = repo.retrieveAtom(package_id)
|
|
|
|
if not silent:
|
|
scan_msg = "%s, %s:" % (
|
|
blue(_("determining missing libraries")),
|
|
darkgreen(atom),
|
|
)
|
|
self.output(
|
|
"[repo:%s] %s" % (
|
|
darkgreen(repository_id),
|
|
scan_msg,
|
|
),
|
|
importance = 1,
|
|
level = "info",
|
|
header = blue(" @@ "),
|
|
back = True,
|
|
count = (count, maxcount,)
|
|
)
|
|
|
|
is_base_repo = repository_id == base_repository_id
|
|
|
|
# list of (needed, elfclass)
|
|
needed = repo.retrieveNeeded(package_id, extended = True)
|
|
for library, elfclass in needed:
|
|
if library in excluded_libraries:
|
|
continue
|
|
resolved = _resolve_needed(repo, package_id, library, elfclass,
|
|
not is_base_repo)
|
|
if not resolved:
|
|
missing_sonames.add((library, elfclass))
|
|
|
|
if not silent:
|
|
self.output(
|
|
"[%s] %s %s: %s, class: %s" % (
|
|
purple(repository_id),
|
|
teal(atom),
|
|
purple(_("requires")),
|
|
brown(library),
|
|
darkgreen(str(elfclass)),
|
|
),
|
|
importance = 1,
|
|
level = "warning",
|
|
header = " ",
|
|
)
|
|
# perhaps the lib is in some other repo?
|
|
resolved_needed = _look_for_available_soname(
|
|
repository_id, library, elfclass)
|
|
for pkg_id, repo_id, path in resolved_needed:
|
|
atom = entropy_client.open_repository(
|
|
repo_id).retrieveAtom(pkg_id)
|
|
if not silent:
|
|
self.output(
|
|
"%s: %s, %s" % (
|
|
brown(_("provided by")),
|
|
darkgreen(atom),
|
|
path,
|
|
),
|
|
importance = 0,
|
|
level = "warning",
|
|
header = darkred(" "),
|
|
)
|
|
if not resolved_needed and not silent:
|
|
self.output(
|
|
"%s: %s" % (
|
|
brown(_("provided by")),
|
|
darkgreen(_("no packages")),
|
|
),
|
|
importance = 0,
|
|
level = "warning",
|
|
header = darkred(" "),
|
|
)
|
|
|
|
if not silent:
|
|
if missing_sonames:
|
|
self.output("", level = "info", importance = 0)
|
|
else:
|
|
self.output(
|
|
darkgreen(_("no missing runtime libraries found")),
|
|
level = "info",
|
|
importance = 1
|
|
)
|
|
|
|
return missing_sonames
|
|
|
|
def test_shared_objects(self, entropy_repository, broken_symbols = False,
|
|
task_bombing_func = None, self_dir_check = True,
|
|
dump_results_to_file = False):
|
|
|
|
"""
|
|
Scan system looking for broken shared object ELF library dependencies.
|
|
|
|
@param entropy_repository: entropy.db.EntropyRepository instance
|
|
@type entropy_repository: entropy.db.EntropyRepository instance
|
|
@keyword broken_symbols: enable or disable broken symbols extra check.
|
|
Symbols which are going to be checked have to be listed into:
|
|
/etc/entropy/brokensyms.conf (regexp supported).
|
|
@type broken_symbols: bool
|
|
@keyword task_bombing_func: callable that will be called on every
|
|
scan iteration to allow external routines to cleanly stop the
|
|
execution of this function.
|
|
@type task_bombing_func: callable
|
|
@keyword dump_results_to_file: dump test results to files (printed)
|
|
@type dump_results_to_file: bool
|
|
@return: tuple of length 3, composed by (1) a dict of matched packages,
|
|
(2) a list (set) of broken ELF objects and (3) the execution status
|
|
(int, 0 means success).
|
|
@rtype: tuple
|
|
"""
|
|
|
|
self.output(
|
|
blue(_("Libraries test")),
|
|
importance = 2,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
syms_list_path = None
|
|
files_list_path = None
|
|
if dump_results_to_file:
|
|
|
|
tmp_dir = tempfile.mkdtemp()
|
|
syms_list_path = os.path.join(tmp_dir, "libtest_syms.txt")
|
|
files_list_path = os.path.join(tmp_dir, "libtest_files.txt")
|
|
|
|
dmp_data = [
|
|
(_("Broken symbols packages list"), syms_list_path,),
|
|
(_("Broken executables list"), files_list_path,),
|
|
]
|
|
mytxt = "%s:" % (purple(_("Dumping results into these files")),)
|
|
self.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = blue(" @@ ")
|
|
)
|
|
for txt, path in dmp_data:
|
|
mytxt = "%s: %s" % (blue(txt), path,)
|
|
self.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = darkgreen(" ## ")
|
|
)
|
|
|
|
myroot = etpConst['systemroot'] + os.path.sep
|
|
if not etpConst['systemroot']:
|
|
myroot = os.path.sep
|
|
|
|
# run ldconfig first
|
|
subprocess.call("ldconfig -r '%s' &> /dev/null" % (myroot,),
|
|
shell = True)
|
|
|
|
reverse_symlink_map = self._settings['system_rev_symlinks']
|
|
broken_syms_list = self._settings['broken_syms']
|
|
broken_libs_mask = self._settings['broken_libs_mask']
|
|
# make possible to pass a mask list from env
|
|
env_broken_libs_mask = os.getenv("ETP_BROKEN_LIBS_MASK", "")
|
|
if env_broken_libs_mask.strip():
|
|
broken_libs_mask += env_broken_libs_mask.strip().split()
|
|
|
|
import re
|
|
|
|
broken_syms_list_regexp = []
|
|
for broken_sym in broken_syms_list:
|
|
if broken_sym.startswith("r:"):
|
|
broken_sym = broken_sym[2:]
|
|
if not broken_sym:
|
|
continue
|
|
reg_sym = re.compile(broken_sym)
|
|
else:
|
|
reg_sym = re.compile(re.escape(broken_sym))
|
|
broken_syms_list_regexp.append(reg_sym)
|
|
|
|
broken_libs_mask_regexp = []
|
|
broken_libs_paths_mask_regexp = []
|
|
for broken_lib in broken_libs_mask:
|
|
if broken_lib.startswith("r:"):
|
|
broken_lib = broken_lib[2:]
|
|
if not broken_lib:
|
|
continue
|
|
reg_lib = re.compile(broken_lib)
|
|
else:
|
|
reg_lib = re.compile(re.escape(broken_lib))
|
|
if os.path.sep in broken_lib:
|
|
# it's a path, not a lib name
|
|
broken_libs_paths_mask_regexp.append(reg_lib)
|
|
else:
|
|
broken_libs_mask_regexp.append(reg_lib)
|
|
|
|
ldpaths = entropy.tools.collect_linker_paths()
|
|
ldpaths += [x for x in entropy.tools.collect_paths() if \
|
|
x not in ldpaths]
|
|
|
|
# some crappy packages put shit here too
|
|
ldpaths += [x for x in self._settings['extra_ldpaths'] if \
|
|
x not in ldpaths]
|
|
|
|
# remove duplicated dirs (due to symlinks) to speed up scanning
|
|
for real_dir in list(reverse_symlink_map.keys()):
|
|
syms = reverse_symlink_map[real_dir]
|
|
for sym in syms:
|
|
if sym in ldpaths:
|
|
while (real_dir in ldpaths):
|
|
ldpaths.remove(real_dir)
|
|
self.output(
|
|
"%s: %s, %s: %s" % (
|
|
brown(_("discarding directory")),
|
|
purple(real_dir),
|
|
brown(_("because it's symlinked on")),
|
|
purple(sym),
|
|
),
|
|
importance = 0,
|
|
level = "info",
|
|
header = darkgreen(" @@ ")
|
|
)
|
|
break
|
|
|
|
executables = set()
|
|
total = len(ldpaths)
|
|
count = 0
|
|
sys_root_len = len(etpConst['systemroot'])
|
|
|
|
def _are_elfs(dt):
|
|
currentdir, subdirs, files = dt
|
|
|
|
def _is_elf(item):
|
|
filepath = os.path.join(currentdir, item)
|
|
try:
|
|
st = os.stat(filepath)
|
|
except (OSError, IOError):
|
|
return None
|
|
|
|
if not stat.S_ISREG(st.st_mode):
|
|
return None
|
|
# shared libraries must be always executable
|
|
if not (stat.S_IMODE(st.st_mode) & stat.S_IXUSR):
|
|
return None
|
|
|
|
if not entropy.tools.is_elf_file(filepath):
|
|
return None
|
|
return filepath[sys_root_len:]
|
|
|
|
return (x for x in map(_is_elf, files) if x is not None)
|
|
|
|
for ldpath in sorted(ldpaths):
|
|
|
|
if hasattr(task_bombing_func, '__call__'):
|
|
task_bombing_func()
|
|
count += 1
|
|
self.output(
|
|
blue("Tree: ")+red(etpConst['systemroot'] + ldpath),
|
|
importance = 0,
|
|
level = "info",
|
|
count = (count, total),
|
|
back = True,
|
|
percent = True,
|
|
header = " "
|
|
)
|
|
try:
|
|
ldpath = ldpath.encode('utf-8')
|
|
except (UnicodeEncodeError,):
|
|
ldpath = ldpath.encode(sys.getfilesystemencoding())
|
|
mywalk_iter = os.walk(etpConst['systemroot'] + ldpath)
|
|
|
|
for x in map(_are_elfs, mywalk_iter):
|
|
executables.update(x)
|
|
|
|
self.output(
|
|
blue(_("Collecting broken executables")),
|
|
importance = 2,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
t = red(_("Attention")) + ": " + \
|
|
blue(_("don't worry about libraries that are shown here but not later."))
|
|
self.output(
|
|
t,
|
|
importance = 1,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
|
|
enc = etpConst['conf_encoding']
|
|
syms_list_f = None
|
|
if syms_list_path:
|
|
syms_list_f = codecs.open(syms_list_path, "w", encoding=enc)
|
|
|
|
files_list_f = None
|
|
if files_list_path:
|
|
files_list_f = codecs.open(files_list_path, "w", encoding=enc)
|
|
|
|
plain_brokenexecs = set()
|
|
total = len(executables)
|
|
count = 0
|
|
scan_txt = blue("%s ..." % (_("Scanning libraries"),))
|
|
for executable in executables:
|
|
|
|
# task bombing hook
|
|
if hasattr(task_bombing_func, '__call__'):
|
|
task_bombing_func()
|
|
|
|
count += 1
|
|
if (count % 10 == 0) or (count == total) or (count == 1):
|
|
self.output(
|
|
scan_txt,
|
|
importance = 0,
|
|
level = "info",
|
|
count = (count, total),
|
|
back = True,
|
|
percent = True,
|
|
header = " "
|
|
)
|
|
|
|
# filter broken paths
|
|
# there are paths known to be broken and must be
|
|
# excluded to avoid noisy false positives
|
|
exec_match = False
|
|
for reg_path in broken_libs_paths_mask_regexp:
|
|
if reg_path.match(executable):
|
|
exec_match = True
|
|
break
|
|
if exec_match:
|
|
continue
|
|
|
|
real_exec_path = etpConst['systemroot'] + executable
|
|
|
|
myelfs = entropy.tools.read_elf_dynamic_libraries(
|
|
real_exec_path)
|
|
|
|
mylibs = set()
|
|
for mylib in myelfs:
|
|
lib_path = entropy.tools.resolve_dynamic_library(mylib,
|
|
executable)
|
|
if not lib_path:
|
|
mylibs.add(mylib)
|
|
|
|
# filter broken libraries
|
|
if mylibs:
|
|
|
|
mylib_filter = set()
|
|
for mylib in mylibs:
|
|
mylib_matched = False
|
|
for reg_lib in broken_libs_mask_regexp:
|
|
if reg_lib.match(mylib):
|
|
mylib_matched = True
|
|
break
|
|
|
|
if mylib_matched: # filter out
|
|
mylib_filter.add(mylib)
|
|
|
|
elif self_dir_check:
|
|
# check inside the same directory of the failing ELF
|
|
# obviously, we're looking for another ELF object
|
|
my_real_exec_dir = os.path.dirname(real_exec_path)
|
|
mylib_guess = os.path.join(my_real_exec_dir, mylib)
|
|
if os.access(mylib_guess, os.R_OK) and \
|
|
os.path.isfile(mylib_guess):
|
|
if entropy.tools.is_elf_file(mylib_guess):
|
|
# we have found the missing library,
|
|
# which wasn't in LDPATH, booooo @ package
|
|
# developers !! boooo!
|
|
mylib_filter.add(mylib)
|
|
|
|
mylibs -= mylib_filter
|
|
|
|
broken_sym_found = set()
|
|
if broken_symbols and not mylibs:
|
|
|
|
read_broken_syms = entropy.tools.read_elf_broken_symbols(
|
|
real_exec_path)
|
|
my_broken_syms = set()
|
|
for read_broken_sym in read_broken_syms:
|
|
for reg_sym in broken_syms_list_regexp:
|
|
if reg_sym.match(read_broken_sym):
|
|
my_broken_syms.add(read_broken_sym)
|
|
break
|
|
broken_sym_found.update(my_broken_syms)
|
|
|
|
if not (mylibs or broken_sym_found):
|
|
continue
|
|
|
|
if mylibs:
|
|
|
|
if files_list_f:
|
|
files_list_f.write(executable + "\n")
|
|
|
|
alllibs = blue(' :: ').join(sorted(mylibs))
|
|
self.output(
|
|
red(real_exec_path)+" [ "+alllibs+" ]",
|
|
importance = 1,
|
|
level = "info",
|
|
percent = True,
|
|
count = (count, total),
|
|
header = " "
|
|
)
|
|
elif broken_sym_found:
|
|
|
|
allsyms = darkred(' :: ').join([brown(x) for x in \
|
|
broken_sym_found])
|
|
if len(allsyms) > 50:
|
|
allsyms = brown(_('various broken symbols'))
|
|
|
|
if syms_list_f and broken_sym_found:
|
|
syms_list_f.write("%s => %s\n" % (real_exec_path,
|
|
sorted(broken_sym_found),))
|
|
|
|
self.output(
|
|
red(real_exec_path)+" { "+allsyms+" }",
|
|
importance = 1,
|
|
level = "info",
|
|
percent = True,
|
|
count = (count, total),
|
|
header = " "
|
|
)
|
|
|
|
plain_brokenexecs.add(executable)
|
|
|
|
# close open files
|
|
if syms_list_f:
|
|
syms_list_f.flush()
|
|
syms_list_f.close()
|
|
if files_list_f:
|
|
files_list_f.flush()
|
|
files_list_f.close()
|
|
|
|
del executables
|
|
pkgs_matched = {}
|
|
|
|
if not etpSys['serverside']:
|
|
|
|
# we are client side
|
|
# this is hackish and must be fixed sooner or later
|
|
# but for now, it works
|
|
# Client class is singleton and is surely already
|
|
# loaded when we get here
|
|
from entropy.client.interfaces import Client
|
|
client = Client()
|
|
|
|
self.output(
|
|
blue(_("Matching broken libraries/executables")),
|
|
importance = 1,
|
|
level = "info",
|
|
header = red(" @@ ")
|
|
)
|
|
matched = set()
|
|
for brokenlib in plain_brokenexecs:
|
|
# test with /usr/lib
|
|
idpackages = entropy_repository.searchBelongs(brokenlib)
|
|
if not idpackages:
|
|
# try with realpath
|
|
# on multilib systems this resolves to /usr/lib64
|
|
# which makes searchBelongs() happy
|
|
idpackages = entropy_repository.searchBelongs(
|
|
os.path.realpath(brokenlib))
|
|
|
|
for idpackage in idpackages:
|
|
|
|
key, slot = entropy_repository.retrieveKeySlot(idpackage)
|
|
mymatch = client.atom_match(key, match_slot = slot)
|
|
if mymatch[0] == -1:
|
|
matched.add(brokenlib)
|
|
continue
|
|
|
|
obj = pkgs_matched.setdefault(brokenlib, set())
|
|
|
|
obj.add(mymatch)
|
|
matched.add(brokenlib)
|
|
|
|
plain_brokenexecs -= matched
|
|
|
|
return pkgs_matched, plain_brokenexecs, 0
|
|
|
|
def _content_test(self, mycontent):
|
|
"""
|
|
Test whether the given list of files contain files
|
|
with broken shared object links.
|
|
|
|
@param mycontent: list of file paths
|
|
@type mycontent: list or set
|
|
@return: dict containing a map between file path
|
|
and list (set) of broken libraries (just the library name,
|
|
the same that is contained inside ELF metadata)
|
|
@rtype: dict
|
|
"""
|
|
def is_contained(needed, content):
|
|
for item in content:
|
|
if os.path.basename(item) == needed:
|
|
return True
|
|
return False
|
|
|
|
mylibs = {}
|
|
for myfile in mycontent:
|
|
myfile = const_convert_to_rawstring(myfile)
|
|
if not os.access(myfile, os.R_OK):
|
|
continue
|
|
if not os.path.isfile(myfile):
|
|
continue
|
|
if not entropy.tools.is_elf_file(myfile):
|
|
continue
|
|
mylibs[myfile] = entropy.tools.read_elf_dynamic_libraries(
|
|
myfile)
|
|
|
|
broken_libs = {}
|
|
for mylib in mylibs:
|
|
for myneeded in mylibs[mylib]:
|
|
# is this inside myself ?
|
|
if is_contained(myneeded, mycontent):
|
|
continue
|
|
found = entropy.tools.resolve_dynamic_library(myneeded,
|
|
mylib)
|
|
if found:
|
|
continue
|
|
if mylib not in broken_libs:
|
|
broken_libs[mylib] = set()
|
|
broken_libs[mylib].add(myneeded)
|
|
|
|
return broken_libs
|
|
|
|
def _get_unresolved_sonames(self, entropy_client, package_match,
|
|
content_root = None):
|
|
"""
|
|
This runtime dependencies function uses ldd to determine what missing
|
|
runtime dependencies a package may expose. ldd implicitly expands the
|
|
list (graph) of shared objects connected with an ELF using ld.so.conf
|
|
and LD* env vars information. This is useful to warn about pontentially
|
|
broken packages, containing untracked runtime dependencies.
|
|
Unfortunately, this is (as opposed to _get_missing_rdepends) not rocket
|
|
science and cannot be used automatically add dependencies to
|
|
a specific package. It's not even 100% reliable because the linking
|
|
depends on the environment (ld.so.conf and LD*). So, just use it to
|
|
warn developers or users.
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client based
|
|
instance object
|
|
@param package_match: Entropy package match: (package id, repo id).
|
|
@type package_match: tuple
|
|
@keyword content_root: alternative root directory containing content
|
|
@return: dictionary composed by package ELF file path as key, and
|
|
list (set) of missing sonames as value.
|
|
@rtype: dict
|
|
"""
|
|
if content_root is None:
|
|
content_root = etpConst['systemroot']
|
|
elif not content_root.endswith(os.path.sep):
|
|
content_root += os.path.sep
|
|
|
|
def is_valid_elf(path):
|
|
if not os.path.lexists(path):
|
|
return False
|
|
if not os.path.isfile(path):
|
|
return False
|
|
if os.path.islink(path):
|
|
return False
|
|
if not os.access(path, os.X_OK | os.R_OK):
|
|
return False
|
|
if not entropy.tools.is_elf_file(path):
|
|
return False
|
|
return True
|
|
|
|
pkg_matches = self.get_deep_dependency_list(entropy_client,
|
|
package_match)
|
|
|
|
all_content = set()
|
|
for pkg_id, pkg_repo in pkg_matches:
|
|
pkg_dbconn = entropy_client.open_repository(pkg_repo)
|
|
all_content.update(pkg_dbconn.retrieveContent(pkg_id))
|
|
|
|
package_id, repo_id = package_match
|
|
entropy_repository = entropy_client.open_repository(repo_id)
|
|
package_content = entropy_repository.retrieveContent(package_id)
|
|
all_content.update(package_content)
|
|
|
|
resolve_cache = {}
|
|
unresolved_sonames = {}
|
|
content = [os.path.normpath(content_root + x) for x in package_content]
|
|
content_dirs = set((x for x in content if os.path.isdir(x)))
|
|
elf_files = filter(is_valid_elf, content)
|
|
|
|
def soname_in_package_content(soname):
|
|
for content_dir in content_dirs:
|
|
if is_valid_elf(os.path.join(content_dir, soname)):
|
|
return True
|
|
return False
|
|
|
|
for elf_file in elf_files:
|
|
sonames = entropy.tools.read_elf_real_dynamic_libraries(elf_file)
|
|
for soname in sonames:
|
|
resolved_soname_path = resolve_cache.setdefault(soname,
|
|
entropy.tools.resolve_dynamic_library(soname, elf_file))
|
|
if resolved_soname_path is None:
|
|
# library not found on system
|
|
# maybe it's into our package?
|
|
if not soname_in_package_content(soname):
|
|
obj = unresolved_sonames.setdefault(elf_file, set())
|
|
obj.add(soname)
|
|
else:
|
|
|
|
# fixup library dir, multilib systems
|
|
real_dir = os.path.realpath(os.path.dirname(
|
|
resolved_soname_path))
|
|
resolved_soname_path = os.path.join(real_dir,
|
|
os.path.basename(resolved_soname_path))
|
|
|
|
# library found on system, need to check if it's in our
|
|
# package dependencies, "all_content"
|
|
if resolved_soname_path not in all_content:
|
|
obj = unresolved_sonames.setdefault(elf_file, set())
|
|
obj.add(soname)
|
|
|
|
return unresolved_sonames
|
|
|
|
def _get_missing_rdepends(self, entropy_client, package_match,
|
|
self_check = False):
|
|
"""
|
|
Service method able to determine whether dependencies are missing
|
|
on the given idpackage (belonging to the given
|
|
entropy.db.EntropyRepository "dbconn" argument) using shared objects
|
|
linking information between packages.
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client base
|
|
class object
|
|
@param package_match: Entropy package match: (package id, repo id).
|
|
@type package_match: tuple
|
|
@keyword self_check: also check inside the given package
|
|
(package id) itself
|
|
@type self_check: bool
|
|
@return: tuple of length 2, composed by a dictionary with the
|
|
following structure:
|
|
{('KEY', 'SLOT': set([list of missing deps for the given key])}
|
|
and a "plain" list (set) of missing dependencies
|
|
set([list of missing dependencies])
|
|
@rtype: tuple
|
|
"""
|
|
package_id, repo = package_match
|
|
repos = list(entropy_client.repositories())
|
|
# max priority to package_match repo
|
|
repos.remove(repo)
|
|
repos.insert(0, repo)
|
|
dbconn = entropy_client.open_repository(repo)
|
|
|
|
rdepends = {}
|
|
rdepends_plain = set()
|
|
neededs = dbconn.retrieveNeeded(package_id, extended = True)
|
|
ldpaths = entropy.tools.collect_linker_paths()
|
|
deps_content = set()
|
|
dependencies = self.get_deep_dependency_list(entropy_client,
|
|
package_match, atoms = True)
|
|
scope_cache = set()
|
|
|
|
def is_soname_available(soname):
|
|
for p_repo in repos:
|
|
p_dbconn = entropy_client.open_repository(p_repo)
|
|
if p_dbconn.isNeededAvailable(soname) > 0:
|
|
return True
|
|
return False
|
|
|
|
def resolve_soname(soname, elfclass):
|
|
for p_repo in repos:
|
|
p_dbconn = entropy_client.open_repository(p_repo)
|
|
data_solved = p_dbconn.resolveNeeded(soname,
|
|
elfclass = elfclass, extended = True)
|
|
if data_solved:
|
|
# found !
|
|
return [(pkg_id, p_repo, path) for pkg_id, path in \
|
|
data_solved]
|
|
# nothing found
|
|
return []
|
|
|
|
def update_depscontent(mycontent):
|
|
return set( \
|
|
[x for x in mycontent if os.path.dirname(x) in ldpaths \
|
|
and is_soname_available(os.path.basename(x))])
|
|
|
|
def is_in_content(myneeded, content):
|
|
for item in content:
|
|
item = os.path.basename(item)
|
|
if myneeded == item:
|
|
return True
|
|
return False
|
|
|
|
def is_system_pkg(pkg_id, repo_db):
|
|
if repo_db.isSystemPackage(pkg_id):
|
|
return True
|
|
visited = set()
|
|
reverse_deps = repo_db.retrieveReverseDependencies(pkg_id,
|
|
key_slot = True)
|
|
# with virtual packages, it can happen that system packages
|
|
# are not directly marked as such. so, check direct inverse deps
|
|
# and see if we find one
|
|
for rev_pkg_key, rev_pkg_slot in reverse_deps:
|
|
rev_pkg_id, rev_repo_id = entropy_client.atom_match(rev_pkg_key,
|
|
match_slot = rev_pkg_slot)
|
|
if rev_pkg_id == -1:
|
|
# can't find
|
|
continue
|
|
rev_repo_db = entropy_client.open_repository(rev_repo_id)
|
|
if rev_repo_db.isSystemPackage(rev_pkg_id):
|
|
return True
|
|
return False
|
|
|
|
for dependency in dependencies:
|
|
pkg_id, repo_id = entropy_client.atom_match(dependency)
|
|
if pkg_id != -1:
|
|
mydbconn = entropy_client.open_repository(repo_id)
|
|
deps_content |= update_depscontent(
|
|
mydbconn.retrieveContent(pkg_id))
|
|
key, slot = mydbconn.retrieveKeySlot(pkg_id)
|
|
scope_cache.add((key, slot))
|
|
|
|
key, slot = dbconn.retrieveKeySlot(package_id)
|
|
pkg_content = dbconn.retrieveContent(package_id)
|
|
deps_content |= update_depscontent(pkg_content)
|
|
scope_cache.add((key, slot))
|
|
|
|
packages_cache = set()
|
|
package_map = {}
|
|
package_map_reverse = {}
|
|
|
|
for needed, elfclass in neededs:
|
|
data_solved = resolve_soname(needed, elfclass)
|
|
data_size = len(data_solved)
|
|
data_solved = [(pkg_id, pkg_repo, path) for \
|
|
pkg_id, pkg_repo, path in data_solved if (pkg_id, pkg_repo)
|
|
not in packages_cache]
|
|
if not data_solved or (data_size != len(data_solved)):
|
|
continue
|
|
|
|
if self_check:
|
|
if is_in_content(needed, pkg_content):
|
|
continue
|
|
|
|
found = False
|
|
for pkg_id, pkg_repo, path in data_solved:
|
|
if path in deps_content:
|
|
found = True
|
|
break
|
|
|
|
if not found:
|
|
|
|
for pkg_id, pkg_repo, path in data_solved:
|
|
pkg_dbconn = entropy_client.open_repository(pkg_repo)
|
|
key, slot = pkg_dbconn.retrieveKeySlot(pkg_id)
|
|
if (key, slot) in scope_cache:
|
|
continue
|
|
system_pkg = is_system_pkg(pkg_id, pkg_dbconn)
|
|
if system_pkg:
|
|
# ignore system package missing dep if this is a
|
|
# system package, it means that further missing
|
|
# deps detection will be anyway wrong.
|
|
# !!! this fixes improper libgcc_s.so binding with
|
|
# gnat-gcc on amd64.
|
|
# but in general, system packages are implicit deps
|
|
break
|
|
|
|
map_key = (needed, elfclass)
|
|
keyslot = "%s%s%s" % (key, etpConst['entropyslotprefix'],
|
|
slot,)
|
|
|
|
obj = rdepends.setdefault(map_key, set())
|
|
obj.add(keyslot)
|
|
|
|
obj = package_map_reverse.setdefault(keyslot, set())
|
|
obj.add(map_key)
|
|
|
|
pkg_match = (pkg_id, pkg_repo)
|
|
obj = package_map.setdefault(map_key, set())
|
|
obj.add(pkg_match)
|
|
|
|
rdepends_plain.add(keyslot)
|
|
packages_cache.add(pkg_match)
|
|
|
|
# now reduce dependencies
|
|
|
|
r_deplist = set()
|
|
for key, package_matches in package_map.items():
|
|
for package_match in package_matches:
|
|
pkg_id, pkg_repo = package_match
|
|
pkg_dbconn = entropy_client.open_repository(pkg_repo)
|
|
r_deplist |= pkg_dbconn.retrieveDependencies(pkg_id,
|
|
exclude_deptypes = \
|
|
[etpConst['dependency_type_ids']['bdepend_id']])
|
|
|
|
r_keyslots = set()
|
|
for r_dep in r_deplist:
|
|
pkg_id, pkg_repo = entropy_client.atom_match(r_dep)
|
|
if pkg_id == -1:
|
|
continue
|
|
pkg_dbconn = entropy_client.open_repository(pkg_repo)
|
|
keyslot = pkg_dbconn.retrieveKeySlotAggregated(pkg_id)
|
|
if keyslot in rdepends_plain:
|
|
r_keyslots.add(keyslot)
|
|
|
|
rdepends_plain -= r_keyslots
|
|
for r_keyslot in r_keyslots:
|
|
keys = [x for x in package_map_reverse.get(keyslot, set()) if \
|
|
x in rdepends]
|
|
for key in keys:
|
|
rdepends[key].discard(r_keyslot)
|
|
if not rdepends[key]:
|
|
del rdepends[key]
|
|
|
|
return rdepends, rdepends_plain
|
|
|
|
def get_deep_dependency_list(self, entropy_client, package_match,
|
|
atoms = False, match_repo = None):
|
|
"""
|
|
Service method which returns a complete, expanded list of dependencies.
|
|
|
|
NOTE: this method will only return dependencies that are NOT build
|
|
dependencies.
|
|
|
|
@param entropy_client: Entropy Client instance
|
|
@type entropy_client: entropy.client.interfaces.client.Client based
|
|
instance object
|
|
@param package_match: Entropy package match (package id, repo id)
|
|
@type package_match: tuple
|
|
@keyword atoms: !! return type modifier !! , make method returning
|
|
a list of atom strings instead of list of db match tuples.
|
|
@type atoms: bool
|
|
@keyword match_repo: list of repositories to match, if None,
|
|
all repositories will be used for matching.
|
|
@param match_repo: list
|
|
@return: list of package matches (of dependencies) or plain dependency
|
|
string list if "atom" is True
|
|
@rtype: set
|
|
"""
|
|
package_id, repo = package_match
|
|
# WARNING: in order to avoid code duplication, an undocumented feature
|
|
# has been implemented, which is required by
|
|
# repository-webinstall-generator
|
|
single_repo_mode = False
|
|
if isinstance(repo, EntropyRepositoryBase):
|
|
dbconn = repo
|
|
single_repo_mode = True
|
|
if entropy_client is not None:
|
|
# written in stone here, the day that the code has to be split
|
|
# this will be even more clear.
|
|
raise AttributeError("in this case, entropy_client == None!")
|
|
else:
|
|
dbconn = entropy_client.open_repository(repo)
|
|
|
|
excluded_dep_types = [etpConst['dependency_type_ids']['bdepend_id']]
|
|
mybuffer = Lifo()
|
|
depcache = set()
|
|
matchcache = set()
|
|
mydeps = dbconn.retrieveDependencies(package_id,
|
|
exclude_deptypes = excluded_dep_types)
|
|
for mydep in mydeps:
|
|
mybuffer.push(mydep)
|
|
try:
|
|
mydep = mybuffer.pop()
|
|
except ValueError:
|
|
mydep = None # stack empty
|
|
|
|
result = set()
|
|
while mydep:
|
|
|
|
if mydep in depcache:
|
|
try:
|
|
mydep = mybuffer.pop()
|
|
except ValueError:
|
|
break # stack empty
|
|
continue
|
|
|
|
if single_repo_mode:
|
|
pkg_id, pkg_rc = dbconn.atomMatch(mydep)
|
|
pkg_repo = dbconn.repository_id()
|
|
else:
|
|
pkg_id, pkg_repo = entropy_client.atom_match(mydep,
|
|
match_repo = match_repo)
|
|
|
|
match = (pkg_id, pkg_repo)
|
|
if match in matchcache:
|
|
# neeeext !
|
|
try:
|
|
mydep = mybuffer.pop()
|
|
except ValueError:
|
|
break # stack empty
|
|
continue
|
|
matchcache.add(match)
|
|
|
|
if atoms:
|
|
result.add(mydep)
|
|
|
|
if pkg_id != -1:
|
|
if not atoms:
|
|
result.add(match)
|
|
if single_repo_mode:
|
|
pkg_dbconn = dbconn
|
|
else:
|
|
pkg_dbconn = entropy_client.open_repository(pkg_repo)
|
|
owndeps = pkg_dbconn.retrieveDependencies(pkg_id,
|
|
exclude_deptypes = excluded_dep_types)
|
|
for owndep in owndeps:
|
|
mybuffer.push(owndep)
|
|
|
|
depcache.add(mydep)
|
|
try:
|
|
mydep = mybuffer.pop()
|
|
except ValueError:
|
|
break # stack empty
|
|
|
|
return result
|
|
|
|
def __analyze_package_edb(self, pkg_path):
|
|
"""
|
|
Check if the physical Entropy package file contains
|
|
a valid Entropy embedded database.
|
|
|
|
@param pkg_path: path to physical entropy package file
|
|
@type pkg_path: string
|
|
@return: package validity
|
|
@rtype: bool
|
|
"""
|
|
from entropy.db import EntropyRepository
|
|
from entropy.db.exceptions import Error
|
|
|
|
fd, tmp_path = None, None
|
|
dbc = None
|
|
try:
|
|
fd, tmp_path = tempfile.mkstemp(
|
|
prefix="entropy.qa.__analyze_package_edb")
|
|
dump_rc = entropy.tools.dump_entropy_metadata(pkg_path, tmp_path)
|
|
if not dump_rc:
|
|
return False # error!
|
|
|
|
valid = True
|
|
try:
|
|
dbc = EntropyRepository(
|
|
readOnly = False,
|
|
dbFile = tmp_path,
|
|
name = "qa_testing",
|
|
xcache = False,
|
|
indexing = False,
|
|
skipChecks = False
|
|
)
|
|
etp_repo_meta = {
|
|
'output_interface': self,
|
|
}
|
|
repo_plug = QAEntropyRepositoryPlugin(self,
|
|
metadata = etp_repo_meta)
|
|
dbc.add_plugin(repo_plug)
|
|
|
|
except Error:
|
|
valid = False
|
|
|
|
if valid:
|
|
try:
|
|
dbc.validate()
|
|
dbc.integrity_check()
|
|
except SystemDatabaseError:
|
|
valid = False
|
|
|
|
if valid:
|
|
try:
|
|
for idpackage in dbc.listAllPackageIds():
|
|
dbc.retrieveContent(idpackage, extended = True,
|
|
formatted = True, insert_formatted = True)
|
|
except Error:
|
|
valid = False
|
|
|
|
finally:
|
|
if dbc is not None:
|
|
dbc.close()
|
|
if fd is not None:
|
|
os.close(fd)
|
|
if tmp_path is not None:
|
|
os.remove(tmp_path)
|
|
|
|
return valid
|
|
|
|
def entropy_package_checks(self, package_path):
|
|
"""
|
|
Main method for the execution of QA tests on physical Entropy
|
|
package files.
|
|
|
|
@param package_path: path to physical Entropy package file path
|
|
@type package_path: string
|
|
@return: True, if all checks passed
|
|
@rtype: bool
|
|
@raise EntropyPackageException: raised by the QA testing function
|
|
"""
|
|
# built-in ones
|
|
qa_methods = [self.__analyze_package_edb]
|
|
|
|
# plugged ones
|
|
for plug_id, plug_inst in self.get_plugins().items():
|
|
qa_methods += plug_inst.get_tests()
|
|
|
|
# let's play!
|
|
for method in qa_methods:
|
|
qa_rc = method(package_path)
|
|
if not qa_rc:
|
|
return False
|
|
return True
|
|
|
|
|
|
class ErrorReportInterface:
|
|
|
|
"""
|
|
|
|
Interface used by Entropy Client to remotely send errors via HTTP POST.
|
|
Some anonymous info about the running system are collected and sent over,
|
|
once the user gives the acknowledgement for this operation.
|
|
User should be asked for valid credentials, such as name, surname and email.
|
|
This has two advantages: block stupid and lazy people and make possible
|
|
for Entropy developers to contact him/her back.
|
|
Moreover, the same applies for a simple description. To improve the
|
|
ability to debug an issue, it is also asked the user to describe his/her
|
|
action prior to the error.
|
|
|
|
Sample code:
|
|
|
|
>>> from entropy.qa import ErrorReportInterface
|
|
>>> error = ErrorReportInterface('http://url_for_http_post')
|
|
>>> error.prepare('traceback_text', 'John Foo', 'john@foo.com',
|
|
report_data = 'extra traceback info',
|
|
description = 'I was installing foo!')
|
|
>>> error.submit()
|
|
|
|
"""
|
|
|
|
def __init__(self, post_url):
|
|
"""
|
|
ErrorReportInterface constructor.
|
|
|
|
@param post_url: HTTP post url where to submit data
|
|
@type post_url: string
|
|
"""
|
|
from entropy.misc import MultipartPostHandler
|
|
if const_is_python3():
|
|
import urllib.request as urlmod
|
|
else:
|
|
import urllib2 as urlmod
|
|
|
|
self.url = post_url
|
|
self.opener = urlmod.build_opener(MultipartPostHandler)
|
|
self.generated = False
|
|
self.params = {}
|
|
|
|
self._settings = SystemSettings()
|
|
proxy_settings = self._settings['system']['proxy']
|
|
mydict = {}
|
|
if proxy_settings['ftp']:
|
|
mydict['ftp'] = proxy_settings['ftp']
|
|
if proxy_settings['http']:
|
|
mydict['http'] = proxy_settings['http']
|
|
if mydict:
|
|
mydict['username'] = proxy_settings['username']
|
|
mydict['password'] = proxy_settings['password']
|
|
entropy.tools.add_proxy_opener(urlmod, mydict)
|
|
else:
|
|
# unset
|
|
urlmod._opener = None
|
|
|
|
def prepare(self, tb_text, name, email, report_data = "", description = ""):
|
|
|
|
"""
|
|
This method must be called prior to submit(). It is used to prepare
|
|
and collect system information before the submission.
|
|
It is intentionally split from submit() to allow easy reimplementation.
|
|
|
|
@param tb_text: Python traceback text to send
|
|
@type tb_text: string
|
|
@param name: submitter name
|
|
@type name: string
|
|
@param email: submitter email address
|
|
@type email: string
|
|
@keyword report_data: extra information
|
|
@type report_data: string
|
|
@keyword description: submitter action description
|
|
@type description: string
|
|
@return: None
|
|
@rtype: None
|
|
"""
|
|
from entropy.tools import getstatusoutput
|
|
from entropy.client.interfaces.client import ClientSystemSettingsPlugin
|
|
enc = etpConst['conf_encoding']
|
|
|
|
self.params['arch'] = etpConst['currentarch']
|
|
self.params['stacktrace'] = tb_text
|
|
self.params['name'] = name
|
|
self.params['email'] = email
|
|
self.params['version'] = etpConst['entropyversion']
|
|
self.params['errordata'] = report_data
|
|
self.params['description'] = description
|
|
self.params['arguments'] = ' '.join(sys.argv)
|
|
self.params['uid'] = etpConst['uid']
|
|
self.params['system_version'] = "N/A"
|
|
self.params['repositories.conf'] = "---NA---"
|
|
self.params['client.conf'] = "---NA---"
|
|
|
|
self.params['processes'] = getstatusoutput('ps auxf')[1]
|
|
self.params['lsof'] = getstatusoutput('lsof -p %s' % (os.getpid(),))[1]
|
|
self.params['lspci'] = getstatusoutput('/usr/sbin/lspci')[1]
|
|
self.params['dmesg'] = getstatusoutput('dmesg')[1]
|
|
self.params['locale'] = getstatusoutput('locale -v')[1]
|
|
|
|
# configuration files paths
|
|
config_files = self._settings.get_setting_files_data()
|
|
client_conf = ClientSystemSettingsPlugin.client_conf_path()
|
|
|
|
try:
|
|
with codecs.open(etpConst['systemreleasefile'], "r",
|
|
encoding=enc) as f_rel:
|
|
self.params['system_version'] = f_rel.readline().strip()
|
|
except IOError:
|
|
pass
|
|
|
|
try:
|
|
with codecs.open(config_files['repositories'], "r",
|
|
encoding=enc) as rc_f:
|
|
self.params['repositories.conf'] = rc_f.read()
|
|
except IOError:
|
|
pass
|
|
|
|
try:
|
|
with codecs.open(client_conf, "r", encoding=enc) as rc_f:
|
|
self.params['client.conf'] = rc_f.read()
|
|
except IOError:
|
|
pass
|
|
|
|
self.generated = True
|
|
|
|
# params is a dict, key(HTTP post item name): value
|
|
def submit(self):
|
|
"""
|
|
Submit collected data remotely via HTTP POST.
|
|
|
|
@raise PermissionDenied: when prepare() hasn't been called.
|
|
@return: None
|
|
@rtype: None
|
|
"""
|
|
if self.generated:
|
|
result = self.opener.open(self.url, self.params).read()
|
|
if result.strip() == "1":
|
|
return True
|
|
return False
|
|
else:
|
|
mytxt = _("Not prepared yet")
|
|
raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
|