diff --git a/Makefile b/Makefile index 0cbe0b812..2f89e6ec4 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,8 @@ entropy-install: install -m 644 conf/entropy.conf $(DESTDIR)/etc/entropy/ install -m 644 conf/fsdirs.conf $(DESTDIR)/etc/entropy/ install -m 644 conf/fsdirsmask.conf $(DESTDIR)/etc/entropy/ + install -m 644 conf/brokensyms.conf $(DESTDIR)/etc/entropy/ + install -m 644 conf/fssymlinks.conf $(DESTDIR)/etc/entropy/ install -m 644 conf/repositories.conf.example $(DESTDIR)/etc/entropy/ install -m 644 conf/socket* $(DESTDIR)/etc/entropy/ install -m 644 conf/entropy.conf $(DESTDIR)/etc/entropy/ diff --git a/client/text_ui.py b/client/text_ui.py index c59c9810f..6ec28384e 100644 --- a/client/text_ui.py +++ b/client/text_ui.py @@ -1672,7 +1672,8 @@ def librariesTest(listfiles = False): etpUi['mute'] = True etpUi['quiet'] = True - packagesMatched, brokenlibs, status = Equo.libraries_test() + QA = Equo.QA() + packagesMatched, brokenlibs, status = QA.libraries_test(Equo.clientDbconn) if status != 0: restore_qstats() return -1,1 diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index 881c55d8d..7b34a6b72 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -1221,181 +1221,6 @@ class MiscMixin: f.close() return tempfile - def libraries_test(self, dbconn = None, broken_symbols = False, task_bombing_func = None): - - if dbconn == None: - dbconn = self.clientDbconn - - self.updateProgress( - blue(_("Libraries test")), - importance = 2, - type = "info", - header = red(" @@ ") - ) - - if not etpConst['systemroot']: - myroot = "/" - else: - myroot = etpConst['systemroot']+"/" - # run ldconfig first - subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) - # open /etc/ld.so.conf - if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"): - self.updateProgress( - blue(_("Cannot find "))+red(etpConst['systemroot']+"/etc/ld.so.conf"), - importance = 1, - type = "error", - header = red(" @@ ") - ) - return {},set(),-1 - - ldpaths = set(self.entropyTools.collect_linker_paths()) - ldpaths |= self.entropyTools.collect_paths() - # speed up when /usr/lib is a /usr/lib64 symlink - if "/usr/lib64" in ldpaths and "/usr/lib" in ldpaths: - if os.path.realpath("/usr/lib64") == "/usr/lib": - ldpaths.discard("/usr/lib") - # some crappy packages put shit here too - ldpaths.add("/usr/share") - # always force /usr/libexec too - ldpaths.add("/usr/libexec") - - executables = set() - total = len(ldpaths) - count = 0 - sys_root_len = len(etpConst['systemroot']) - for ldpath in ldpaths: - if callable(task_bombing_func): task_bombing_func() - count += 1 - self.updateProgress( - blue("Tree: ")+red(etpConst['systemroot']+ldpath), - importance = 0, - type = "info", - count = (count,total), - back = True, - percent = True, - header = " " - ) - ldpath = ldpath.encode(sys.getfilesystemencoding()) - mywalk_iter = os.walk(etpConst['systemroot']+ldpath) - - def mywimf(dt): - - currentdir, subdirs, files = dt - - def mymf(item): - filepath = os.path.join(currentdir,item) - if filepath in etpConst['libtest_files_blacklist']: - return 0 - if not os.access(filepath,os.R_OK): - return 0 - if not os.path.isfile(filepath): - return 0 - if not self.entropyTools.is_elf_file(filepath): - return 0 - return filepath[sys_root_len:] - - return set([x for x in map(mymf,files) if type(x) != int]) - - for x in map(mywimf,mywalk_iter): executables |= x - - self.updateProgress( - blue(_("Collecting broken executables")), - importance = 2, - type = "info", - header = red(" @@ ") - ) - t = red(_("Attention")) + ": " + \ - blue(_("don't worry about libraries that are shown here but not later.")) - self.updateProgress( - t, - importance = 1, - type = "info", - header = red(" @@ ") - ) - - myQA = self.QA() - - plain_brokenexecs = set() - total = len(executables) - count = 0 - scan_txt = blue("%s ..." % (_("Scanning libraries"),)) - for executable in executables: - if callable(task_bombing_func): task_bombing_func() - count += 1 - if (count%10 == 0) or (count == total) or (count == 1): - self.updateProgress( - scan_txt, - importance = 0, - type = "info", - count = (count,total), - back = True, - percent = True, - header = " " - ) - myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable) - def mymf2(mylib): - return not myQA.resolve_dynamic_library(mylib, executable) - mylibs = set(filter(mymf2,myelfs)) - broken_sym_found = set() - if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable) - if not (mylibs or broken_sym_found): - continue - - if mylibs: - alllibs = blue(' :: ').join(list(mylibs)) - self.updateProgress( - red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]", - importance = 1, - type = "info", - percent = True, - count = (count,total), - header = " " - ) - elif broken_sym_found: - allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)]) - if len(allsyms) > 50: allsyms = brown(_('various broken symbols')) - self.updateProgress( - red(etpConst['systemroot']+executable)+" { "+allsyms+" }", - importance = 1, - type = "info", - percent = True, - count = (count,total), - header = " " - ) - plain_brokenexecs.add(executable) - - del executables - packagesMatched = {} - - if not etpSys['serverside']: - - self.updateProgress( - blue(_("Matching broken libraries/executables")), - importance = 1, - type = "info", - header = red(" @@ ") - ) - matched = set() - for brokenlib in plain_brokenexecs: - idpackages = self.clientDbconn.searchBelongs(brokenlib) - for idpackage in idpackages: - key, slot = self.clientDbconn.retrieveKeySlot(idpackage) - mymatch = self.atom_match(key, matchSlot = slot) - if mymatch[0] == -1: - matched.add(brokenlib) - continue - cmpstat = self.get_package_action(mymatch) - if cmpstat == 0: - continue - if not packagesMatched.has_key(brokenlib): - packagesMatched[brokenlib] = set() - packagesMatched[brokenlib].add(mymatch) - matched.add(brokenlib) - plain_brokenexecs -= matched - - return packagesMatched,plain_brokenexecs,0 - def set_branch(self, branch): """ Set new Entropy branch. This is NOT thread-safe. diff --git a/libraries/entropy/qa.py b/libraries/entropy/qa.py index c63d9fff7..e62121e02 100644 --- a/libraries/entropy/qa.py +++ b/libraries/entropy/qa.py @@ -21,8 +21,11 @@ ''' # pylint ~ok import os +import sys +import subprocess import tempfile -from entropy.const import etpConst + +from entropy.const import etpConst, etpSys from entropy.output import blue, darkgreen, red, darkred, bold, purple, brown from entropy.exceptions import IncorrectParameter, PermissionDenied, \ SystemDatabaseError @@ -262,6 +265,224 @@ class QAInterface: return taint + def libraries_test(self, dbconn, broken_symbols = False, + task_bombing_func = None): + + self.Output.updateProgress( + blue(_("Libraries test")), + importance = 2, + type = "info", + header = red(" @@ ") + ) + + myroot = etpConst['systemroot'] + "/" + if not etpConst['systemroot']: + myroot = "/" + + # run ldconfig first + subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) + # open /etc/ld.so.conf + ld_conf = etpConst['systemroot'] + "/etc/ld.so.conf" + + if not os.path.isfile(ld_conf): + self.Output.updateProgress( + blue(_("Cannot find "))+red(ld_conf), + importance = 1, + type = "error", + header = red(" @@ ") + ) + return {}, set(), -1 + + reverse_symlink_map = self.SystemSettings['system_rev_symlinks'] + ldpaths = set(self.entropyTools.collect_linker_paths()) + ldpaths |= self.entropyTools.collect_paths() + + # remove duplicated dirs (due to symlinks) to speed up scanning + for real_dir in reverse_symlink_map.keys(): + syms = reverse_symlink_map[real_dir] + for sym in syms: + if sym in ldpaths: + ldpaths.discard(real_dir) + self.Output.updateProgress( + "%s: %s, %s: %s" % ( + brown(_("discarding directory")), + purple(real_dir), + brown(_("because it's symlinked on")), + purple(sym), + ), + importance = 0, + type = "info", + header = darkgreen(" @@ ") + ) + break + + # some crappy packages put shit here too + ldpaths.add("/usr/share") + # always force /usr/libexec too + ldpaths.add("/usr/libexec") + + executables = set() + total = len(ldpaths) + count = 0 + sys_root_len = len(etpConst['systemroot']) + for ldpath in ldpaths: + + if callable(task_bombing_func): + task_bombing_func() + count += 1 + self.Output.updateProgress( + blue("Tree: ")+red(etpConst['systemroot'] + ldpath), + importance = 0, + type = "info", + count = (count,total), + back = True, + percent = True, + header = " " + ) + ldpath = ldpath.encode(sys.getfilesystemencoding()) + mywalk_iter = os.walk(etpConst['systemroot'] + ldpath) + + def mywimf(dt): + + currentdir, subdirs, files = dt + + def mymf(item): + filepath = os.path.join(currentdir,item) + if not os.access(filepath, os.R_OK): + return 0 + if not os.path.isfile(filepath): + return 0 + if not self.entropyTools.is_elf_file(filepath): + return 0 + return filepath[sys_root_len:] + + return set([x for x in map(mymf, files) if type(x) != int]) + + for x in map(mywimf,mywalk_iter): + executables |= x + + self.Output.updateProgress( + blue(_("Collecting broken executables")), + importance = 2, + type = "info", + header = red(" @@ ") + ) + t = red(_("Attention")) + ": " + \ + blue(_("don't worry about libraries that are shown here but not later.")) + self.Output.updateProgress( + t, + importance = 1, + type = "info", + header = red(" @@ ") + ) + + plain_brokenexecs = set() + total = len(executables) + count = 0 + scan_txt = blue("%s ..." % (_("Scanning libraries"),)) + for executable in executables: + + if callable(task_bombing_func): + task_bombing_func() + count += 1 + if (count % 10 == 0) or (count == total) or (count == 1): + self.Output.updateProgress( + scan_txt, + importance = 0, + type = "info", + count = (count,total), + back = True, + percent = True, + header = " " + ) + + myelfs = self.entropyTools.read_elf_dynamic_libraries( + etpConst['systemroot'] + executable) + + def mymf2(mylib): + return not self.resolve_dynamic_library(mylib, executable) + + mylibs = set(filter(mymf2,myelfs)) + broken_sym_found = set() + if broken_symbols and not mylibs: broken_sym_found |= \ + self.entropyTools.read_elf_broken_symbols( + etpConst['systemroot'] + executable) + + if not (mylibs or broken_sym_found): + continue + + if mylibs: + alllibs = blue(' :: ').join(list(mylibs)) + self.Output.updateProgress( + red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]", + importance = 1, + type = "info", + percent = True, + count = (count,total), + header = " " + ) + elif broken_sym_found: + + allsyms = darkred(' :: ').join( + [brown(x) for x in list(broken_sym_found)]) + if len(allsyms) > 50: + allsyms = brown(_('various broken symbols')) + + self.Output.updateProgress( + red(etpConst['systemroot']+executable)+" { "+allsyms+" }", + importance = 1, + type = "info", + percent = True, + count = (count,total), + header = " " + ) + + plain_brokenexecs.add(executable) + + del executables + packagesMatched = {} + + if not etpSys['serverside']: + + # we are client side + # this is hackish and must be fixed sooner or later + # but for now, it works + # Client class is singleton and is surely already + # loaded when we get here + from entropy.client.interfaces import Client + client = Client() + + self.Output.updateProgress( + blue(_("Matching broken libraries/executables")), + importance = 1, + type = "info", + header = red(" @@ ") + ) + matched = set() + for brokenlib in plain_brokenexecs: + idpackages = dbconn.searchBelongs(brokenlib) + + for idpackage in idpackages: + + key, slot = dbconn.retrieveKeySlot(idpackage) + mymatch = client.atom_match(key, matchSlot = slot) + if mymatch[0] == -1: + matched.add(brokenlib) + continue + + cmpstat = client.get_package_action(mymatch) + if cmpstat == 0: + continue + if not packagesMatched.has_key(brokenlib): + packagesMatched[brokenlib] = set() + + packagesMatched[brokenlib].add(mymatch) + matched.add(brokenlib) + + plain_brokenexecs -= matched + + return packagesMatched, plain_brokenexecs, 0 + def content_test(self, mycontent): def is_contained(needed, content): diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py index 5a178c62c..0716e1322 100644 --- a/libraries/entropy/server/interfaces/main.py +++ b/libraries/entropy/server/interfaces/main.py @@ -1087,9 +1087,9 @@ class Server(Singleton, TextInterface): # load db dbconn = self.open_server_repository(read_only = True, no_upload = True, repo = repo) + QA = self.QA() packages_matched, brokenexecs, status = \ - self.ClientService.libraries_test(dbconn = dbconn, - broken_symbols = False) + QA.libraries_test(dbconn, broken_symbols = False) if status != 0: return 1, None diff --git a/sulfur/src/sulfur/events.py b/sulfur/src/sulfur/events.py index 3e2af3a4a..5397ba74f 100644 --- a/sulfur/src/sulfur/events.py +++ b/sulfur/src/sulfur/events.py @@ -934,11 +934,13 @@ class SulfurApplicationEventsMixin: do_start() - packages_matched, broken_execs = {},set() + packages_matched, broken_execs = {}, set() self.libtest_abort = False + QA = self.Equo.QA() + def exec_task(): try: - x, y, z = self.Equo.libraries_test( + x, y, z = QA.libraries_test(self.Equo.clientDbconn, task_bombing_func = task_bombing) packages_matched.update(x) broken_execs.update(y)