Entropy/entropyTools:

- some optimizations to the elf metadata reading/parsing functions
- new function read_elf_broken_symbols
Entropy/QAInterface:
- libraries_test: implement broken symbols check, optional
Entropy/ServerInterface:
- always enable broken symbols check on libtest


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@2830 cd1c1023-2f26-0410-ae45-c471fc1f0318

libraries/entropy.py

@@ -1265,7 +1265,7 @@ class EquoInterface(TextInterface):
             editor = "/bin/emacs"
         return editor
 
-    def libraries_test(self, dbconn = None):
+    def libraries_test(self, dbconn = None, broken_symbols = False):
 
         if dbconn == None:
             dbconn = self.clientDbconn
@@ -1307,7 +1307,7 @@ class EquoInterface(TextInterface):
         executables = set()
         total = len(ldpaths)
         count = 0
-        # FIXME: merge the two for cycles below
+        sys_root_len = len(etpConst['systemroot'])
         for ldpath in ldpaths:
             count += 1
             self.updateProgress(
@@ -1320,18 +1320,27 @@ class EquoInterface(TextInterface):
                 header = "  "
             )
             ldpath = ldpath.encode(sys.getfilesystemencoding())
-            for currentdir,subdirs,files in os.walk(etpConst['systemroot']+ldpath):
-                for item in files:
+            mywalk_iter = os.walk(etpConst['systemroot']+ldpath)
+
+            def mywimf(dt):
+
+                currentdir, subdirs, files = dt
+
+                def mymf(item):
                     filepath = os.path.join(currentdir,item)
                     if filepath in etpConst['libtest_files_blacklist']:
-                        continue
-                    if not os.path.isfile(filepath):
-                        continue
+                        return 0
                     if not os.access(filepath,os.R_OK):
-                        continue
+                        return 0
+                    if not os.path.isfile(filepath):
+                        return 0
                     if not self.entropyTools.is_elf_file(filepath):
-                        continue
-                    executables.add(filepath[len(etpConst['systemroot']):])
+                        return 0
+                    return filepath[sys_root_len:]
+
+                return set([x for x in map(mymf,files) if type(x) != int])
+
+            for x in map(mywimf,mywalk_iter): executables |= x
 
         self.updateProgress(
             blue(_("Collecting broken executables")),
@@ -1367,24 +1376,35 @@ class EquoInterface(TextInterface):
                     header = "  "
                 )
             myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable)
-            mylibs = set()
-            for mylib in myelfs:
-                found = myQA.resolve_dynamic_library(mylib, executable)
-                if found:
-                    continue
-                mylibs.add(mylib)
-            if not mylibs:
+            def mymf(mylib):
+                return not myQA.resolve_dynamic_library(mylib, executable)
+            mylibs = set(filter(mymf,myelfs))
+            broken_sym_found = set()
+            if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable)
+            if not (mylibs or broken_sym_found):
                 continue
 
-            alllibs = blue(' :: ').join(list(mylibs))
-            self.updateProgress(
-                red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]",
-                importance = 1,
-                type = "info",
-                percent = True,
-                count = (count,total),
-                header = "  "
-            )
+            if mylibs:
+                alllibs = blue(' :: ').join(list(mylibs))
+                self.updateProgress(
+                    red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]",
+                    importance = 1,
+                    type = "info",
+                    percent = True,
+                    count = (count,total),
+                    header = "  "
+                )
+            elif broken_sym_found:
+                allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)])
+                if len(allsyms) > 50: allsyms = brown(_('various broken symbols'))
+                self.updateProgress(
+                    red(etpConst['systemroot']+executable)+" { "+allsyms+" }",
+                    importance = 1,
+                    type = "info",
+                    percent = True,
+                    count = (count,total),
+                    header = "  "
+                )
             plain_brokenexecs.add(executable)
 
         del executables
@@ -15823,7 +15843,7 @@ class ServerInterface(TextInterface):
 
         # load db
         dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
-        packagesMatched, brokenexecs, status = self.ClientService.libraries_test(dbconn = dbconn)
+        packagesMatched, brokenexecs, status = self.ClientService.libraries_test(dbconn = dbconn, broken_symbols = True)
         if status != 0:
             return 1,None
 

libraries/entropyTools.py

@@ -27,6 +27,7 @@ import exceptionTools
 import re
 import threading
 import time
+import commands
 
 def isRoot():
     if (etpConst['uid'] == 0):
@@ -2203,36 +2204,37 @@ def is_elf_file(elf_file):
         return True
     return False
 
+readelf_avail_check = False
+ldd_avail_check = False
+
 # FIXME: reimplement this
 def read_elf_dynamic_libraries(elf_file):
-    import commands
-    if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
-        raise exceptionTools.FileNotFound('FileNotFound: no readelf')
-    data = commands.getoutput('readelf -d %s' % (elf_file,)).split("\n")
-    mylibs = set()
-    for line in data:
-        if line.find("(NEEDED)") == -1:
-            continue
-        mylib = line.strip().split()[-1]
-        if not (mylib.endswith("]") and mylib.startswith("[")):
-            continue
-        mylibs.add(mylib[1:-1])
-    return mylibs
+    global readelf_avail_check
+    if not readelf_avail_check:
+        if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
+            raise exceptionTools.FileNotFound('FileNotFound: no readelf')
+        readelf_avail_check = True
+    return set([x.strip().split()[-1][1:-1] for x in commands.getoutput('/usr/bin/readelf -d %s' % (elf_file,)).split("\n") if (x.find("(NEEDED)") != -1)])
+
+def read_elf_broken_symbols(elf_file):
+    global ldd_avail_check
+    if not ldd_avail_check:
+        if not os.access(etpConst['systemroot']+"/usr/bin/ldd",os.X_OK):
+            raise exceptionTools.FileNotFound('FileNotFound: no ldd')
+        ldd_avail_check = True
+    return set([x.strip().split("\t")[0].split()[-1] for x in commands.getoutput('/usr/bin/ldd -r %s' % (elf_file,)).split("\n") if (x.find("undefined symbol:") != -1)])
+
 
 # FIXME: reimplement this
 def read_elf_linker_paths(elf_file):
-    import commands
-    if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
-        raise exceptionTools.FileNotFound('FileNotFound: no readelf')
-    data = commands.getoutput('readelf -d %s' % (elf_file,)).split("\n")
+    global readelf_avail_check
+    if not readelf_avail_check:
+        if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
+            raise exceptionTools.FileNotFound('FileNotFound: no readelf')
+        readelf_avail_check = True
+    data = [x.strip().split()[-1][1:-1].split(":") for x in commands.getoutput('readelf -d %s' % (elf_file,)).split("\n") if not ((x.find("(RPATH)") == -1) and (x.find("(RUNPATH)") == -1))]
     mypaths = []
-    for line in data:
-        if (line.find("(RPATH)") == -1) and (line.find("(RUNPATH)") == -1):
-            continue
-        mylib = line.strip().split()[-1]
-        if not (mylib.endswith("]") and mylib.startswith("[")):
-            continue
-        mypath = mylib[1:-1].split(":")
+    for mypath in data:
         for xpath in mypath:
             xpath = xpath.replace("$ORIGIN",os.path.dirname(elf_file))
             mypaths.append(xpath)