1
2 """
3
4 @author: Fabio Erculiani <lxnay@sabayonlinux.org>
5 @contact: lxnay@sabayonlinux.org
6 @copyright: Fabio Erculiani
7 @license: GPL-2
8
9 B{Entropy Framework QA module}.
10
11 This module contains various Quality Assurance routines used by Entropy.
12
13 B{QAInterface} is the main class for QA routines used by Entropy Server
14 and Entropy Client such as binary packages health check, dependency
15 test, broken or missing library tes.
16
17 B{ErrorReportInterface} is the HTTP POST based class for Entropy Client
18 exceptions (errors) submission.
19
20 """
21 import os
22 import sys
23 import subprocess
24 import tempfile
25
26 from entropy.const import etpConst, etpSys, const_debug_write, const_debug_write
27 from entropy.output import blue, darkgreen, red, darkred, bold, purple, brown
28 from entropy.exceptions import IncorrectParameter, PermissionDenied, \
29 SystemDatabaseError
30 from entropy.i18n import _
31 from entropy.core import EntropyPluginStore
32 from entropy.core.settings.base import SystemSettings
33 from entropy.db.skel import EntropyRepositoryPlugin
34
36
37 - def __init__(self, qa_interface, metadata = None):
38 """
39 Entropy QA-side repository EntropyRepository Plugin class.
40 This class will be instantiated and automatically added to
41 EntropyRepository instances generated by Entropy QA.
42
43 @param qa_interface: Entropy Client interface instance
44 @type qa_interface: entropy.qa.QAInterface class
45 @param metadata: any dict form metadata map (key => value)
46 @type metadata: dict
47 """
48 EntropyRepositoryPlugin.__init__(self)
49 self._qa = qa_interface
50 if metadata is None:
51 self._metadata = {}
52 else:
53 self._metadata = metadata
54
55
56
57
58
59 self._metadata['client_repo'] = True
60
63
66
79
81 """
82 Inherit this class to create a QAInterface tests Plugin.
83 You need to implement
84
85 """
87 """
88 Return a list of callable functions that will be used by QAInterface,
89 for tests execution.
90 Note: the callable functions signature must be:
91 bool function(package_path)
92
93 @return: list of callable objects
94 @rtype: list
95 """
96 raise NotImplementedError()
97
99 """
100 Return an identifier string for this Plugin.
101
102 @return: identifier string
103 @rtype: string
104 """
105 raise NotImplementedError()
106
108
109 """
110 Entropy QA interface. This class contains all the Entropy
111 QA routines used by Entropy Server and Entropy Client.
112
113 An instance of QAInterface can be easily retrieved from
114 entropy.client.interfaces.Client or entropy.server.interfaces.Server
115 through an exposed QA() method.
116 This is anyway a stand-alone class.
117
118 @todo: remove non-QA methods
119
120 """
121
122 import entropy.tools as entropyTools
123 from entropy.misc import Lifo
125 """
126 QAInterface constructor.
127
128 @param OutputInterface: class instance used to print output.
129 Even if not enforced at the moment, it should be a subclass of
130 entropy.qa.TextInterface exposing the updateProgress() method
131 with proper signature.
132 @type OutputInterface: TextInterface class or subclass instance
133 """
134 EntropyPluginStore.__init__(self)
135 self.Output = OutputInterface
136 self.SystemSettings = SystemSettings()
137
138 if not hasattr(self.Output, 'updateProgress'):
139 mytxt = _("Output interface has no updateProgress method")
140 raise AttributeError("AttributeError: %s" % (mytxt,))
141 elif not hasattr(self.Output.updateProgress, '__call__'):
142 mytxt = _("Output interface has no updateProgress method")
143 raise AttributeError("AttributeError: %s" % (mytxt,))
144
146 """
147 Add a QAInterface plugin to the testing list.
148
149 @param plugin: QAInterfacePlugin based instance
150 @type plugin: QAInterface based instance
151 @raise AttributeError: if plugin is not a QAInterfacePlugin instance
152 """
153 if not isinstance(plugin, QAInterfacePlugin):
154 raise AttributeError("Specify a QAInterfacePlugin based class")
155 return EntropyPluginStore.add_plugin(self, plugin.get_id(), plugin)
156
158 """
159 Scan for broken shared objects linking for the given idpackages on
160 the given entropy.db.EntropyRepository based instance.
161 Note: this only works for packages actually installed on the running
162 system.
163 It is used by Entropy Server during packages injection into database
164 to warn about potentially broken packages.
165
166 @param idpackages: list of valid idpackages (int) on the given dbconn
167 argument passed
168 @type idpackages: list
169 @param dbconn: entropy.db.EntropyRepository instance containing the
170 given idpackages list
171 @type dbconn: entropy.db.EntropyRepository
172 @keyword repo: repository identifer from which dbconn and idpackages
173 arguments belong. Note: at the moment it's only used for output
174 purposes.
175 @type repo: string
176 @return: True if any breakage is found, otherwise False
177 @rtype: bool
178 """
179 if repo is None:
180 repo = self.SystemSettings['repositories']['default_repository']
181
182 scan_msg = blue(_("Now searching for broken depends"))
183 self.Output.updateProgress(
184 "[repo:%s] %s..." % (
185 darkgreen(repo),
186 scan_msg,
187 ),
188 importance = 1,
189 type = "info",
190 header = red(" @@ ")
191 )
192
193 broken = False
194
195 count = 0
196 maxcount = len(idpackages)
197 for idpackage in idpackages:
198 count += 1
199 atom = dbconn.retrieveAtom(idpackage)
200 scan_msg = "%s, %s:" % (
201 blue(_("scanning for broken depends")),
202 darkgreen(atom),
203 )
204 self.Output.updateProgress(
205 "[repo:%s] %s" % (
206 darkgreen(repo),
207 scan_msg,
208 ),
209 importance = 1,
210 type = "info",
211 header = blue(" @@ "),
212 back = True,
213 count = (count, maxcount,)
214 )
215 mydepends = dbconn.retrieveReverseDependencies(idpackage)
216 if not mydepends:
217 continue
218 for mydepend in mydepends:
219 myatom = dbconn.retrieveAtom(mydepend)
220 self.Output.updateProgress(
221 "[repo:%s] %s => %s" % (
222 darkgreen(repo),
223 darkgreen(atom),
224 darkred(myatom),
225 ),
226 importance = 0,
227 type = "info",
228 header = blue(" @@ "),
229 back = True,
230 count = (count, maxcount,)
231 )
232 mycontent = dbconn.retrieveContent(mydepend)
233 mybreakages = self._content_test(mycontent)
234 if not mybreakages:
235 continue
236 broken = True
237 self.Output.updateProgress(
238 "[repo:%s] %s %s => %s" % (
239 darkgreen(repo),
240 darkgreen(atom),
241 darkred(myatom),
242 bold(_("broken libraries detected")),
243 ),
244 importance = 1,
245 type = "warning",
246 header = purple(" @@ "),
247 count = (count, maxcount,)
248 )
249 for mylib in mybreakages:
250 self.Output.updateProgress(
251 "%s %s:" % (
252 darkgreen(mylib),
253 red(_("needs")),
254 ),
255 importance = 1,
256 type = "warning",
257 header = brown(" ## ")
258 )
259 for needed in mybreakages[mylib]:
260 self.Output.updateProgress(
261 "%s" % (
262 red(needed),
263 ),
264 importance = 1,
265 type = "warning",
266 header = purple(" # ")
267 )
268 return broken
269
270 - def test_missing_dependencies(self, idpackages, dbconn, ask = True,
271 self_check = False, repo = None, black_list = None,
272 black_list_adder = None):
273 """
274 Scan missing dependencies for the given idpackages on the given
275 entropy.db.EntropyRepository "dbconn" instance. In addition, this method
276 will allow the user through OutputInterface to interactively add (if ask
277 == True) missing dependencies or blacklist them.
278
279 @param idpackages: list of valid idpackages (int) on the given dbconn
280 argument passed
281 @type idpackages: list
282 @param dbconn: entropy.db.EntropyRepository instance containing the
283 given idpackages list
284 @type dbconn: entropy.db.EntropyRepository
285 @keyword ask: request user interaction when finding missing dependencies
286 @type ask: bool
287 @keyword self_check: also introspect inside the complaining package
288 (to avoid reporting false positives when circular dependencies
289 occur)
290 @type self_check: bool
291 @keyword repo: repository identifier of the given
292 entropy.db.EntropyRepository dbconn instance.
293 It is used to correctly place blacklisted items.
294 @type repo: string
295 @keyword black_list: list of dependencies already blacklisted.
296 @type black_list: set
297 @keyword black_list_adder: callable function that accepts two arguments:
298 (1) list (set) of new dependencies to blacklist for the
299 given (2) repository identifier.
300 @type black_list_adder: callable
301 @return: tainting status, if any dependency has been added
302 @rtype: bool
303 """
304 if repo is None:
305 repo = self.SystemSettings['repositories']['default_repository']
306
307 if not isinstance(black_list, set):
308 black_list = set()
309
310 taint = False
311 scan_msg = blue(_("Now searching for missing RDEPENDs"))
312 self.Output.updateProgress(
313 "[repo:%s] %s..." % (
314 darkgreen(repo),
315 scan_msg,
316 ),
317 importance = 1,
318 type = "info",
319 header = red(" @@ ")
320 )
321 scan_msg = blue(_("scanning for missing RDEPENDs"))
322 count = 0
323 maxcount = len(idpackages)
324 for idpackage in idpackages:
325 count += 1
326 atom = dbconn.retrieveAtom(idpackage)
327 if not atom:
328 continue
329 self.Output.updateProgress(
330 "[repo:%s] %s: %s" % (
331 darkgreen(repo),
332 scan_msg,
333 darkgreen(atom),
334 ),
335 importance = 1,
336 type = "info",
337 header = blue(" @@ "),
338 back = True,
339 count = (count, maxcount,)
340 )
341 missing_extended, missing = self._get_missing_rdepends(dbconn,
342 idpackage, self_check = self_check)
343 missing -= black_list
344 for item in list(missing_extended.keys()):
345 missing_extended[item] -= black_list
346 if not missing_extended[item]:
347 del missing_extended[item]
348 if (not missing) or (not missing_extended):
349 continue
350 self.Output.updateProgress(
351 "[repo:%s] %s: %s %s:" % (
352 darkgreen(repo),
353 blue("package"),
354 darkgreen(atom),
355 blue(_("is missing the following dependencies")),
356 ),
357 importance = 1,
358 type = "info",
359 header = red(" @@ "),
360 count = (count, maxcount,)
361 )
362 for missing_data in missing_extended:
363 self.Output.updateProgress(
364 "%s:" % (brown(repr(missing_data)),),
365 importance = 0,
366 type = "info",
367 header = purple(" ## ")
368 )
369 for dependency in missing_extended[missing_data]:
370 self.Output.updateProgress(
371 "%s" % (darkred(dependency),),
372 importance = 0,
373 type = "info",
374 header = blue(" # ")
375 )
376 if ask:
377 rc_ask = self.Output.askQuestion(_("Do you want to add them?"))
378 if rc_ask == _("No"):
379 continue
380 rc_ask = self.Output.askQuestion(_("Selectively?"))
381 if rc_ask == _("Yes"):
382 newmissing = set()
383 new_blacklist = set()
384 for dependency in missing:
385 self.Output.updateProgress(
386 "[repo:%s|%s] %s" % (
387 darkgreen(repo),
388 brown(atom),
389 blue(dependency),
390 ),
391 importance = 0,
392 type = "info",
393 header = blue(" @@ ")
394 )
395 rc_ask = self.Output.askQuestion(_("Want to add?"))
396 if rc_ask == _("Yes"):
397 newmissing.add(dependency)
398 else:
399 rc_ask = self.Output.askQuestion(
400 _("Want to blacklist?"))
401 if rc_ask == _("Yes"):
402 new_blacklist.add(dependency)
403 if new_blacklist and (black_list_adder != None):
404 black_list_adder(new_blacklist, repo = repo)
405 missing = newmissing
406 if missing:
407 taint = True
408 dbconn.insertDependencies(idpackage, missing)
409 dbconn.commitChanges()
410 self.Output.updateProgress(
411 "[repo:%s] %s: %s" % (
412 darkgreen(repo),
413 darkgreen(atom),
414 blue(_("missing dependencies added")),
415 ),
416 importance = 1,
417 type = "info",
418 header = red(" @@ "),
419 count = (count, maxcount,)
420 )
421
422 return taint
423
424 - def test_shared_objects(self, dbconn, broken_symbols = False,
425 task_bombing_func = None, self_dir_check = True,
426 dump_results_to_file = False):
427
428 """
429 Scan system looking for broken shared object ELF library dependencies.
430
431 @param dbconn: entropy.db.EntropyRepository instance which contains
432 information on packages installed on the system (for example:
433 entropy.client.interfaces.Client.clientDbconn ).
434 @type dbconn: entropy.db.EntropyRepository instance
435 @keyword broken_symbols: enable or disable broken symbols extra check.
436 Symbols which are going to be checked have to be listed into:
437 /etc/entropy/brokensyms.conf (regexp supported).
438 @type broken_symbols: bool
439 @keyword task_bombing_func: callable that will be called on every
440 scan iteration to allow external routines to cleanly stop the
441 execution of this function.
442 @type task_bombing_func: callable
443 @keyword dump_results_to_file: dump test results to files (printed)
444 @type dump_results_to_file: bool
445 @return: tuple of length 3, composed by (1) a dict of matched packages,
446 (2) a list (set) of broken ELF objects and (3) the execution status
447 (int, 0 means success).
448 @rtype: tuple
449 """
450
451 self.Output.updateProgress(
452 blue(_("Libraries test")),
453 importance = 2,
454 type = "info",
455 header = red(" @@ ")
456 )
457
458 syms_list_path = None
459 files_list_path = None
460 if dump_results_to_file:
461
462 tmp_dir = os.path.dirname(self.entropyTools.get_random_temp_file())
463 syms_list_path = os.path.join(tmp_dir, "libtest_syms.txt")
464 files_list_path = os.path.join(tmp_dir, "libtest_files.txt")
465
466 dmp_data = [
467 (_("Broken symbols packages list"), syms_list_path,),
468 (_("Broken executables list"), files_list_path,),
469 ]
470 mytxt = "%s:" % (purple(_("Dumping results into these files")),)
471 self.Output.updateProgress(
472 mytxt,
473 importance = 1,
474 type = "info",
475 header = blue(" @@ ")
476 )
477 for txt, path in dmp_data:
478 mytxt = "%s: %s" % (blue(txt), path,)
479 self.Output.updateProgress(
480 mytxt,
481 importance = 0,
482 type = "info",
483 header = darkgreen(" ## ")
484 )
485
486 myroot = etpConst['systemroot'] + os.path.sep
487 if not etpConst['systemroot']:
488 myroot = os.path.sep
489
490
491 subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True)
492
493 ld_conf = etpConst['systemroot'] + "/etc/ld.so.conf"
494
495 if not os.path.isfile(ld_conf):
496 self.Output.updateProgress(
497 blue(_("Cannot find "))+red(ld_conf),
498 importance = 1,
499 type = "error",
500 header = red(" @@ ")
501 )
502 return {}, set(), -1
503
504 reverse_symlink_map = self.SystemSettings['system_rev_symlinks']
505 broken_syms_list = self.SystemSettings['broken_syms']
506 broken_libs_mask = self.SystemSettings['broken_libs_mask']
507
508 import re
509
510 broken_syms_list_regexp = []
511 for broken_sym in broken_syms_list:
512 if broken_sym.startswith("r:"):
513 broken_sym = broken_sym[2:]
514 if not broken_sym:
515 continue
516 reg_sym = re.compile(broken_sym)
517 else:
518 reg_sym = re.compile(re.escape(broken_sym))
519 broken_syms_list_regexp.append(reg_sym)
520
521 broken_libs_mask_regexp = []
522 broken_libs_paths_mask_regexp = []
523 for broken_lib in broken_libs_mask:
524 if broken_lib.startswith("r:"):
525 broken_lib = broken_lib[2:]
526 if not broken_lib:
527 continue
528 reg_lib = re.compile(broken_lib)
529 else:
530 reg_lib = re.compile(re.escape(broken_lib))
531 if os.path.sep in broken_lib:
532
533 broken_libs_paths_mask_regexp.append(reg_lib)
534 else:
535 broken_libs_mask_regexp.append(reg_lib)
536
537 ldpaths = set(self.entropyTools.collect_linker_paths())
538 ldpaths |= self.entropyTools.collect_paths()
539
540
541 ldpaths.add("/usr/share")
542
543 ldpaths.add("/usr/libexec")
544
545
546 for real_dir in list(reverse_symlink_map.keys()):
547 syms = reverse_symlink_map[real_dir]
548 for sym in syms:
549 if sym in ldpaths:
550 ldpaths.discard(real_dir)
551 self.Output.updateProgress(
552 "%s: %s, %s: %s" % (
553 brown(_("discarding directory")),
554 purple(real_dir),
555 brown(_("because it's symlinked on")),
556 purple(sym),
557 ),
558 importance = 0,
559 type = "info",
560 header = darkgreen(" @@ ")
561 )
562 break
563
564 executables = set()
565 total = len(ldpaths)
566 count = 0
567 sys_root_len = len(etpConst['systemroot'])
568 for ldpath in ldpaths:
569
570 if hasattr(task_bombing_func, '__call__'):
571 task_bombing_func()
572 count += 1
573 self.Output.updateProgress(
574 blue("Tree: ")+red(etpConst['systemroot'] + ldpath),
575 importance = 0,
576 type = "info",
577 count = (count, total),
578 back = True,
579 percent = True,
580 header = " "
581 )
582 try:
583 ldpath = ldpath.encode('utf-8')
584 except (UnicodeEncodeError,):
585 ldpath = ldpath.encode(sys.getfilesystemencoding())
586 mywalk_iter = os.walk(etpConst['systemroot'] + ldpath)
587
588 def mywimf(dt):
589
590 currentdir, subdirs, files = dt
591
592 def mymf(item):
593 filepath = os.path.join(currentdir, item)
594 if not os.access(filepath, os.R_OK):
595 return 0
596 if not os.path.isfile(filepath):
597 return 0
598 if not self.entropyTools.is_elf_file(filepath):
599 return 0
600 return filepath[sys_root_len:]
601
602 return set([x for x in map(mymf, files) if not isinstance(x, int)])
603
604 for x in map(mywimf, mywalk_iter):
605 executables |= x
606
607 self.Output.updateProgress(
608 blue(_("Collecting broken executables")),
609 importance = 2,
610 type = "info",
611 header = red(" @@ ")
612 )
613 t = red(_("Attention")) + ": " + \
614 blue(_("don't worry about libraries that are shown here but not later."))
615 self.Output.updateProgress(
616 t,
617 importance = 1,
618 type = "info",
619 header = red(" @@ ")
620 )
621
622 syms_list_f = None
623 if syms_list_path:
624 syms_list_f = open(syms_list_path, "w")
625
626 files_list_f = None
627 if files_list_path:
628 files_list_f = open(files_list_path, "w")
629
630 plain_brokenexecs = set()
631 total = len(executables)
632 count = 0
633 scan_txt = blue("%s ..." % (_("Scanning libraries"),))
634 for executable in executables:
635
636
637 if hasattr(task_bombing_func, '__call__'):
638 task_bombing_func()
639
640 count += 1
641 if (count % 10 == 0) or (count == total) or (count == 1):
642 self.Output.updateProgress(
643 scan_txt,
644 importance = 0,
645 type = "info",
646 count = (count, total),
647 back = True,
648 percent = True,
649 header = " "
650 )
651
652
653
654
655 exec_match = False
656 for reg_path in broken_libs_paths_mask_regexp:
657 if reg_path.match(executable):
658 exec_match = True
659 break
660 if exec_match:
661 continue
662
663 real_exec_path = etpConst['systemroot'] + executable
664
665 myelfs = self.entropyTools.read_elf_dynamic_libraries(
666 real_exec_path)
667
668 mylibs = set()
669 for mylib in myelfs:
670 lib_path = self.entropyTools.resolve_dynamic_library(mylib,
671 executable)
672 if not lib_path:
673 mylibs.add(mylib)
674
675
676 if mylibs:
677
678 mylib_filter = set()
679 for mylib in mylibs:
680 mylib_matched = False
681 for reg_lib in broken_libs_mask_regexp:
682 if reg_lib.match(mylib):
683 mylib_matched = True
684 break
685
686 if mylib_matched:
687 mylib_filter.add(mylib)
688
689 elif self_dir_check:
690
691
692 my_real_exec_dir = os.path.dirname(real_exec_path)
693 mylib_guess = os.path.join(my_real_exec_dir, mylib)
694 if os.access(mylib_guess, os.R_OK) and \
695 os.path.isfile(mylib_guess):
696 if self.entropyTools.is_elf_file(mylib_guess):
697
698
699
700 mylib_filter.add(mylib)
701
702 mylibs -= mylib_filter
703
704 broken_sym_found = set()
705 if broken_symbols and not mylibs:
706
707 read_broken_syms = self.entropyTools.read_elf_broken_symbols(
708 real_exec_path)
709 my_broken_syms = set()
710 for read_broken_sym in read_broken_syms:
711 for reg_sym in broken_syms_list_regexp:
712 if reg_sym.match(read_broken_sym):
713 my_broken_syms.add(read_broken_sym)
714 break
715 broken_sym_found.update(my_broken_syms)
716
717 if not (mylibs or broken_sym_found):
718 continue
719
720 if mylibs:
721
722 if files_list_f:
723 files_list_f.write(executable + "\n")
724
725 alllibs = blue(' :: ').join(sorted(mylibs))
726 self.Output.updateProgress(
727 red(real_exec_path)+" [ "+alllibs+" ]",
728 importance = 1,
729 type = "info",
730 percent = True,
731 count = (count, total),
732 header = " "
733 )
734 elif broken_sym_found:
735
736 allsyms = darkred(' :: ').join([brown(x) for x in \
737 broken_sym_found])
738 if len(allsyms) > 50:
739 allsyms = brown(_('various broken symbols'))
740
741 if syms_list_f and broken_sym_found:
742 syms_list_f.write("%s => %s\n" % (real_exec_path,
743 sorted(broken_sym_found),))
744
745 self.Output.updateProgress(
746 red(real_exec_path)+" { "+allsyms+" }",
747 importance = 1,
748 type = "info",
749 percent = True,
750 count = (count, total),
751 header = " "
752 )
753
754 plain_brokenexecs.add(executable)
755
756
757 if syms_list_f:
758 syms_list_f.flush()
759 syms_list_f.close()
760 if files_list_f:
761 files_list_f.flush()
762 files_list_f.close()
763
764 del executables
765 packagesMatched = {}
766
767 if not etpSys['serverside']:
768
769
770
771
772
773
774 from entropy.client.interfaces import Client
775 client = Client()
776
777 self.Output.updateProgress(
778 blue(_("Matching broken libraries/executables")),
779 importance = 1,
780 type = "info",
781 header = red(" @@ ")
782 )
783 matched = set()
784 for brokenlib in plain_brokenexecs:
785 idpackages = dbconn.searchBelongs(brokenlib)
786
787 for idpackage in idpackages:
788
789 key, slot = dbconn.retrieveKeySlot(idpackage)
790 mymatch = client.atom_match(key, matchSlot = slot)
791 if mymatch[0] == -1:
792 matched.add(brokenlib)
793 continue
794
795 cmpstat = client.get_package_action(mymatch)
796 if cmpstat == 0:
797 continue
798 if brokenlib not in packagesMatched:
799 packagesMatched[brokenlib] = set()
800
801 packagesMatched[brokenlib].add(mymatch)
802 matched.add(brokenlib)
803
804 plain_brokenexecs -= matched
805
806 return packagesMatched, plain_brokenexecs, 0
807
808 - def _content_test(self, mycontent):
809 """
810 Test whether the given list of files contain files
811 with broken shared object links.
812
813 @param mycontent: list of file paths
814 @type mycontent: list or set
815 @return: dict containing a map between file path
816 and list (set) of broken libraries (just the library name,
817 the same that is contained inside ELF metadata)
818 @rtype: dict
819 """
820 def is_contained(needed, content):
821 for item in content:
822 if os.path.basename(item) == needed:
823 return True
824 return False
825
826 mylibs = {}
827 for myfile in mycontent:
828 myfile = myfile.encode('raw_unicode_escape')
829 if not os.access(myfile, os.R_OK):
830 continue
831 if not os.path.isfile(myfile):
832 continue
833 if not self.entropyTools.is_elf_file(myfile):
834 continue
835 mylibs[myfile] = self.entropyTools.read_elf_dynamic_libraries(
836 myfile)
837
838 broken_libs = {}
839 for mylib in mylibs:
840 for myneeded in mylibs[mylib]:
841
842 if is_contained(myneeded, mycontent):
843 continue
844 found = self.entropyTools.resolve_dynamic_library(myneeded,
845 mylib)
846 if found:
847 continue
848 if mylib not in broken_libs:
849 broken_libs[mylib] = set()
850 broken_libs[mylib].add(myneeded)
851
852 return broken_libs
853
855 """
856 Service method able to determine whether dependencies are missing
857 on the given idpackage (belonging to the given
858 entropy.db.EntropyRepository "dbconn" argument) using shared objects
859 linking information between packages.
860
861 @todo: swap the first two arguments?
862 @param dbconn: entropy.db.EntropyRepository instance from which idpackage
863 argument belongs
864 @type dbconn: entropy.db.EntropyRepository instance
865 @param idpackage: entropy.db.EntropyRepository package identifier
866 @type idpackage: int
867 @keyword self_check: also check inside the given package
868 (idpackage) itself
869 @type self_check: bool
870 @return: tuple of length 2, composed by a dictionary with the
871 following structure:
872 {('KEY', 'SLOT': set([list of missing deps for the given key])}
873 and a "plain" list (set) of missing dependencies
874 set([list of missing dependencies])
875 @rtype: tuple
876 """
877 rdepends = {}
878 rdepends_plain = set()
879 neededs = dbconn.retrieveNeeded(idpackage, extended = True)
880 ldpaths = set(self.entropyTools.collect_linker_paths())
881 deps_content = set()
882 dependencies = self.get_deep_dependency_list(dbconn, idpackage,
883 atoms = True)
884 scope_cache = set()
885
886 def update_depscontent(mycontent, dbconn, ldpaths):
887 return set( \
888 [x for x in mycontent if os.path.dirname(x) in ldpaths \
889 and (dbconn.isNeededAvailable(os.path.basename(x)) > 0) ])
890
891 def is_in_content(myneeded, content):
892 for item in content:
893 item = os.path.basename(item)
894 if myneeded == item:
895 return True
896 return False
897
898 for dependency in dependencies:
899 match = dbconn.atomMatch(dependency)
900 if match[0] != -1:
901 mycontent = dbconn.retrieveContent(match[0])
902 deps_content |= update_depscontent(mycontent, dbconn, ldpaths)
903 key, slot = dbconn.retrieveKeySlot(match[0])
904 scope_cache.add((key, slot))
905
906 key, slot = dbconn.retrieveKeySlot(idpackage)
907 mycontent = dbconn.retrieveContent(idpackage)
908 deps_content |= update_depscontent(mycontent, dbconn, ldpaths)
909 scope_cache.add((key, slot))
910
911 idpackages_cache = set()
912 idpackage_map = {}
913 idpackage_map_reverse = {}
914 for needed, elfclass in neededs:
915 data_solved = dbconn.resolveNeeded(needed, elfclass = elfclass,
916 extended = True)
917 data_size = len(data_solved)
918 data_solved = set([x for x in data_solved if x[0] \
919 not in idpackages_cache])
920 if not data_solved or (data_size != len(data_solved)):
921 continue
922
923 if self_check:
924 if is_in_content(needed, mycontent):
925 continue
926
927 found = False
928 for data in data_solved:
929 if data[1] in deps_content:
930 found = True
931 break
932 if not found:
933 for data in data_solved:
934 r_idpackage = data[0]
935 key, slot = dbconn.retrieveKeySlot(r_idpackage)
936 if (key, slot) not in scope_cache:
937 if not dbconn.isSystemPackage(r_idpackage):
938 if (needed, elfclass) not in rdepends:
939 rdepends[(needed, elfclass)] = set()
940 if (needed, elfclass) not in idpackage_map:
941 idpackage_map[(needed, elfclass)] = set()
942 keyslot = "%s:%s" % (key, slot,)
943 obj = idpackage_map_reverse.setdefault(
944 keyslot, set())
945 obj.add((needed, elfclass,))
946 rdepends[(needed, elfclass)].add(keyslot)
947 idpackage_map[(needed, elfclass)].add(r_idpackage)
948 rdepends_plain.add(keyslot)
949 idpackages_cache.add(r_idpackage)
950
951
952
953 r_deplist = set()
954 for key in idpackage_map:
955 r_idpackages = idpackage_map.get(key)
956 for r_idpackage in r_idpackages:
957 r_deplist |= dbconn.retrieveDependencies(r_idpackage)
958
959 r_keyslots = set()
960 for r_dep in r_deplist:
961 m_idpackage, m_rc = dbconn.atomMatch(r_dep)
962 if m_rc != 0:
963 continue
964 keyslot = dbconn.retrieveKeySlotAggregated(m_idpackage)
965 if keyslot in rdepends_plain:
966 r_keyslots.add(keyslot)
967
968 rdepends_plain -= r_keyslots
969 for r_keyslot in r_keyslots:
970 keys = [x for x in idpackage_map_reverse.get(keyslot, set()) if \
971 x in rdepends]
972 for key in keys:
973 rdepends[key].discard(r_keyslot)
974 if not rdepends[key]:
975 del rdepends[key]
976
977 return rdepends, rdepends_plain
978
980 """
981 Service method which returns a complete, expanded list of dependencies
982 for the given idpackage on the given entropy.db.EntropyRepository
983 "dbconn" instance.
984
985 @param dbconn: entropy.db.EntropyRepository instance which contains
986 the given idpackage item.
987 @type dbconn: entropy.db.EntropyRepository instance
988 @param idpackage: Entropy database package key
989 @type idpackage: int
990 @keyword atoms: !! return type modifier !! , make method returning
991 a list of atom strings instead of list of db match tuples.
992 @type atoms: bool
993 @return: list of dependencies in form of matching tuple list
994 ( [(idpackage, repoid,) ... ] ) or plain dependency list (if
995 atom == True -- set([atom_string1, atom_string2, atom_string3])
996 @rtype: list or set
997 """
998 mybuffer = self.Lifo()
999 matchcache = set()
1000 depcache = set()
1001 mydeps = dbconn.retrieveDependencies(idpackage)
1002 for mydep in mydeps:
1003 mybuffer.push(mydep)
1004 try:
1005 mydep = mybuffer.pop()
1006 except ValueError:
1007 mydep = None
1008
1009 while mydep:
1010
1011 if mydep in depcache:
1012 try:
1013 mydep = mybuffer.pop()
1014 except ValueError:
1015 break
1016 continue
1017
1018 my_idpackage, my_rc = dbconn.atomMatch(mydep)
1019 if atoms:
1020 matchcache.add(mydep)
1021 else:
1022 matchcache.add(my_idpackage)
1023
1024 if my_idpackage != -1:
1025 owndeps = dbconn.retrieveDependencies(my_idpackage)
1026 for owndep in owndeps:
1027 mybuffer.push(owndep)
1028
1029 depcache.add(mydep)
1030 try:
1031 mydep = mybuffer.pop()
1032 except ValueError:
1033 break
1034
1035
1036 matchcache.discard(-1)
1037 return matchcache
1038
1040 """
1041 Check if the physical Entropy package file contains
1042 a valid Entropy embedded database.
1043
1044 @param pkg_path: path to physical entropy package file
1045 @type pkg_path: string
1046 @return: package validity
1047 @rtype: bool
1048 """
1049 from entropy.db import EntropyRepository, dbapi2
1050 fd, tmp_path = tempfile.mkstemp()
1051 extract_path = self.entropyTools.extract_edb(pkg_path, tmp_path)
1052 if extract_path is None:
1053 if os.path.isfile(tmp_path):
1054 os.remove(tmp_path)
1055 os.close(fd)
1056 return False
1057 try:
1058 dbc = EntropyRepository(
1059 readOnly = False,
1060 dbFile = tmp_path,
1061 dbname = 'qa_testing',
1062 xcache = False,
1063 indexing = False,
1064 skipChecks = False
1065 )
1066 etp_repo_meta = {
1067 'output_interface': self.Output,
1068 }
1069 repo_plug = QAEntropyRepositoryPlugin(self,
1070 metadata = etp_repo_meta)
1071 dbc.add_plugin(repo_plug)
1072
1073 except dbapi2.Error:
1074 os.remove(tmp_path)
1075 os.close(fd)
1076 return False
1077
1078 valid = True
1079 try:
1080 dbc.validateDatabase()
1081 except SystemDatabaseError:
1082 valid = False
1083
1084 if valid:
1085 try:
1086 for idpackage in dbc.listAllIdpackages():
1087 dbc.retrieveContent(idpackage, extended = True,
1088 formatted = True, insert_formatted = True)
1089 except dbapi2.Error:
1090 valid = False
1091
1092 dbc.closeDB()
1093 os.remove(tmp_path)
1094 os.close(fd)
1095
1096 return valid
1097
1099 """
1100 Main method for the execution of QA tests on physical Entropy
1101 package files.
1102
1103 @param package_path: path to physical Entropy package file path
1104 @type package_path: string
1105 @return: True, if all checks passed
1106 @rtype: bool
1107 """
1108
1109 qa_methods = [self.__analyze_package_edb]
1110
1111
1112 for plug_id, plug_inst in self.get_plugins().items():
1113 qa_methods += plug_inst.get_tests()
1114
1115
1116 for method in qa_methods:
1117 qa_rc = method(package_path)
1118 if not qa_rc:
1119 return False
1120 return True
1121
1122
1124
1125 """
1126
1127 Interface used by Entropy Client to remotely send errors via HTTP POST.
1128 Some anonymous info about the running system are collected and sent over,
1129 once the user gives the acknowledgement for this operation.
1130 User should be asked for valid credentials, such as name, surname and email.
1131 This has two advantages: block stupid and lazy people and make possible
1132 for Entropy developers to contact him/her back.
1133 Moreover, the same applies for a simple description. To improve the
1134 ability to debug an issue, it is also asked the user to describe his/her
1135 action prior to the error.
1136
1137 Sample code:
1138
1139 >>> from entropy.qa import ErrorReportInterface
1140 >>> error = ErrorReportInterface('http://url_for_http_post')
1141 >>> error.prepare('traceback_text', 'John Foo', 'john@foo.com',
1142 report_data = 'extra traceback info',
1143 description = 'I was installing foo!')
1144 >>> error.submit()
1145
1146 """
1147
1148 import entropy.tools as entropyTools
1150 """
1151 ErrorReportInterface constructor.
1152
1153 @param post_url: HTTP post url where to submit data
1154 @type post_url: string
1155 """
1156 from entropy.misc import MultipartPostHandler
1157 if sys.hexversion >= 0x3000000:
1158 import urllib.request as urlmod
1159 else:
1160 import urllib2 as urlmod
1161
1162 self.url = post_url
1163 self.opener = urlmod.build_opener(MultipartPostHandler)
1164 self.generated = False
1165 self.params = {}
1166
1167 sys_settings = SystemSettings()
1168 proxy_settings = sys_settings['system']['proxy']
1169 mydict = {}
1170 if proxy_settings['ftp']:
1171 mydict['ftp'] = proxy_settings['ftp']
1172 if proxy_settings['http']:
1173 mydict['http'] = proxy_settings['http']
1174 if mydict:
1175 mydict['username'] = proxy_settings['username']
1176 mydict['password'] = proxy_settings['password']
1177 self.entropyTools.add_proxy_opener(urlmod, mydict)
1178 else:
1179
1180 urlmod._opener = None
1181
1182 - def prepare(self, tb_text, name, email, report_data = "", description = ""):
1183
1184 """
1185 This method must be called prior to submit(). It is used to prepare
1186 and collect system information before the submission.
1187 It is intentionally split from submit() to allow easy reimplementation.
1188
1189 @param tb_text: Python traceback text to send
1190 @type tb_text: string
1191 @param name: submitter name
1192 @type name: string
1193 @param email: submitter email address
1194 @type email: string
1195 @keyword report_data: extra information
1196 @type report_data: string
1197 @keyword description: submitter action description
1198 @type description: string
1199 @return: None
1200 @rtype: None
1201 """
1202 from entropy.tools import getstatusoutput
1203 self.params['arch'] = etpConst['currentarch']
1204 self.params['stacktrace'] = tb_text
1205 self.params['name'] = name
1206 self.params['email'] = email
1207 self.params['version'] = etpConst['entropyversion']
1208 self.params['errordata'] = report_data
1209 self.params['description'] = description
1210 self.params['arguments'] = ' '.join(sys.argv)
1211 self.params['uid'] = etpConst['uid']
1212 self.params['system_version'] = "N/A"
1213 if os.access(etpConst['systemreleasefile'], os.R_OK):
1214 f_rel = open(etpConst['systemreleasefile'], "r")
1215 self.params['system_version'] = f_rel.readline().strip()
1216 f_rel.close()
1217
1218 self.params['processes'] = getstatusoutput('ps auxf')[1]
1219 self.params['lspci'] = getstatusoutput('/usr/sbin/lspci')[1]
1220 self.params['dmesg'] = getstatusoutput('dmesg')[1]
1221 self.params['locale'] = getstatusoutput('locale -v')[1]
1222
1223 self.generated = True
1224
1225
1227 """
1228 Submit collected data remotely via HTTP POST.
1229
1230 @raise PermissionDenied: when prepare() hasn't been called.
1231 @return: None
1232 @rtype: None
1233 """
1234 if self.generated:
1235 result = self.opener.open(self.url, self.params).read()
1236 if result.strip() == "1":
1237 return True
1238 return False
1239 else:
1240 mytxt = _("Not prepared yet")
1241 raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
1242