entropy.core

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework core module}. 10 11 This module contains base classes used by entropy.client, 12 entropy.server and entropy.services. 13 14 "Singleton" is a class that is inherited from singleton objects. 15 16 SystemSettings is a singleton, pluggable interface which contains 17 all the runtime settings (mostly parsed from configuration files 18 and inherited from entropy.const -- which contains almost all the 19 default values). 20 SystemSettings works as a I{dict} object. Due to limitations of 21 multiple inherittance when using the Singleton class, SystemSettings 22 ONLY mimics a I{dict} AND it's not a subclass of it. 23 24 SystemSettingsPlugin is the base class for building valid SystemSettings 25 plugin modules (see entropy.client.interfaces.client or 26 entropy.server.interfaces for working examples). 27 28 """ 29 from __future__ import with_statement 30 import os 31 from entropy.exceptions import IncorrectParameter, SystemDatabaseError 32 from entropy.const import etpConst, etpUi, etpSys, const_setup_perms, \ 33 const_secure_config_file, const_set_nice_level, \ 34 const_extract_cli_repo_params, etpCache 35 from entropy.i18n import _ 36 from threading import RLock 37

38 -class Singleton(object):

39 40 """ 41 If your class wants to become a sexy Singleton, 42 subclass this and replace __init__ with init_singleton 43 """ 44 45 __is_destroyed = False 46 __is_singleton = True

47 - def __new__(cls, *args, **kwds):

48 instance = cls.__dict__.get("__it__") 49 if instance != None: 50 if not instance.is_destroyed(): 51 return instance 52 cls.__it__ = instance = object.__new__(cls) 53 instance.init_singleton(*args, **kwds) 54 return instance

55

56 - def is_destroyed(self):

57 """ 58 In our world, Singleton instances may be destroyed, 59 this is done by setting a private bool var __is_destroyed 60 61 @rtype: bool 62 @return: instance status, if destroyed or not 63 """ 64 return self.__is_destroyed

65

66 - def is_singleton(self):

67 """ 68 Return if the instance is a singleton 69 70 @rtype: bool 71 @return: class singleton property, if singleton or not 72 """ 73 return self.__is_singleton

74

75 -class SystemSettingsPlugin:

76 77 """ 78 79 This is a plugin base class for all SystemSettings plugins. 80 It allows to add extra parsers (though metadata) to 81 SystemSettings. 82 Just inherit from this class and call add_parser to add 83 your custom parsers. 84 SystemSettings will call the parse method, as explained below. 85 86 Sample code: 87 88 >>> # load SystemSettings 89 >>> from entropy.core import SystemSettings, SystemSettingsPlugin 90 >>> system_settings = SystemSettings() 91 >>> class MyPlugin(SystemSettingsPlugin): 92 >>> pass 93 >>> my_plugin = MyPlugin('mystuff', None) 94 >>> def myparsing_function(): 95 >>> return {'abc': 1 } 96 >>> my_plugin.add_parser('parser_no_1', myparsing_function) 97 >>> system_settings.add_plugin(my_plugin) 98 >>> print(system_settings['mystuff']['parser_no_1']) 99 {'abc': 1 } 100 >>> # let's remove it 101 >>> system_settings.remove_plugin('mystuff') # through its plugin_id 102 >>> print(system_settings.get('mystuff')) 103 None 104 105 """ 106

107 - def __init__(self, plugin_id, helper_interface):

108 """ 109 SystemSettingsPlugin constructor. 110 111 @param plugin_id: plugin identifier, must be unique 112 @type plugin_id: string 113 @param helper_interface: any Python object that could 114 be of help to your parsers 115 @type handler_instance: Python object 116 @rtype: None 117 @return: None 118 """ 119 self.__parsers = [] 120 self.__plugin_id = plugin_id 121 self._helper = helper_interface 122 parser_postfix = "_parser" 123 for method in sorted(dir(self)): 124 if method == "add_parser": 125 continue 126 elif method.endswith(parser_postfix) and (method != parser_postfix): 127 parser_id = method[:len(parser_postfix)*-1] 128 self.__parsers.append((parser_id, getattr(self, method),))

129

130 - def get_id(self):

131 """ 132 Returns the unique plugin id passed at construction time. 133 134 @return: plugin identifier 135 @rtype: string 136 """ 137 return self.__plugin_id

138

139 - def add_parser(self, parser_id, parser_callable):

140 """ 141 You must call this method in order to add your custom 142 parsers to the plugin. 143 Please note, if your parser method ends with "_parser" 144 it will be automatically added this way: 145 146 method: foo_parser 147 parser_id => foo 148 method: another_fabulous_parser 149 parser_id => another_fabulous 150 151 @param parser_id: parser identifier, must be unique 152 @type parser_id: string 153 @param parser_callable: any callable function which has 154 the following signature: callable(system_settings_instance) 155 can return True to stop further parsers calls 156 @type parser_callable: callable 157 @return: None 158 @rtype: None 159 """ 160 self.__parsers.append((parser_id, parser_callable,))

161

162 - def parse(self, system_settings_instance):

163 """ 164 This method is called by SystemSettings instance 165 when building its settings metadata. 166 167 Returned data from parser will be put into the SystemSettings 168 dict using plugin_id and parser_id keys. 169 If returned data is None, SystemSettings dict won't be changed. 170 171 @param system_settings_instance: SystemSettings instance 172 @type system_settings_instance: SystemSettings instance 173 @return: None 174 @rtype: None 175 """ 176 plugin_id = self.get_id() 177 for parser_id, parser in self.__parsers: 178 data = parser(system_settings_instance) 179 if data == None: 180 continue 181 if not system_settings_instance.has_key(plugin_id): 182 system_settings_instance[plugin_id] = {} 183 system_settings_instance[plugin_id][parser_id] = data

184

185 - def post_setup(self, system_settings_instance):

186 """ 187 This method is called by SystemSettings instance 188 after having built all the SystemSettings metadata. 189 You can reimplement this and hook your refinement code 190 into this method. 191 192 @param system_settings_instance: SystemSettings instance 193 @type system_settings_instance: SystemSettings instance 194 @return: None 195 @rtype: None 196 """ 197 pass

198

199 -class SystemSettings(Singleton):

200 201 """ 202 This is the place where all the Entropy settings are stored if 203 they are not considered instance constants (etpConst). 204 For example, here we store package masking cache information and 205 settings, client-side, server-side and services settings. 206 Also, this class mimics a dictionary (even if not inheriting it 207 due to development choices). 208 209 Sample code: 210 211 >>> from entropy.core import SystemSettings 212 >>> system_settings = SystemSettings() 213 >>> system_settings.clear() 214 >>> system_settings.destroy() 215 216 """ 217 218 import entropy.tools as entropyTools

219 - def init_singleton(self):

220 221 """ 222 Replaces __init__ because SystemSettings is a Singleton. 223 see Singleton API reference for more information. 224 225 """ 226 227 from entropy.cache import EntropyCacher 228 self.__cacher = EntropyCacher() 229 self.__data = {} 230 self.__is_destroyed = False 231 self.__mutex = RLock() # reentrant lock on purpose 232 233 self.__plugins = {} 234 self.__setting_files_order = [] 235 self.__setting_files_pre_run = [] 236 self.__setting_files = {} 237 self.__mtime_files = {} 238 self.__persistent_settings = { 239 'pkg_masking_reasons': { 240 0: _('reason not available'), 241 1: _('user package.mask'), 242 2: _('system keywords'), 243 3: _('user package.unmask'), 244 4: _('user repo package.keywords (all packages)'), 245 5: _('user repo package.keywords'), 246 6: _('user package.keywords'), 247 7: _('completely masked'), 248 8: _('repository general packages.db.mask'), 249 10: _('user license.mask'), 250 11: _('user live unmask'), 251 12: _('user live mask'), 252 }, 253 'pkg_masking_reference': { 254 'reason_not_avail': 0, 255 'user_package_mask': 1, 256 'system_keyword': 2, 257 'user_package_unmask': 3, 258 'user_repo_package_keywords_all': 4, 259 'user_repo_package_keywords': 5, 260 'user_package_keywords': 6, 261 'completely_masked': 7, 262 'repository_packages_db_mask': 8, 263 'repository_in_branch_pacakges_db_mask': 9, 264 'user_license_mask': 10, 265 'user_live_unmask': 11, 266 'user_live_mask': 12, 267 }, 268 'backed_up': {}, 269 # package masking, live 270 'live_packagemasking': { 271 'unmask_matches': set(), 272 'mask_matches': set(), 273 }, 274 } 275 276 self.__setup_const() 277 self.__scan()

278

279 - def destroy(self):

280 """ 281 Overloaded method from Singleton. 282 "Destroys" the instance. 283 284 @return: None 285 @rtype: None 286 """ 287 with self.__mutex: 288 self.__is_destroyed = True

289

290 - def add_plugin(self, system_settings_plugin_instance):

291 """ 292 This method lets you add custom parsers to SystemSettings. 293 Mind that you are responsible of handling your plugin instance 294 and remove it before it is destroyed. You can remove the plugin 295 instance at any time by issuing remove_plugin. 296 Every add_plugin or remove_plugin method will also issue clear() 297 for you. This could be bad and it might be removed in future. 298 299 @param plugin_id: plugin identifier 300 @type plugin_id: string 301 @param system_settings_plugin_instance: valid SystemSettingsPlugin 302 instance 303 @type system_settings_plugin_instance: SystemSettingsPlugin instance 304 @return: None 305 @rtype: None 306 """ 307 inst = system_settings_plugin_instance 308 if not isinstance(inst,SystemSettingsPlugin): 309 raise AttributeError("SystemSettings: expected valid " + \ 310 "SystemSettingsPlugin instance") 311 with self.__mutex: 312 self.__plugins[inst.get_id()] = inst 313 self.clear()

314

315 - def remove_plugin(self, plugin_id):

316 """ 317 This method lets you remove previously added custom parsers from 318 SystemSettings through its plugin identifier. If plugin_id is not 319 available, KeyError exception will be raised. 320 Every add_plugin or remove_plugin method will also issue clear() 321 for you. This could be bad and it might be removed in future. 322 323 @param plugin_id: plugin identifier 324 @type plugin_id: basestring 325 @return: None 326 @rtype: None 327 """ 328 with self.__mutex: 329 del self.__plugins[plugin_id] 330 self.clear()

331

332 - def __setup_const(self):

333 334 """ 335 Internal method. Does constants initialization. 336 337 @return: None 338 @rtype: None 339 """ 340 341 del self.__setting_files_order[:] 342 del self.__setting_files_pre_run[:] 343 self.__setting_files.clear() 344 self.__mtime_files.clear() 345 346 self.__setting_files.update({ 347 # keywording configuration files 348 'keywords': etpConst['confpackagesdir']+"/package.keywords", 349 # unmasking configuration files 350 'unmask': etpConst['confpackagesdir']+"/package.unmask", 351 # masking configuration files 352 'mask': etpConst['confpackagesdir']+"/package.mask", 353 # satisfied packages configuration file 354 'satisfied': etpConst['confpackagesdir']+"/package.satisfied", 355 # masking configuration files 356 'license_mask': etpConst['confpackagesdir']+"/license.mask", 357 'system_mask': etpConst['confpackagesdir']+"/system.mask", 358 'system_dirs': etpConst['confdir']+"/fsdirs.conf", 359 'system_dirs_mask': etpConst['confdir']+"/fsdirsmask.conf", 360 'system_rev_symlinks': etpConst['confdir']+"/fssymlinks.conf", 361 'broken_syms': etpConst['confdir']+"/brokensyms.conf", 362 'broken_libs_mask': etpConst['confdir']+"/brokenlibsmask.conf", 363 'hw_hash': etpConst['confdir']+"/.hw.hash", 364 'socket_service': etpConst['socketconf'], 365 'system': etpConst['entropyconf'], 366 'repositories': etpConst['repositoriesconf'], 367 'system_package_sets': {}, 368 }) 369 self.__setting_files_order.extend([ 370 'keywords', 'unmask', 'mask', 'satisfied', 'license_mask', 371 'system_mask', 'system_package_sets', 'system_dirs', 372 'system_dirs_mask', 'socket_service', 'system', 373 'system_rev_symlinks', 'hw_hash', 'broken_syms', 'broken_libs_mask' 374 ]) 375 self.__setting_files_pre_run.extend(['repositories']) 376 377 dmp_dir = etpConst['dumpstoragedir'] 378 self.__mtime_files.update({ 379 'keywords_mtime': dmp_dir+"/keywords.mtime", 380 'unmask_mtime': dmp_dir+"/unmask.mtime", 381 'mask_mtime': dmp_dir+"/mask.mtime", 382 'satisfied_mtime': dmp_dir+"/satisfied.mtime", 383 'license_mask_mtime': dmp_dir+"/license_mask.mtime", 384 'system_mask_mtime': dmp_dir+"/system_mask.mtime", 385 })

386 387

388 - def __scan(self):

389 390 """ 391 Internal method. Scan settings and fill variables. 392 393 @return: None 394 @rtype: None 395 """ 396 397 def enforce_persistent(): 398 # merge persistent settings back 399 self.__data.update(self.__persistent_settings) 400 # restore backed-up settings 401 self.__data.update(self.__persistent_settings['backed_up'].copy())

402 403 self.__parse() 404 enforce_persistent() 405 406 # plugins support 407 for plugin_id in sorted(self.__plugins): 408 self.__plugins[plugin_id].parse(self) 409 410 enforce_persistent() 411 412 # run post-SystemSettings setup, plugins hook 413 for plugin_id in sorted(self.__plugins): 414 self.__plugins[plugin_id].post_setup(self)

415

416 - def __setitem__(self, mykey, myvalue):

417 """ 418 dict method. See Python dict API reference. 419 """ 420 with self.__mutex: 421 # backup here too 422 if self.__persistent_settings.has_key(mykey): 423 self.__persistent_settings[mykey] = myvalue 424 self.__data[mykey] = myvalue

425

426 - def __getitem__(self, mykey):

427 """ 428 dict method. See Python dict API reference. 429 """ 430 with self.__mutex: 431 return self.__data[mykey]

432

433 - def __delitem__(self, mykey):

434 """ 435 dict method. See Python dict API reference. 436 """ 437 with self.__mutex: 438 del self.__data[mykey]

439

440 - def __iter__(self):

441 """ 442 dict method. See Python dict API reference. 443 """ 444 with self.__mutex: 445 return iter(self.__data)

446

447 - def __contains__(self, item):

448 """ 449 dict method. See Python dict API reference. 450 """ 451 with self.__mutex: 452 return item in self.__data

453

454 - def __cmp__(self, other):

455 """ 456 dict method. See Python dict API reference. 457 """ 458 with self.__mutex: 459 return cmp(self.__data, other)

460

461 - def __hash__(self):

462 """ 463 dict method. See Python dict API reference. 464 """ 465 with self.__mutex: 466 return hash(self.__data)

467

468 - def __len__(self):

469 """ 470 dict method. See Python dict API reference. 471 """ 472 with self.__mutex: 473 return len(self.__data)

474

475 - def get(self, mykey, alt_obj = None):

476 """ 477 dict method. See Python dict API reference. 478 """ 479 with self.__mutex: 480 return self.__data.get(mykey, alt_obj)

481

482 - def has_key(self, mykey):

483 """ 484 dict method. See Python dict API reference. 485 """ 486 with self.__mutex: 487 return self.__data.has_key(mykey)

488

489 - def copy(self):

490 """ 491 dict method. See Python dict API reference. 492 """ 493 with self.__mutex: 494 return self.__data.copy()

495

496 - def fromkeys(self, seq, val = None):

497 """ 498 dict method. See Python dict API reference. 499 """ 500 with self.__mutex: 501 return self.__data.fromkeys(seq, val)

502

503 - def items(self):

504 """ 505 dict method. See Python dict API reference. 506 """ 507 with self.__mutex: 508 return self.__data.items()

509

510 - def iteritems(self):

511 """ 512 dict method. See Python dict API reference. 513 """ 514 with self.__mutex: 515 return self.__data.iteritems()

516

517 - def iterkeys(self):

518 """ 519 dict method. See Python dict API reference. 520 """ 521 with self.__mutex: 522 return self.__data.iterkeys()

523

524 - def keys(self):

525 """ 526 dict method. See Python dict API reference. 527 """ 528 with self.__mutex: 529 return self.__data.keys()

530

531 - def pop(self, mykey, default = None):

532 """ 533 dict method. See Python dict API reference. 534 """ 535 with self.__mutex: 536 return self.__data.pop(mykey, default)

537

538 - def popitem(self):

539 """ 540 dict method. See Python dict API reference. 541 """ 542 with self.__mutex: 543 return self.__data.popitem()

544

545 - def setdefault(self, mykey, default = None):

546 """ 547 dict method. See Python dict API reference. 548 """ 549 with self.__mutex: 550 return self.__data.setdefault(mykey, default)

551

552 - def update(self, kwargs):

553 """ 554 dict method. See Python dict API reference. 555 """ 556 with self.__mutex: 557 return self.__data.update(kwargs)

558

559 - def values(self):

560 """ 561 dict method. See Python dict API reference. 562 """ 563 with self.__mutex: 564 return self.__data.values()

565

566 - def clear(self):

567 """ 568 dict method. See Python dict API reference. 569 Settings are also re-initialized here. 570 571 @return None 572 """ 573 with self.__mutex: 574 self.__data.clear() 575 self.__setup_const() 576 self.__scan()

577

578 - def set_persistent_setting(self, persistent_dict):

579 """ 580 Make metadata persistent, the input dict will be merged 581 with the base one at every reset call (clear()). 582 583 @param persistent_dict: dictionary to merge 584 @type persistent_dict: dict 585 586 @return: None 587 @rtype: None 588 """ 589 with self.__mutex: 590 self.__persistent_settings.update(persistent_dict)

591

592 - def unset_persistent_setting(self, persistent_key):

593 """ 594 Remove dict key from persistent dictionary 595 596 @param persistent_key: key to remove 597 @type persistent_dict: dict 598 599 @return: None 600 @rtype: None 601 """ 602 with self.__mutex: 603 del self.__persistent_settings[persistent_key] 604 del self.__data[persistent_key]

605

606 - def __setup_package_sets_vars(self):

607 608 """ 609 This function setups the *files* dictionary about package sets 610 that will be read and parsed afterwards by the respective 611 internal parser. 612 613 @return: None 614 @rtype: None 615 """ 616 617 # user defined package sets 618 sets_dir = etpConst['confsetsdir'] 619 pkg_set_data = {} 620 if (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 621 set_files = [x for x in os.listdir(sets_dir) if \ 622 (os.path.isfile(os.path.join(sets_dir, x)) and \ 623 os.access(os.path.join(sets_dir, x),os.R_OK))] 624 for set_file in set_files: 625 try: 626 set_file = str(set_file) 627 except (UnicodeDecodeError, UnicodeEncodeError,): 628 continue 629 pkg_set_data[set_file] = os.path.join(sets_dir, set_file) 630 self.__setting_files['system_package_sets'].update(pkg_set_data)

631

632 - def __parse(self):

633 """ 634 This is the main internal parsing method. 635 *files* and *mtimes* dictionaries are prepared and 636 parsed just a few lines later. 637 638 @return: None 639 @rtype: None 640 """ 641 # some parsers must be run BEFORE everything: 642 for item in self.__setting_files_pre_run: 643 myattr = '_%s_parser' % (item,) 644 if not hasattr(self, myattr): 645 continue 646 func = getattr(self, myattr) 647 self.__data[item] = func() 648 649 # parse main settings 650 self.__setup_package_sets_vars() 651 652 data = {} 653 for item in self.__setting_files_order: 654 myattr = '_%s_parser' % (item,) 655 if not hasattr(self, myattr): 656 continue 657 func = getattr(self, myattr) 658 self.__data[item] = func()

659

660 - def get_setting_files_data(self):

661 """ 662 Return a copy of the internal *files* dictionary. 663 This dict contains config file paths and their identifiers. 664 665 @return: dict __setting_files 666 @rtype: dict 667 """ 668 with self.__mutex: 669 return self.__setting_files.copy()

670

671 - def _keywords_parser(self):

672 """ 673 Parser returning package keyword masking metadata 674 read from package.keywords file. 675 This file contains package mask or unmask directives 676 based on package keywords. 677 678 @return: parsed metadata 679 @rtype: dict 680 """ 681 # merge universal keywords 682 data = { 683 'universal': set(), 684 'packages': {}, 685 'repositories': {}, 686 } 687 688 self.validate_entropy_cache(self.__setting_files['keywords'], 689 self.__mtime_files['keywords_mtime']) 690 content = [x.split() for x in \ 691 self.__generic_parser(self.__setting_files['keywords']) \ 692 if len(x.split()) < 4] 693 for keywordinfo in content: 694 # skip wrong lines 695 if len(keywordinfo) > 3: 696 continue 697 # inversal keywording, check if it's not repo= 698 if len(keywordinfo) == 1: 699 if keywordinfo[0].startswith("repo="): 700 continue 701 # convert into entropy format 702 if keywordinfo[0] == "**": 703 keywordinfo[0] = "" 704 data['universal'].add(keywordinfo[0]) 705 continue 706 # inversal keywording, check if it's not repo= 707 if len(keywordinfo) in (2, 3,): 708 # repo=? 709 if keywordinfo[0].startswith("repo="): 710 continue 711 # add to repo? 712 items = keywordinfo[1:] 713 # convert into entropy format 714 if keywordinfo[0] == "**": 715 keywordinfo[0] = "" 716 reponame = [x for x in items if x.startswith("repo=") \ 717 and (len(x.split("=")) == 2)] 718 if reponame: 719 reponame = reponame[0].split("=")[1] 720 if reponame not in data['repositories']: 721 data['repositories'][reponame] = {} 722 # repository unmask or package in repository unmask? 723 if keywordinfo[0] not in data['repositories'][reponame]: 724 data['repositories'][reponame][keywordinfo[0]] = set() 725 if len(items) == 1: 726 # repository unmask 727 data['repositories'][reponame][keywordinfo[0]].add('*') 728 elif "*" not in \ 729 data['repositories'][reponame][keywordinfo[0]]: 730 731 item = [x for x in items if not x.startswith("repo=")] 732 data['repositories'][reponame][keywordinfo[0]].add( 733 item[0]) 734 elif len(items) == 2: 735 # it's going to be a faulty line!!?? 736 # can't have two items and no repo= 737 continue 738 else: 739 # add keyword to packages 740 if keywordinfo[0] not in data['packages']: 741 data['packages'][keywordinfo[0]] = set() 742 data['packages'][keywordinfo[0]].add(items[0]) 743 744 # merge universal keywords 745 etpConst['keywords'].clear() 746 etpConst['keywords'].update(etpSys['keywords']) 747 for keyword in data['universal']: 748 etpConst['keywords'].add(keyword) 749 750 return data

751 752

753 - def _unmask_parser(self):

754 """ 755 Parser returning package unmasking metadata read from 756 package.unmask file. 757 This file contains package unmask directives, allowing 758 to enable experimental or *secret* packages. 759 760 @return: parsed metadata 761 @rtype: dict 762 """ 763 self.validate_entropy_cache(self.__setting_files['unmask'], 764 self.__mtime_files['unmask_mtime']) 765 return self.__generic_parser(self.__setting_files['unmask'])

766

767 - def _mask_parser(self):

768 """ 769 Parser returning package masking metadata read from 770 package.mask file. 771 This file contains package mask directives, allowing 772 to disable experimental or *secret* packages. 773 774 @return: parsed metadata 775 @rtype: dict 776 """ 777 self.validate_entropy_cache(self.__setting_files['mask'], 778 self.__mtime_files['mask_mtime']) 779 return self.__generic_parser(self.__setting_files['mask'])

780

781 - def _satisfied_parser(self):

782 """ 783 Parser returning package forced satisfaction metadata 784 read from package.satisfied file. 785 This file contains packages which updates as dependency are 786 filtered out. 787 788 @return: parsed metadata 789 @rtype: dict 790 """ 791 self.validate_entropy_cache(self.__setting_files['satisfied'], 792 self.__mtime_files['satisfied_mtime']) 793 return self.__generic_parser(self.__setting_files['satisfied'])

794

795 - def _system_mask_parser(self):

796 """ 797 Parser returning system packages mask metadata read from 798 package.system_mask file. 799 This file contains packages that should be always kept 800 installed, extending the already defined (in repository database) 801 set of atoms. 802 803 @return: parsed metadata 804 @rtype: dict 805 """ 806 self.validate_entropy_cache(self.__setting_files['system_mask'], 807 self.__mtime_files['system_mask_mtime']) 808 return self.__generic_parser(self.__setting_files['system_mask'])

809

810 - def _license_mask_parser(self):

811 """ 812 Parser returning packages masked by license metadata read from 813 license.mask file. 814 Packages shipped with licenses listed there will be masked. 815 816 @return: parsed metadata 817 @rtype: dict 818 """ 819 self.validate_entropy_cache(self.__setting_files['license_mask'], 820 self.__mtime_files['license_mask_mtime']) 821 return self.__generic_parser(self.__setting_files['license_mask'])

822

823 - def _system_package_sets_parser(self):

824 """ 825 Parser returning system defined package sets read from 826 /etc/entropy/packages/sets. 827 828 @return: parsed metadata 829 @rtype: dict 830 """ 831 data = {} 832 for set_name in self.__setting_files['system_package_sets']: 833 set_filepath = self.__setting_files['system_package_sets'][set_name] 834 set_elements = self.entropyTools.extract_packages_from_set_file( 835 set_filepath) 836 if set_elements: 837 data[set_name] = set_elements.copy() 838 return data

839

840 - def _system_dirs_parser(self):

841 """ 842 Parser returning directories considered part of the base system. 843 844 @return: parsed metadata 845 @rtype: dict 846 """ 847 return self.__generic_parser(self.__setting_files['system_dirs'])

848

849 - def _system_dirs_mask_parser(self):

850 """ 851 Parser returning directories NOT considered part of the base system. 852 Settings here overlay system_dirs_parser. 853 854 @return: parsed metadata 855 @rtype: dict 856 """ 857 return self.__generic_parser(self.__setting_files['system_dirs_mask'])

858

859 - def _broken_syms_parser(self):

860 """ 861 Parser returning a list of shared objects symbols that can be used by 862 QA tools to scan the filesystem or a subset of it. 863 864 @return: parsed metadata 865 @rtype: dict 866 """ 867 return self.__generic_parser(self.__setting_files['broken_syms'])

868

869 - def _broken_libs_mask_parser(self):

870 """ 871 Parser returning a list of broken shared libraries which are 872 always considered sane. 873 874 @return: parsed metadata 875 @rtype: dict 876 """ 877 return self.__generic_parser(self.__setting_files['broken_libs_mask'])

878

879 - def _hw_hash_parser(self):

880 """ 881 Hardware hash metadata parser and generator. It returns a theorically 882 unique SHA256 hash bound to the computer running this Framework. 883 884 @return: string containing SHA256 hexdigest 885 @rtype: string 886 """ 887 hw_hash_file = self.__setting_files['hw_hash'] 888 if os.access(hw_hash_file, os.R_OK | os.F_OK): 889 hash_f = open(hw_hash_file, "r") 890 hash_data = hash_f.readline().strip() 891 hash_f.close() 892 return hash_data 893 894 hash_file_dir = os.path.dirname(hw_hash_file) 895 hw_hash_exec = etpConst['etp_hw_hash_gen'] 896 if os.access(hash_file_dir, os.W_OK) and \ 897 os.access(hw_hash_exec, os.X_OK | os.F_OK | os.R_OK): 898 pipe = os.popen('{ ' + hw_hash_exec + '; } 2>&1', 'r') 899 hash_data = pipe.read().strip() 900 sts = pipe.close() 901 if sts != None: 902 return None 903 hash_f = open(hw_hash_file, "w") 904 hash_f.write(hash_data) 905 hash_f.flush() 906 hash_f.close() 907 return hash_data

908

909 - def _system_rev_symlinks_parser(self):

910 """ 911 Parser returning important system symlinks mapping. For example: 912 {'/usr/lib': ['/usr/lib64']} 913 Useful for reverse matching files belonging to packages. 914 915 @return: dict containing the mapping 916 @rtype: dict 917 """ 918 setting_file = self.__setting_files['system_rev_symlinks'] 919 raw_data = self.__generic_parser(setting_file) 920 data = {} 921 for line in raw_data: 922 line = line.split() 923 if len(line) < 2: 924 continue 925 data[line[0]] = frozenset(line[1:]) 926 return data

927

928 - def _socket_service_parser(self):

929 """ 930 Parses socket service configuration file. 931 This file contains information about Entropy remote service ports 932 and SSL. 933 934 @return: parsed metadata 935 @rtype: dict 936 """ 937 938 data = etpConst['socket_service'].copy() 939 940 sock_conf = self.__setting_files['socket_service'] 941 if not (os.path.isfile(sock_conf) and \ 942 os.access(sock_conf,os.R_OK)): 943 return data 944 945 socket_f = open(sock_conf,"r") 946 socketconf = [x.strip() for x in socket_f.readlines() if \ 947 x.strip() and not x.strip().startswith("#")] 948 socket_f.close() 949 950 for line in socketconf: 951 952 split_line = line.split("|") 953 split_line_len = len(split_line) 954 955 if line.startswith("listen|") and (split_line_len > 1): 956 957 item = split_line[1].strip() 958 if item: 959 data['hostname'] = item 960 961 elif line.startswith("listen-port|") and \ 962 (split_line_len > 1): 963 964 item = split_line[1].strip() 965 try: 966 item = int(item) 967 data['port'] = item 968 except ValueError: 969 continue 970 971 elif line.startswith("listen-timeout|") and \ 972 (split_line_len > 1): 973 974 item = split_line[1].strip() 975 try: 976 item = int(item) 977 data['timeout'] = item 978 except ValueError: 979 continue 980 981 elif line.startswith("listen-threads|") and \ 982 (split_line_len > 1): 983 984 item = split_line[1].strip() 985 try: 986 item = int(item) 987 data['threads'] = item 988 except ValueError: 989 continue 990 991 elif line.startswith("session-ttl|") and \ 992 (split_line_len > 1): 993 994 item = split_line[1].strip() 995 try: 996 item = int(item) 997 data['session_ttl'] = item 998 except ValueError: 999 continue 1000 1001 elif line.startswith("max-connections|") and \ 1002 (split_line_len > 1): 1003 1004 item = split_line[1].strip() 1005 try: 1006 item = int(item) 1007 data['max_connections'] = item 1008 except ValueError: 1009 continue 1010 1011 elif line.startswith("ssl-port|") and \ 1012 (split_line_len > 1): 1013 1014 item = split_line[1].strip() 1015 try: 1016 item = int(item) 1017 data['ssl_port'] = item 1018 except ValueError: 1019 continue 1020 1021 elif line.startswith("disabled-commands|") and \ 1022 (split_line_len > 1): 1023 1024 disabled_cmds = split_line[1].strip().split() 1025 for disabled_cmd in disabled_cmds: 1026 data['disabled_cmds'].add(disabled_cmd) 1027 1028 elif line.startswith("ip-blacklist|") and \ 1029 (split_line_len > 1): 1030 1031 ips_blacklist = split_line[1].strip().split() 1032 for ip_blacklist in ips_blacklist: 1033 data['ip_blacklist'].add(ip_blacklist) 1034 1035 return data

1036

1037 - def _system_parser(self):

1038 1039 """ 1040 Parses Entropy system configuration file. 1041 1042 @return: parsed metadata 1043 @rtype: dict 1044 """ 1045 1046 data = {} 1047 data['proxy'] = etpConst['proxy'].copy() 1048 data['name'] = etpConst['systemname'] 1049 data['log_level'] = etpConst['entropyloglevel'] 1050 1051 etp_conf = self.__setting_files['system'] 1052 if not os.path.isfile(etp_conf) and \ 1053 os.access(etp_conf,os.R_OK): 1054 return data 1055 1056 const_secure_config_file(etp_conf) 1057 entropy_f = open(etp_conf,"r") 1058 entropyconf = [x.strip() for x in entropy_f.readlines() if \ 1059 x.strip() and not x.strip().startswith("#")] 1060 entropy_f.close() 1061 1062 for line in entropyconf: 1063 1064 split_line = line.split("|") 1065 split_line_len = len(split_line) 1066 1067 if line.startswith("loglevel|") and \ 1068 (len(line.split("loglevel|")) == 2): 1069 1070 loglevel = line.split("loglevel|")[1] 1071 try: 1072 loglevel = int(loglevel) 1073 except ValueError: 1074 pass 1075 if (loglevel > -1) and (loglevel < 3): 1076 data['log_level'] = loglevel 1077 1078 elif line.startswith("ftp-proxy|") and \ 1079 (split_line_len == 2): 1080 1081 ftpproxy = split_line[1].strip().split() 1082 if ftpproxy: 1083 data['proxy']['ftp'] = ftpproxy[-1] 1084 1085 elif line.startswith("http-proxy|") and \ 1086 (split_line_len == 2): 1087 1088 httpproxy = split_line[1].strip().split() 1089 if httpproxy: 1090 data['proxy']['http'] = httpproxy[-1] 1091 1092 elif line.startswith("proxy-username|") and \ 1093 (split_line_len == 2): 1094 1095 httpproxy = split_line[1].strip().split() 1096 if httpproxy: 1097 data['proxy']['username'] = httpproxy[-1] 1098 1099 elif line.startswith("proxy-password|") and \ 1100 (split_line_len == 2): 1101 1102 httpproxy = split_line[1].strip().split() 1103 if httpproxy: 1104 data['proxy']['password'] = httpproxy[-1] 1105 1106 elif line.startswith("system-name|") and \ 1107 (split_line_len == 2): 1108 1109 data['name'] = split_line[1].strip() 1110 1111 elif line.startswith("nice-level|") and \ 1112 (split_line_len == 2): 1113 1114 mylevel = split_line[1].strip() 1115 try: 1116 mylevel = int(mylevel) 1117 if (mylevel >= -19) and (mylevel <= 19): 1118 const_set_nice_level(mylevel) 1119 except (ValueError,): 1120 continue 1121 1122 return data

1123

1124 - def _repositories_parser(self):

1125 1126 """ 1127 Setup Entropy Client repository settings reading them from 1128 the relative config file specified in etpConst['repositoriesconf'] 1129 1130 @return: parsed metadata 1131 @rtype: dict 1132 """ 1133 1134 data = { 1135 'available': {}, 1136 'excluded': {}, 1137 'order': [], 1138 'product': etpConst['product'], 1139 'branch': etpConst['branch'], 1140 'default_repository': etpConst['officialrepositoryid'], 1141 'transfer_limit': etpConst['downloadspeedlimit'], 1142 'security_advisories_url': etpConst['securityurl'], 1143 } 1144 1145 repo_conf = etpConst['repositoriesconf'] 1146 if not (os.path.isfile(repo_conf) and os.access(repo_conf, os.R_OK)): 1147 return data 1148 1149 repo_f = open(repo_conf,"r") 1150 repositoriesconf = [x.strip() for x in repo_f.readlines() if x.strip()] 1151 repo_f.close() 1152 1153 # setup product and branch first 1154 for line in repositoriesconf: 1155 1156 split_line = line.split("|") 1157 split_line_len = len(split_line) 1158 1159 if (line.find("product|") != -1) and \ 1160 (not line.startswith("#")) and (split_line_len == 2): 1161 1162 data['product'] = split_line[1] 1163 1164 elif (line.find("branch|") != -1) and \ 1165 (not line.startswith("#")) and (split_line_len == 2): 1166 1167 branch = split_line[1].strip() 1168 data['branch'] = branch 1169 if not os.path.isdir(etpConst['packagesbindir']+"/"+branch) \ 1170 and (etpConst['uid'] == 0): 1171 1172 try: 1173 os.makedirs(etpConst['packagesbindir']+"/"+branch) 1174 except (OSError, IOError,): 1175 continue 1176 1177 for line in repositoriesconf: 1178 1179 split_line = line.split("|") 1180 split_line_len = len(split_line) 1181 1182 # populate data['available'] 1183 if (line.find("repository|") != -1) and (split_line_len == 5): 1184 1185 excluded = False 1186 my_repodata = data['available'] 1187 if line.startswith("##"): 1188 continue 1189 elif line.startswith("#"): 1190 excluded = True 1191 my_repodata = data['excluded'] 1192 line = line[1:] 1193 1194 reponame, repodata = const_extract_cli_repo_params(line, 1195 data['branch'], data['product']) 1196 if my_repodata.has_key(reponame): 1197 1198 my_repodata[reponame]['plain_packages'].extend( 1199 repodata['plain_packages']) 1200 my_repodata[reponame]['packages'].extend( 1201 repodata['packages']) 1202 1203 if (not my_repodata[reponame]['plain_database']) and \ 1204 repodata['plain_database']: 1205 1206 my_repodata[reponame]['plain_database'] = \ 1207 repodata['plain_database'] 1208 my_repodata[reponame]['database'] = \ 1209 repodata['database'] 1210 my_repodata[reponame]['dbrevision'] = \ 1211 repodata['dbrevision'] 1212 my_repodata[reponame]['dbcformat'] = \ 1213 repodata['dbcformat'] 1214 else: 1215 1216 my_repodata[reponame] = repodata.copy() 1217 if not excluded: 1218 data['order'].append(reponame) 1219 1220 elif (line.find("officialrepositoryid|") != -1) and \ 1221 (not line.startswith("#")) and (split_line_len == 2): 1222 1223 officialreponame = split_line[1] 1224 data['default_repository'] = officialreponame 1225 1226 elif (line.find("downloadspeedlimit|") != -1) and \ 1227 (not line.startswith("#")) and (split_line_len == 2): 1228 1229 try: 1230 myval = int(split_line[1]) 1231 if myval > 0: 1232 data['transfer_limit'] = myval 1233 else: 1234 data['transfer_limit'] = None 1235 except (ValueError, IndexError,): 1236 data['transfer_limit'] = None 1237 1238 elif (line.find("securityurl|") != -1) and \ 1239 (not line.startswith("#")) and (split_line_len == 2): 1240 1241 try: 1242 data['security_advisories_url'] = split_line[1] 1243 except (IndexError, ValueError, TypeError,): 1244 continue 1245 1246 return data

1247

1248 - def _clear_repository_cache(self, repoid = None):

1249 """ 1250 Internal method, go away! 1251 """ 1252 self.__cacher.discard() 1253 self._clear_dump_cache(etpCache['world_available']) 1254 self._clear_dump_cache(etpCache['world_update']) 1255 self._clear_dump_cache(etpCache['critical_update']) 1256 self._clear_dump_cache(etpCache['check_package_update']) 1257 self._clear_dump_cache(etpCache['filter_satisfied_deps']) 1258 self._clear_dump_cache(etpCache['atomMatch']) 1259 self._clear_dump_cache(etpCache['dep_tree']) 1260 if repoid != None: 1261 self._clear_dump_cache("%s/%s%s/" % ( 1262 etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) 1263 self._clear_dump_cache("%s/%s%s/" % ( 1264 etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,))

1265

1266 - def _clear_dump_cache(self, dump_name, skip = []):

1267 """ 1268 Internal method, go away! 1269 """ 1270 dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) 1271 dump_dir = os.path.dirname(dump_path) 1272 #dump_file = os.path.basename(dump_path) 1273 for currentdir, subdirs, files in os.walk(dump_dir): 1274 path = os.path.join(dump_dir,currentdir) 1275 if skip: 1276 found = False 1277 for myskip in skip: 1278 if path.find(myskip) != -1: 1279 found = True 1280 break 1281 if found: continue 1282 for item in files: 1283 if item.endswith(etpConst['cachedumpext']): 1284 item = os.path.join(path,item) 1285 try: 1286 os.remove(item) 1287 except (OSError, IOError,): 1288 pass 1289 try: 1290 if not os.listdir(path): 1291 os.rmdir(path) 1292 except (OSError, IOError,): 1293 pass

1294

1295 - def __generic_parser(self, filepath):

1296 """ 1297 Internal method. This is the generic file parser here. 1298 1299 @param filepath: valid path 1300 @type filepath: string 1301 @return: raw text extracted from file 1302 @rtype: list 1303 """ 1304 return self.entropyTools.generic_file_content_parser(filepath)

1305

1306 - def __remove_repo_cache(self, repoid = None):

1307 """ 1308 Internal method. Remove repository cache, because not valid anymore. 1309 1310 @keyword repoid: repository identifier or None 1311 @type repoid: string or None 1312 @return: None 1313 @rtype: None 1314 """ 1315 if os.path.isdir(etpConst['dumpstoragedir']): 1316 if repoid: 1317 self._clear_repository_cache(repoid = repoid) 1318 return 1319 for repoid in self['repositories']['order']: 1320 self._clear_repository_cache(repoid = repoid) 1321 else: 1322 try: 1323 os.makedirs(etpConst['dumpstoragedir']) 1324 except IOError, e: 1325 if e.errno == 30: # readonly filesystem 1326 etpUi['pretend'] = True 1327 return 1328 except OSError: 1329 return

1330

1331 - def __save_file_mtime(self, toread, tosaveinto):

1332 """ 1333 Internal method. Save mtime of a file to another file. 1334 1335 @param toread: file path to read 1336 @type toread: string 1337 @param tosaveinto: path where to save retrieved mtime information 1338 @type tosaveinto: string 1339 @return: None 1340 @rtype: None 1341 """ 1342 if not os.path.isfile(toread): 1343 currmtime = 0.0 1344 else: 1345 currmtime = os.path.getmtime(toread) 1346 1347 if not os.path.isdir(etpConst['dumpstoragedir']): 1348 try: 1349 os.makedirs(etpConst['dumpstoragedir'], 0775) 1350 const_setup_perms(etpConst['dumpstoragedir'], 1351 etpConst['entropygid']) 1352 except IOError, e: 1353 if e.errno == 30: # readonly filesystem 1354 etpUi['pretend'] = True 1355 return 1356 except (OSError,), e: 1357 # unable to create the storage directory 1358 # useless to continue 1359 return 1360 1361 try: 1362 mtime_f = open(tosaveinto,"w") 1363 except IOError, e: # unable to write? 1364 if e.errno == 30: # readonly filesystem 1365 etpUi['pretend'] = True 1366 return 1367 else: 1368 mtime_f.write(str(currmtime)) 1369 mtime_f.flush() 1370 mtime_f.close() 1371 os.chmod(tosaveinto, 0664) 1372 if etpConst['entropygid'] != None: 1373 os.chown(tosaveinto, 0, etpConst['entropygid'])

1374 1375

1376 - def validate_entropy_cache(self, settingfile, mtimefile, repoid = None):

1377 """ 1378 Internal method. Validates Entropy Cache based on a setting file 1379 and its stored (cache bound) mtime. 1380 1381 @param settingfile: path of the setting file 1382 @type settingfile: string 1383 @param mtimefile: path where to save retrieved mtime information 1384 @type mtimefile: string 1385 @keyword repoid: repository identifier or None 1386 @type repoid: string or None 1387 @return: None 1388 @rtype: None 1389 """ 1390 1391 # can't validate if running as user, moreover 1392 # users can't make changes, so... 1393 if os.getuid() != 0: 1394 return 1395 1396 # handle on-disk cache validation 1397 # in this case, repositories cache 1398 # if file is changed, we must destroy cache 1399 if not os.path.isfile(mtimefile): 1400 # we can't know if it has been updated 1401 # remove repositories caches 1402 self.__remove_repo_cache(repoid = repoid) 1403 self.__save_file_mtime(settingfile, mtimefile) 1404 else: 1405 # check mtime 1406 try: 1407 mtime_f = open(mtimefile,"r") 1408 mtime = mtime_f.readline().strip() 1409 mtime_f.close() 1410 # compare with current mtime 1411 try: 1412 currmtime = str(os.path.getmtime(settingfile)) 1413 except OSError: 1414 currmtime = "0.0" 1415 if mtime != currmtime: 1416 self.__remove_repo_cache(repoid = repoid) 1417 self.__save_file_mtime(settingfile, mtimefile) 1418 except (OSError, IOError,): 1419 self.__remove_repo_cache(repoid = repoid) 1420 self.__save_file_mtime(settingfile, mtimefile)

1421

Source Code for Module entropy.core