entropy.core

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework core module}. 10 11 This module contains base classes used by entropy.client, 12 entropy.server and entropy.services. 13 14 "Singleton" is a class that is inherited from singleton objects. 15 16 SystemSettings is a singleton, pluggable interface which contains 17 all the runtime settings (mostly parsed from configuration files 18 and inherited from entropy.const -- which contains almost all the 19 default values). 20 SystemSettings works as a I{dict} object. Due to limitations of 21 multiple inherittance when using the Singleton class, SystemSettings 22 ONLY mimics a I{dict} AND it's not a subclass of it. 23 24 SystemSettingsPlugin is the base class for building valid SystemSettings 25 plugin modules (see entropy.client.interfaces.client or 26 entropy.server.interfaces for working examples). 27 28 """ 29 from __future__ import with_statement 30 import os 31 import sys 32 from entropy.exceptions import IncorrectParameter, SystemDatabaseError 33 from entropy.const import etpConst, etpUi, etpSys, const_setup_perms, \ 34 const_secure_config_file, const_set_nice_level, \ 35 const_extract_cli_repo_params, etpCache 36 from entropy.i18n import _ 37 from threading import RLock 38

39 -class Singleton(object):

40 41 """ 42 If your class wants to become a sexy Singleton, 43 subclass this and replace __init__ with init_singleton 44 """ 45 46 __is_destroyed = False 47 __is_singleton = True

48 - def __new__(cls, *args, **kwds):

49 instance = cls.__dict__.get("__it__") 50 if instance != None: 51 if not instance.is_destroyed(): 52 return instance 53 cls.__it__ = instance = object.__new__(cls) 54 instance.init_singleton(*args, **kwds) 55 return instance

56

57 - def is_destroyed(self):

58 """ 59 In our world, Singleton instances may be destroyed, 60 this is done by setting a private bool var __is_destroyed 61 62 @rtype: bool 63 @return: instance status, if destroyed or not 64 """ 65 return self.__is_destroyed

66

67 - def is_singleton(self):

68 """ 69 Return if the instance is a singleton 70 71 @rtype: bool 72 @return: class singleton property, if singleton or not 73 """ 74 return self.__is_singleton

75

76 -class SystemSettingsPlugin:

77 78 """ 79 80 This is a plugin base class for all SystemSettings plugins. 81 It allows to add extra parsers (though metadata) to 82 SystemSettings. 83 Just inherit from this class and call add_parser to add 84 your custom parsers. 85 SystemSettings will call the parse method, as explained below. 86 87 Sample code: 88 89 >>> # load SystemSettings 90 >>> from entropy.core import SystemSettings, SystemSettingsPlugin 91 >>> system_settings = SystemSettings() 92 >>> class MyPlugin(SystemSettingsPlugin): 93 >>> pass 94 >>> my_plugin = MyPlugin('mystuff', None) 95 >>> def myparsing_function(): 96 >>> return {'abc': 1 } 97 >>> my_plugin.add_parser('parser_no_1', myparsing_function) 98 >>> system_settings.add_plugin(my_plugin) 99 >>> print(system_settings['mystuff']['parser_no_1']) 100 {'abc': 1 } 101 >>> # let's remove it 102 >>> system_settings.remove_plugin('mystuff') # through its plugin_id 103 >>> print(system_settings.get('mystuff')) 104 None 105 106 """ 107

108 - def __init__(self, plugin_id, helper_interface):

109 """ 110 SystemSettingsPlugin constructor. 111 112 @param plugin_id: plugin identifier, must be unique 113 @type plugin_id: string 114 @param helper_interface: any Python object that could 115 be of help to your parsers 116 @type handler_instance: Python object 117 @rtype: None 118 @return: None 119 """ 120 self.__parsers = [] 121 self.__plugin_id = plugin_id 122 self._helper = helper_interface 123 parser_postfix = "_parser" 124 for method in sorted(dir(self)): 125 if method == "add_parser": 126 continue 127 elif method.endswith(parser_postfix) and (method != parser_postfix): 128 parser_id = method[:len(parser_postfix)*-1] 129 self.__parsers.append((parser_id, getattr(self, method),))

130

131 - def get_id(self):

132 """ 133 Returns the unique plugin id passed at construction time. 134 135 @return: plugin identifier 136 @rtype: string 137 """ 138 return self.__plugin_id

139

140 - def add_parser(self, parser_id, parser_callable):

141 """ 142 You must call this method in order to add your custom 143 parsers to the plugin. 144 Please note, if your parser method ends with "_parser" 145 it will be automatically added this way: 146 147 method: foo_parser 148 parser_id => foo 149 method: another_fabulous_parser 150 parser_id => another_fabulous 151 152 @param parser_id: parser identifier, must be unique 153 @type parser_id: string 154 @param parser_callable: any callable function which has 155 the following signature: callable(system_settings_instance) 156 can return True to stop further parsers calls 157 @type parser_callable: callable 158 @return: None 159 @rtype: None 160 """ 161 self.__parsers.append((parser_id, parser_callable,))

162

163 - def parse(self, system_settings_instance):

164 """ 165 This method is called by SystemSettings instance 166 when building its settings metadata. 167 168 Returned data from parser will be put into the SystemSettings 169 dict using plugin_id and parser_id keys. 170 If returned data is None, SystemSettings dict won't be changed. 171 172 @param system_settings_instance: SystemSettings instance 173 @type system_settings_instance: SystemSettings instance 174 @return: None 175 @rtype: None 176 """ 177 plugin_id = self.get_id() 178 for parser_id, parser in self.__parsers: 179 data = parser(system_settings_instance) 180 if data == None: 181 continue 182 if not system_settings_instance.has_key(plugin_id): 183 system_settings_instance[plugin_id] = {} 184 system_settings_instance[plugin_id][parser_id] = data

185

186 - def post_setup(self, system_settings_instance):

187 """ 188 This method is called by SystemSettings instance 189 after having built all the SystemSettings metadata. 190 You can reimplement this and hook your refinement code 191 into this method. 192 193 @param system_settings_instance: SystemSettings instance 194 @type system_settings_instance: SystemSettings instance 195 @return: None 196 @rtype: None 197 """ 198 pass

199

200 -class SystemSettings(Singleton):

201 202 """ 203 This is the place where all the Entropy settings are stored if 204 they are not considered instance constants (etpConst). 205 For example, here we store package masking cache information and 206 settings, client-side, server-side and services settings. 207 Also, this class mimics a dictionary (even if not inheriting it 208 due to development choices). 209 210 Sample code: 211 212 >>> from entropy.core import SystemSettings 213 >>> system_settings = SystemSettings() 214 >>> system_settings.clear() 215 >>> system_settings.destroy() 216 217 """ 218 219 import entropy.tools as entropyTools

220 - def init_singleton(self):

221 222 """ 223 Replaces __init__ because SystemSettings is a Singleton. 224 see Singleton API reference for more information. 225 226 """ 227 228 from entropy.cache import EntropyCacher 229 self.__cacher = EntropyCacher() 230 self.__data = {} 231 self.__is_destroyed = False 232 self.__mutex = RLock() # reentrant lock on purpose 233 234 self.__plugins = {} 235 self.__setting_files_order = [] 236 self.__setting_files_pre_run = [] 237 self.__setting_files = {} 238 self.__mtime_files = {} 239 self.__persistent_settings = { 240 'pkg_masking_reasons': { 241 0: _('reason not available'), 242 1: _('user package.mask'), 243 2: _('system keywords'), 244 3: _('user package.unmask'), 245 4: _('user repo package.keywords (all packages)'), 246 5: _('user repo package.keywords'), 247 6: _('user package.keywords'), 248 7: _('completely masked'), 249 8: _('repository general packages.db.mask'), 250 10: _('user license.mask'), 251 11: _('user live unmask'), 252 12: _('user live mask'), 253 }, 254 'pkg_masking_reference': { 255 'reason_not_avail': 0, 256 'user_package_mask': 1, 257 'system_keyword': 2, 258 'user_package_unmask': 3, 259 'user_repo_package_keywords_all': 4, 260 'user_repo_package_keywords': 5, 261 'user_package_keywords': 6, 262 'completely_masked': 7, 263 'repository_packages_db_mask': 8, 264 'repository_in_branch_pacakges_db_mask': 9, 265 'user_license_mask': 10, 266 'user_live_unmask': 11, 267 'user_live_mask': 12, 268 }, 269 'backed_up': {}, 270 # package masking, live 271 'live_packagemasking': { 272 'unmask_matches': set(), 273 'mask_matches': set(), 274 }, 275 } 276 277 self.__setup_const() 278 self.__scan()

279

280 - def destroy(self):

281 """ 282 Overloaded method from Singleton. 283 "Destroys" the instance. 284 285 @return: None 286 @rtype: None 287 """ 288 with self.__mutex: 289 self.__is_destroyed = True

290

291 - def add_plugin(self, system_settings_plugin_instance):

292 """ 293 This method lets you add custom parsers to SystemSettings. 294 Mind that you are responsible of handling your plugin instance 295 and remove it before it is destroyed. You can remove the plugin 296 instance at any time by issuing remove_plugin. 297 Every add_plugin or remove_plugin method will also issue clear() 298 for you. This could be bad and it might be removed in future. 299 300 @param plugin_id: plugin identifier 301 @type plugin_id: string 302 @param system_settings_plugin_instance: valid SystemSettingsPlugin 303 instance 304 @type system_settings_plugin_instance: SystemSettingsPlugin instance 305 @return: None 306 @rtype: None 307 """ 308 inst = system_settings_plugin_instance 309 if not isinstance(inst,SystemSettingsPlugin): 310 raise AttributeError("SystemSettings: expected valid " + \ 311 "SystemSettingsPlugin instance") 312 with self.__mutex: 313 self.__plugins[inst.get_id()] = inst 314 self.clear()

315

316 - def remove_plugin(self, plugin_id):

317 """ 318 This method lets you remove previously added custom parsers from 319 SystemSettings through its plugin identifier. If plugin_id is not 320 available, KeyError exception will be raised. 321 Every add_plugin or remove_plugin method will also issue clear() 322 for you. This could be bad and it might be removed in future. 323 324 @param plugin_id: plugin identifier 325 @type plugin_id: basestring 326 @return: None 327 @rtype: None 328 """ 329 with self.__mutex: 330 del self.__plugins[plugin_id] 331 self.clear()

332

333 - def __setup_const(self):

334 335 """ 336 Internal method. Does constants initialization. 337 338 @return: None 339 @rtype: None 340 """ 341 342 del self.__setting_files_order[:] 343 del self.__setting_files_pre_run[:] 344 self.__setting_files.clear() 345 self.__mtime_files.clear() 346 347 self.__setting_files.update({ 348 # keywording configuration files 349 'keywords': etpConst['confpackagesdir']+"/package.keywords", 350 # unmasking configuration files 351 'unmask': etpConst['confpackagesdir']+"/package.unmask", 352 # masking configuration files 353 'mask': etpConst['confpackagesdir']+"/package.mask", 354 # satisfied packages configuration file 355 'satisfied': etpConst['confpackagesdir']+"/package.satisfied", 356 # masking configuration files 357 'license_mask': etpConst['confpackagesdir']+"/license.mask", 358 'system_mask': etpConst['confpackagesdir']+"/system.mask", 359 'system_dirs': etpConst['confdir']+"/fsdirs.conf", 360 'system_dirs_mask': etpConst['confdir']+"/fsdirsmask.conf", 361 'system_rev_symlinks': etpConst['confdir']+"/fssymlinks.conf", 362 'broken_syms': etpConst['confdir']+"/brokensyms.conf", 363 'broken_libs_mask': etpConst['confdir']+"/brokenlibsmask.conf", 364 'hw_hash': etpConst['confdir']+"/.hw.hash", 365 'socket_service': etpConst['socketconf'], 366 'system': etpConst['entropyconf'], 367 'repositories': etpConst['repositoriesconf'], 368 'system_package_sets': {}, 369 }) 370 self.__setting_files_order.extend([ 371 'keywords', 'unmask', 'mask', 'satisfied', 'license_mask', 372 'system_mask', 'system_package_sets', 'system_dirs', 373 'system_dirs_mask', 'socket_service', 'system', 374 'system_rev_symlinks', 'hw_hash', 'broken_syms', 'broken_libs_mask' 375 ]) 376 self.__setting_files_pre_run.extend(['repositories']) 377 378 dmp_dir = etpConst['dumpstoragedir'] 379 self.__mtime_files.update({ 380 'keywords_mtime': dmp_dir+"/keywords.mtime", 381 'unmask_mtime': dmp_dir+"/unmask.mtime", 382 'mask_mtime': dmp_dir+"/mask.mtime", 383 'satisfied_mtime': dmp_dir+"/satisfied.mtime", 384 'license_mask_mtime': dmp_dir+"/license_mask.mtime", 385 'system_mask_mtime': dmp_dir+"/system_mask.mtime", 386 })

387 388

389 - def __scan(self):

390 391 """ 392 Internal method. Scan settings and fill variables. 393 394 @return: None 395 @rtype: None 396 """ 397 398 def enforce_persistent(): 399 # merge persistent settings back 400 self.__data.update(self.__persistent_settings) 401 # restore backed-up settings 402 self.__data.update(self.__persistent_settings['backed_up'].copy())

403 404 self.__parse() 405 enforce_persistent() 406 407 # plugins support 408 for plugin_id in sorted(self.__plugins): 409 self.__plugins[plugin_id].parse(self) 410 411 enforce_persistent() 412 413 # run post-SystemSettings setup, plugins hook 414 for plugin_id in sorted(self.__plugins): 415 self.__plugins[plugin_id].post_setup(self)

416

417 - def __setitem__(self, mykey, myvalue):

418 """ 419 dict method. See Python dict API reference. 420 """ 421 with self.__mutex: 422 # backup here too 423 if self.__persistent_settings.has_key(mykey): 424 self.__persistent_settings[mykey] = myvalue 425 self.__data[mykey] = myvalue

426

427 - def __getitem__(self, mykey):

428 """ 429 dict method. See Python dict API reference. 430 """ 431 with self.__mutex: 432 return self.__data[mykey]

433

434 - def __delitem__(self, mykey):

435 """ 436 dict method. See Python dict API reference. 437 """ 438 with self.__mutex: 439 del self.__data[mykey]

440

441 - def __iter__(self):

442 """ 443 dict method. See Python dict API reference. 444 """ 445 with self.__mutex: 446 return iter(self.__data)

447

448 - def __contains__(self, item):

449 """ 450 dict method. See Python dict API reference. 451 """ 452 with self.__mutex: 453 return item in self.__data

454

455 - def __cmp__(self, other):

456 """ 457 dict method. See Python dict API reference. 458 """ 459 with self.__mutex: 460 return cmp(self.__data, other)

461

462 - def __hash__(self):

463 """ 464 dict method. See Python dict API reference. 465 """ 466 with self.__mutex: 467 return hash(self.__data)

468

469 - def __len__(self):

470 """ 471 dict method. See Python dict API reference. 472 """ 473 with self.__mutex: 474 return len(self.__data)

475

476 - def get(self, mykey, alt_obj = None):

477 """ 478 dict method. See Python dict API reference. 479 """ 480 with self.__mutex: 481 return self.__data.get(mykey, alt_obj)

482

483 - def has_key(self, mykey):

484 """ 485 dict method. See Python dict API reference. 486 """ 487 with self.__mutex: 488 return self.__data.has_key(mykey)

489

490 - def copy(self):

491 """ 492 dict method. See Python dict API reference. 493 """ 494 with self.__mutex: 495 return self.__data.copy()

496

497 - def fromkeys(self, seq, val = None):

498 """ 499 dict method. See Python dict API reference. 500 """ 501 with self.__mutex: 502 return self.__data.fromkeys(seq, val)

503

504 - def items(self):

505 """ 506 dict method. See Python dict API reference. 507 """ 508 with self.__mutex: 509 return self.__data.items()

510

511 - def iteritems(self):

512 """ 513 dict method. See Python dict API reference. 514 """ 515 with self.__mutex: 516 return self.__data.iteritems()

517

518 - def iterkeys(self):

519 """ 520 dict method. See Python dict API reference. 521 """ 522 with self.__mutex: 523 return self.__data.iterkeys()

524

525 - def keys(self):

526 """ 527 dict method. See Python dict API reference. 528 """ 529 with self.__mutex: 530 return self.__data.keys()

531

532 - def pop(self, mykey, default = None):

533 """ 534 dict method. See Python dict API reference. 535 """ 536 with self.__mutex: 537 return self.__data.pop(mykey, default)

538

539 - def popitem(self):

540 """ 541 dict method. See Python dict API reference. 542 """ 543 with self.__mutex: 544 return self.__data.popitem()

545

546 - def setdefault(self, mykey, default = None):

547 """ 548 dict method. See Python dict API reference. 549 """ 550 with self.__mutex: 551 return self.__data.setdefault(mykey, default)

552

553 - def update(self, kwargs):

554 """ 555 dict method. See Python dict API reference. 556 """ 557 with self.__mutex: 558 return self.__data.update(kwargs)

559

560 - def values(self):

561 """ 562 dict method. See Python dict API reference. 563 """ 564 with self.__mutex: 565 return self.__data.values()

566

567 - def clear(self):

568 """ 569 dict method. See Python dict API reference. 570 Settings are also re-initialized here. 571 572 @return None 573 """ 574 with self.__mutex: 575 self.__data.clear() 576 self.__setup_const() 577 self.__scan()

578

579 - def set_persistent_setting(self, persistent_dict):

580 """ 581 Make metadata persistent, the input dict will be merged 582 with the base one at every reset call (clear()). 583 584 @param persistent_dict: dictionary to merge 585 @type persistent_dict: dict 586 587 @return: None 588 @rtype: None 589 """ 590 with self.__mutex: 591 self.__persistent_settings.update(persistent_dict)

592

593 - def unset_persistent_setting(self, persistent_key):

594 """ 595 Remove dict key from persistent dictionary 596 597 @param persistent_key: key to remove 598 @type persistent_dict: dict 599 600 @return: None 601 @rtype: None 602 """ 603 with self.__mutex: 604 del self.__persistent_settings[persistent_key] 605 del self.__data[persistent_key]

606

607 - def __setup_package_sets_vars(self):

608 609 """ 610 This function setups the *files* dictionary about package sets 611 that will be read and parsed afterwards by the respective 612 internal parser. 613 614 @return: None 615 @rtype: None 616 """ 617 618 # user defined package sets 619 sets_dir = etpConst['confsetsdir'] 620 pkg_set_data = {} 621 if (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 622 set_files = [x for x in os.listdir(sets_dir) if \ 623 (os.path.isfile(os.path.join(sets_dir, x)) and \ 624 os.access(os.path.join(sets_dir, x),os.R_OK))] 625 for set_file in set_files: 626 try: 627 set_file = set_file.decode(sys.getfilesystemencoding()) 628 except (UnicodeDecodeError, UnicodeEncodeError,): 629 continue 630 pkg_set_data[set_file] = os.path.join(sets_dir, set_file) 631 self.__setting_files['system_package_sets'].update(pkg_set_data)

632

633 - def __parse(self):

634 """ 635 This is the main internal parsing method. 636 *files* and *mtimes* dictionaries are prepared and 637 parsed just a few lines later. 638 639 @return: None 640 @rtype: None 641 """ 642 # some parsers must be run BEFORE everything: 643 for item in self.__setting_files_pre_run: 644 myattr = '_%s_parser' % (item,) 645 if not hasattr(self, myattr): 646 continue 647 func = getattr(self, myattr) 648 self.__data[item] = func() 649 650 # parse main settings 651 self.__setup_package_sets_vars() 652 653 data = {} 654 for item in self.__setting_files_order: 655 myattr = '_%s_parser' % (item,) 656 if not hasattr(self, myattr): 657 continue 658 func = getattr(self, myattr) 659 self.__data[item] = func()

660

661 - def get_setting_files_data(self):

662 """ 663 Return a copy of the internal *files* dictionary. 664 This dict contains config file paths and their identifiers. 665 666 @return: dict __setting_files 667 @rtype: dict 668 """ 669 with self.__mutex: 670 return self.__setting_files.copy()

671

672 - def _keywords_parser(self):

673 """ 674 Parser returning package keyword masking metadata 675 read from package.keywords file. 676 This file contains package mask or unmask directives 677 based on package keywords. 678 679 @return: parsed metadata 680 @rtype: dict 681 """ 682 # merge universal keywords 683 data = { 684 'universal': set(), 685 'packages': {}, 686 'repositories': {}, 687 } 688 689 self.validate_entropy_cache(self.__setting_files['keywords'], 690 self.__mtime_files['keywords_mtime']) 691 content = [x.split() for x in \ 692 self.__generic_parser(self.__setting_files['keywords']) \ 693 if len(x.split()) < 4] 694 for keywordinfo in content: 695 # skip wrong lines 696 if len(keywordinfo) > 3: 697 continue 698 # inversal keywording, check if it's not repo= 699 if len(keywordinfo) == 1: 700 if keywordinfo[0].startswith("repo="): 701 continue 702 # convert into entropy format 703 if keywordinfo[0] == "**": 704 keywordinfo[0] = "" 705 data['universal'].add(keywordinfo[0]) 706 continue 707 # inversal keywording, check if it's not repo= 708 if len(keywordinfo) in (2, 3,): 709 # repo=? 710 if keywordinfo[0].startswith("repo="): 711 continue 712 # add to repo? 713 items = keywordinfo[1:] 714 # convert into entropy format 715 if keywordinfo[0] == "**": 716 keywordinfo[0] = "" 717 reponame = [x for x in items if x.startswith("repo=") \ 718 and (len(x.split("=")) == 2)] 719 if reponame: 720 reponame = reponame[0].split("=")[1] 721 if reponame not in data['repositories']: 722 data['repositories'][reponame] = {} 723 # repository unmask or package in repository unmask? 724 if keywordinfo[0] not in data['repositories'][reponame]: 725 data['repositories'][reponame][keywordinfo[0]] = set() 726 if len(items) == 1: 727 # repository unmask 728 data['repositories'][reponame][keywordinfo[0]].add('*') 729 elif "*" not in \ 730 data['repositories'][reponame][keywordinfo[0]]: 731 732 item = [x for x in items if not x.startswith("repo=")] 733 data['repositories'][reponame][keywordinfo[0]].add( 734 item[0]) 735 elif len(items) == 2: 736 # it's going to be a faulty line!!?? 737 # can't have two items and no repo= 738 continue 739 else: 740 # add keyword to packages 741 if keywordinfo[0] not in data['packages']: 742 data['packages'][keywordinfo[0]] = set() 743 data['packages'][keywordinfo[0]].add(items[0]) 744 745 # merge universal keywords 746 etpConst['keywords'].clear() 747 etpConst['keywords'].update(etpSys['keywords']) 748 for keyword in data['universal']: 749 etpConst['keywords'].add(keyword) 750 751 return data

752 753

754 - def _unmask_parser(self):

755 """ 756 Parser returning package unmasking metadata read from 757 package.unmask file. 758 This file contains package unmask directives, allowing 759 to enable experimental or *secret* packages. 760 761 @return: parsed metadata 762 @rtype: dict 763 """ 764 self.validate_entropy_cache(self.__setting_files['unmask'], 765 self.__mtime_files['unmask_mtime']) 766 return self.__generic_parser(self.__setting_files['unmask'])

767

768 - def _mask_parser(self):

769 """ 770 Parser returning package masking metadata read from 771 package.mask file. 772 This file contains package mask directives, allowing 773 to disable experimental or *secret* packages. 774 775 @return: parsed metadata 776 @rtype: dict 777 """ 778 self.validate_entropy_cache(self.__setting_files['mask'], 779 self.__mtime_files['mask_mtime']) 780 return self.__generic_parser(self.__setting_files['mask'])

781

782 - def _satisfied_parser(self):

783 """ 784 Parser returning package forced satisfaction metadata 785 read from package.satisfied file. 786 This file contains packages which updates as dependency are 787 filtered out. 788 789 @return: parsed metadata 790 @rtype: dict 791 """ 792 self.validate_entropy_cache(self.__setting_files['satisfied'], 793 self.__mtime_files['satisfied_mtime']) 794 return self.__generic_parser(self.__setting_files['satisfied'])

795

796 - def _system_mask_parser(self):

797 """ 798 Parser returning system packages mask metadata read from 799 package.system_mask file. 800 This file contains packages that should be always kept 801 installed, extending the already defined (in repository database) 802 set of atoms. 803 804 @return: parsed metadata 805 @rtype: dict 806 """ 807 self.validate_entropy_cache(self.__setting_files['system_mask'], 808 self.__mtime_files['system_mask_mtime']) 809 return self.__generic_parser(self.__setting_files['system_mask'])

810

811 - def _license_mask_parser(self):

812 """ 813 Parser returning packages masked by license metadata read from 814 license.mask file. 815 Packages shipped with licenses listed there will be masked. 816 817 @return: parsed metadata 818 @rtype: dict 819 """ 820 self.validate_entropy_cache(self.__setting_files['license_mask'], 821 self.__mtime_files['license_mask_mtime']) 822 return self.__generic_parser(self.__setting_files['license_mask'])

823

824 - def _system_package_sets_parser(self):

825 """ 826 Parser returning system defined package sets read from 827 /etc/entropy/packages/sets. 828 829 @return: parsed metadata 830 @rtype: dict 831 """ 832 data = {} 833 for set_name in self.__setting_files['system_package_sets']: 834 set_filepath = self.__setting_files['system_package_sets'][set_name] 835 set_elements = self.entropyTools.extract_packages_from_set_file( 836 set_filepath) 837 if set_elements: 838 data[set_name] = set_elements.copy() 839 return data

840

841 - def _system_dirs_parser(self):

842 """ 843 Parser returning directories considered part of the base system. 844 845 @return: parsed metadata 846 @rtype: dict 847 """ 848 return self.__generic_parser(self.__setting_files['system_dirs'])

849

850 - def _system_dirs_mask_parser(self):

851 """ 852 Parser returning directories NOT considered part of the base system. 853 Settings here overlay system_dirs_parser. 854 855 @return: parsed metadata 856 @rtype: dict 857 """ 858 return self.__generic_parser(self.__setting_files['system_dirs_mask'])

859

860 - def _broken_syms_parser(self):

861 """ 862 Parser returning a list of shared objects symbols that can be used by 863 QA tools to scan the filesystem or a subset of it. 864 865 @return: parsed metadata 866 @rtype: dict 867 """ 868 return self.__generic_parser(self.__setting_files['broken_syms'])

869

870 - def _broken_libs_mask_parser(self):

871 """ 872 Parser returning a list of broken shared libraries which are 873 always considered sane. 874 875 @return: parsed metadata 876 @rtype: dict 877 """ 878 return self.__generic_parser(self.__setting_files['broken_libs_mask'])

879

880 - def _hw_hash_parser(self):

881 """ 882 Hardware hash metadata parser and generator. It returns a theorically 883 unique SHA256 hash bound to the computer running this Framework. 884 885 @return: string containing SHA256 hexdigest 886 @rtype: string 887 """ 888 hw_hash_file = self.__setting_files['hw_hash'] 889 if os.access(hw_hash_file, os.R_OK | os.F_OK): 890 hash_f = open(hw_hash_file, "r") 891 hash_data = hash_f.readline().strip() 892 hash_f.close() 893 return hash_data 894 895 hash_file_dir = os.path.dirname(hw_hash_file) 896 hw_hash_exec = etpConst['etp_hw_hash_gen'] 897 if os.access(hash_file_dir, os.W_OK) and \ 898 os.access(hw_hash_exec, os.X_OK | os.F_OK | os.R_OK): 899 pipe = os.popen('{ ' + hw_hash_exec + '; } 2>&1', 'r') 900 hash_data = pipe.read().strip() 901 sts = pipe.close() 902 if sts != None: 903 return None 904 hash_f = open(hw_hash_file, "w") 905 hash_f.write(hash_data) 906 hash_f.flush() 907 hash_f.close() 908 return hash_data

909

910 - def _system_rev_symlinks_parser(self):

911 """ 912 Parser returning important system symlinks mapping. For example: 913 {'/usr/lib': ['/usr/lib64']} 914 Useful for reverse matching files belonging to packages. 915 916 @return: dict containing the mapping 917 @rtype: dict 918 """ 919 setting_file = self.__setting_files['system_rev_symlinks'] 920 raw_data = self.__generic_parser(setting_file) 921 data = {} 922 for line in raw_data: 923 line = line.split() 924 if len(line) < 2: 925 continue 926 data[line[0]] = frozenset(line[1:]) 927 return data

928

929 - def _socket_service_parser(self):

930 """ 931 Parses socket service configuration file. 932 This file contains information about Entropy remote service ports 933 and SSL. 934 935 @return: parsed metadata 936 @rtype: dict 937 """ 938 939 data = etpConst['socket_service'].copy() 940 941 sock_conf = self.__setting_files['socket_service'] 942 if not (os.path.isfile(sock_conf) and \ 943 os.access(sock_conf,os.R_OK)): 944 return data 945 946 socket_f = open(sock_conf,"r") 947 socketconf = [x.strip() for x in socket_f.readlines() if \ 948 x.strip() and not x.strip().startswith("#")] 949 socket_f.close() 950 951 for line in socketconf: 952 953 split_line = line.split("|") 954 split_line_len = len(split_line) 955 956 if line.startswith("listen|") and (split_line_len > 1): 957 958 item = split_line[1].strip() 959 if item: 960 data['hostname'] = item 961 962 elif line.startswith("listen-port|") and \ 963 (split_line_len > 1): 964 965 item = split_line[1].strip() 966 try: 967 item = int(item) 968 data['port'] = item 969 except ValueError: 970 continue 971 972 elif line.startswith("listen-timeout|") and \ 973 (split_line_len > 1): 974 975 item = split_line[1].strip() 976 try: 977 item = int(item) 978 data['timeout'] = item 979 except ValueError: 980 continue 981 982 elif line.startswith("listen-threads|") and \ 983 (split_line_len > 1): 984 985 item = split_line[1].strip() 986 try: 987 item = int(item) 988 data['threads'] = item 989 except ValueError: 990 continue 991 992 elif line.startswith("session-ttl|") and \ 993 (split_line_len > 1): 994 995 item = split_line[1].strip() 996 try: 997 item = int(item) 998 data['session_ttl'] = item 999 except ValueError: 1000 continue 1001 1002 elif line.startswith("max-connections|") and \ 1003 (split_line_len > 1): 1004 1005 item = split_line[1].strip() 1006 try: 1007 item = int(item) 1008 data['max_connections'] = item 1009 except ValueError: 1010 continue 1011 1012 elif line.startswith("ssl-port|") and \ 1013 (split_line_len > 1): 1014 1015 item = split_line[1].strip() 1016 try: 1017 item = int(item) 1018 data['ssl_port'] = item 1019 except ValueError: 1020 continue 1021 1022 elif line.startswith("disabled-commands|") and \ 1023 (split_line_len > 1): 1024 1025 disabled_cmds = split_line[1].strip().split() 1026 for disabled_cmd in disabled_cmds: 1027 data['disabled_cmds'].add(disabled_cmd) 1028 1029 elif line.startswith("ip-blacklist|") and \ 1030 (split_line_len > 1): 1031 1032 ips_blacklist = split_line[1].strip().split() 1033 for ip_blacklist in ips_blacklist: 1034 data['ip_blacklist'].add(ip_blacklist) 1035 1036 return data

1037

1038 - def _system_parser(self):

1039 1040 """ 1041 Parses Entropy system configuration file. 1042 1043 @return: parsed metadata 1044 @rtype: dict 1045 """ 1046 1047 data = {} 1048 data['proxy'] = etpConst['proxy'].copy() 1049 data['name'] = etpConst['systemname'] 1050 data['log_level'] = etpConst['entropyloglevel'] 1051 1052 etp_conf = self.__setting_files['system'] 1053 if not os.path.isfile(etp_conf) and \ 1054 os.access(etp_conf,os.R_OK): 1055 return data 1056 1057 const_secure_config_file(etp_conf) 1058 entropy_f = open(etp_conf,"r") 1059 entropyconf = [x.strip() for x in entropy_f.readlines() if \ 1060 x.strip() and not x.strip().startswith("#")] 1061 entropy_f.close() 1062 1063 for line in entropyconf: 1064 1065 split_line = line.split("|") 1066 split_line_len = len(split_line) 1067 1068 if line.startswith("loglevel|") and \ 1069 (len(line.split("loglevel|")) == 2): 1070 1071 loglevel = line.split("loglevel|")[1] 1072 try: 1073 loglevel = int(loglevel) 1074 except ValueError: 1075 pass 1076 if (loglevel > -1) and (loglevel < 3): 1077 data['log_level'] = loglevel 1078 1079 elif line.startswith("ftp-proxy|") and \ 1080 (split_line_len == 2): 1081 1082 ftpproxy = split_line[1].strip().split() 1083 if ftpproxy: 1084 data['proxy']['ftp'] = ftpproxy[-1] 1085 1086 elif line.startswith("http-proxy|") and \ 1087 (split_line_len == 2): 1088 1089 httpproxy = split_line[1].strip().split() 1090 if httpproxy: 1091 data['proxy']['http'] = httpproxy[-1] 1092 1093 elif line.startswith("proxy-username|") and \ 1094 (split_line_len == 2): 1095 1096 httpproxy = split_line[1].strip().split() 1097 if httpproxy: 1098 data['proxy']['username'] = httpproxy[-1] 1099 1100 elif line.startswith("proxy-password|") and \ 1101 (split_line_len == 2): 1102 1103 httpproxy = split_line[1].strip().split() 1104 if httpproxy: 1105 data['proxy']['password'] = httpproxy[-1] 1106 1107 elif line.startswith("system-name|") and \ 1108 (split_line_len == 2): 1109 1110 data['name'] = split_line[1].strip() 1111 1112 elif line.startswith("nice-level|") and \ 1113 (split_line_len == 2): 1114 1115 mylevel = split_line[1].strip() 1116 try: 1117 mylevel = int(mylevel) 1118 if (mylevel >= -19) and (mylevel <= 19): 1119 const_set_nice_level(mylevel) 1120 except (ValueError,): 1121 continue 1122 1123 return data

1124

1125 - def _repositories_parser(self):

1126 1127 """ 1128 Setup Entropy Client repository settings reading them from 1129 the relative config file specified in etpConst['repositoriesconf'] 1130 1131 @return: parsed metadata 1132 @rtype: dict 1133 """ 1134 1135 data = { 1136 'available': {}, 1137 'excluded': {}, 1138 'order': [], 1139 'product': etpConst['product'], 1140 'branch': etpConst['branch'], 1141 'default_repository': etpConst['officialrepositoryid'], 1142 'transfer_limit': etpConst['downloadspeedlimit'], 1143 'security_advisories_url': etpConst['securityurl'], 1144 } 1145 1146 repo_conf = etpConst['repositoriesconf'] 1147 if not (os.path.isfile(repo_conf) and os.access(repo_conf, os.R_OK)): 1148 return data 1149 1150 repo_f = open(repo_conf,"r") 1151 repositoriesconf = [x.strip() for x in repo_f.readlines() if x.strip()] 1152 repo_f.close() 1153 1154 # setup product and branch first 1155 for line in repositoriesconf: 1156 1157 split_line = line.split("|") 1158 split_line_len = len(split_line) 1159 1160 if (line.find("product|") != -1) and \ 1161 (not line.startswith("#")) and (split_line_len == 2): 1162 1163 data['product'] = split_line[1] 1164 1165 elif (line.find("branch|") != -1) and \ 1166 (not line.startswith("#")) and (split_line_len == 2): 1167 1168 branch = split_line[1].strip() 1169 data['branch'] = branch 1170 if not os.path.isdir(etpConst['packagesbindir']+"/"+branch) \ 1171 and (etpConst['uid'] == 0): 1172 1173 try: 1174 os.makedirs(etpConst['packagesbindir']+"/"+branch) 1175 except (OSError, IOError,): 1176 continue 1177 1178 for line in repositoriesconf: 1179 1180 split_line = line.split("|") 1181 split_line_len = len(split_line) 1182 1183 # populate data['available'] 1184 if (line.find("repository|") != -1) and (split_line_len == 5): 1185 1186 excluded = False 1187 my_repodata = data['available'] 1188 if line.startswith("##"): 1189 continue 1190 elif line.startswith("#"): 1191 excluded = True 1192 my_repodata = data['excluded'] 1193 line = line[1:] 1194 1195 reponame, repodata = const_extract_cli_repo_params(line, 1196 data['branch'], data['product']) 1197 if my_repodata.has_key(reponame): 1198 1199 my_repodata[reponame]['plain_packages'].extend( 1200 repodata['plain_packages']) 1201 my_repodata[reponame]['packages'].extend( 1202 repodata['packages']) 1203 1204 if (not my_repodata[reponame]['plain_database']) and \ 1205 repodata['plain_database']: 1206 1207 my_repodata[reponame]['plain_database'] = \ 1208 repodata['plain_database'] 1209 my_repodata[reponame]['database'] = \ 1210 repodata['database'] 1211 my_repodata[reponame]['dbrevision'] = \ 1212 repodata['dbrevision'] 1213 my_repodata[reponame]['dbcformat'] = \ 1214 repodata['dbcformat'] 1215 else: 1216 1217 my_repodata[reponame] = repodata.copy() 1218 if not excluded: 1219 data['order'].append(reponame) 1220 1221 elif (line.find("officialrepositoryid|") != -1) and \ 1222 (not line.startswith("#")) and (split_line_len == 2): 1223 1224 officialreponame = split_line[1] 1225 data['default_repository'] = officialreponame 1226 1227 elif (line.find("downloadspeedlimit|") != -1) and \ 1228 (not line.startswith("#")) and (split_line_len == 2): 1229 1230 try: 1231 myval = int(split_line[1]) 1232 if myval > 0: 1233 data['transfer_limit'] = myval 1234 else: 1235 data['transfer_limit'] = None 1236 except (ValueError, IndexError,): 1237 data['transfer_limit'] = None 1238 1239 elif (line.find("securityurl|") != -1) and \ 1240 (not line.startswith("#")) and (split_line_len == 2): 1241 1242 try: 1243 data['security_advisories_url'] = split_line[1] 1244 except (IndexError, ValueError, TypeError,): 1245 continue 1246 1247 return data

1248

1249 - def _clear_repository_cache(self, repoid = None):

1250 """ 1251 Internal method, go away! 1252 """ 1253 self.__cacher.discard() 1254 self._clear_dump_cache(etpCache['world_available']) 1255 self._clear_dump_cache(etpCache['world_update']) 1256 self._clear_dump_cache(etpCache['critical_update']) 1257 self._clear_dump_cache(etpCache['check_package_update']) 1258 self._clear_dump_cache(etpCache['filter_satisfied_deps']) 1259 self._clear_dump_cache(etpCache['atomMatch']) 1260 self._clear_dump_cache(etpCache['dep_tree']) 1261 if repoid != None: 1262 self._clear_dump_cache("%s/%s%s/" % ( 1263 etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) 1264 self._clear_dump_cache("%s/%s%s/" % ( 1265 etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,))

1266

1267 - def _clear_dump_cache(self, dump_name, skip = []):

1268 """ 1269 Internal method, go away! 1270 """ 1271 dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) 1272 dump_dir = os.path.dirname(dump_path) 1273 #dump_file = os.path.basename(dump_path) 1274 for currentdir, subdirs, files in os.walk(dump_dir): 1275 path = os.path.join(dump_dir,currentdir) 1276 if skip: 1277 found = False 1278 for myskip in skip: 1279 if path.find(myskip) != -1: 1280 found = True 1281 break 1282 if found: continue 1283 for item in files: 1284 if item.endswith(etpConst['cachedumpext']): 1285 item = os.path.join(path,item) 1286 try: 1287 os.remove(item) 1288 except (OSError, IOError,): 1289 pass 1290 try: 1291 if not os.listdir(path): 1292 os.rmdir(path) 1293 except (OSError, IOError,): 1294 pass

1295

1296 - def __generic_parser(self, filepath):

1297 """ 1298 Internal method. This is the generic file parser here. 1299 1300 @param filepath: valid path 1301 @type filepath: string 1302 @return: raw text extracted from file 1303 @rtype: list 1304 """ 1305 return self.entropyTools.generic_file_content_parser(filepath)

1306

1307 - def __remove_repo_cache(self, repoid = None):

1308 """ 1309 Internal method. Remove repository cache, because not valid anymore. 1310 1311 @keyword repoid: repository identifier or None 1312 @type repoid: string or None 1313 @return: None 1314 @rtype: None 1315 """ 1316 if os.path.isdir(etpConst['dumpstoragedir']): 1317 if repoid: 1318 self._clear_repository_cache(repoid = repoid) 1319 return 1320 for repoid in self['repositories']['order']: 1321 self._clear_repository_cache(repoid = repoid) 1322 else: 1323 try: 1324 os.makedirs(etpConst['dumpstoragedir']) 1325 except IOError, e: 1326 if e.errno == 30: # readonly filesystem 1327 etpUi['pretend'] = True 1328 return 1329 except OSError: 1330 return

1331

1332 - def __save_file_mtime(self, toread, tosaveinto):

1333 """ 1334 Internal method. Save mtime of a file to another file. 1335 1336 @param toread: file path to read 1337 @type toread: string 1338 @param tosaveinto: path where to save retrieved mtime information 1339 @type tosaveinto: string 1340 @return: None 1341 @rtype: None 1342 """ 1343 if not os.path.isfile(toread): 1344 currmtime = 0.0 1345 else: 1346 currmtime = os.path.getmtime(toread) 1347 1348 if not os.path.isdir(etpConst['dumpstoragedir']): 1349 try: 1350 os.makedirs(etpConst['dumpstoragedir'], 0775) 1351 const_setup_perms(etpConst['dumpstoragedir'], 1352 etpConst['entropygid']) 1353 except IOError, e: 1354 if e.errno == 30: # readonly filesystem 1355 etpUi['pretend'] = True 1356 return 1357 except (OSError,), e: 1358 # unable to create the storage directory 1359 # useless to continue 1360 return 1361 1362 try: 1363 mtime_f = open(tosaveinto,"w") 1364 except IOError, e: # unable to write? 1365 if e.errno == 30: # readonly filesystem 1366 etpUi['pretend'] = True 1367 return 1368 else: 1369 mtime_f.write(str(currmtime)) 1370 mtime_f.flush() 1371 mtime_f.close() 1372 os.chmod(tosaveinto, 0664) 1373 if etpConst['entropygid'] != None: 1374 os.chown(tosaveinto, 0, etpConst['entropygid'])

1375 1376

1377 - def validate_entropy_cache(self, settingfile, mtimefile, repoid = None):

1378 """ 1379 Internal method. Validates Entropy Cache based on a setting file 1380 and its stored (cache bound) mtime. 1381 1382 @param settingfile: path of the setting file 1383 @type settingfile: string 1384 @param mtimefile: path where to save retrieved mtime information 1385 @type mtimefile: string 1386 @keyword repoid: repository identifier or None 1387 @type repoid: string or None 1388 @return: None 1389 @rtype: None 1390 """ 1391 1392 # can't validate if running as user, moreover 1393 # users can't make changes, so... 1394 if os.getuid() != 0: 1395 return 1396 1397 # handle on-disk cache validation 1398 # in this case, repositories cache 1399 # if file is changed, we must destroy cache 1400 if not os.path.isfile(mtimefile): 1401 # we can't know if it has been updated 1402 # remove repositories caches 1403 self.__remove_repo_cache(repoid = repoid) 1404 self.__save_file_mtime(settingfile, mtimefile) 1405 else: 1406 # check mtime 1407 try: 1408 mtime_f = open(mtimefile,"r") 1409 mtime = mtime_f.readline().strip() 1410 mtime_f.close() 1411 # compare with current mtime 1412 try: 1413 currmtime = str(os.path.getmtime(settingfile)) 1414 except OSError: 1415 currmtime = "0.0" 1416 if mtime != currmtime: 1417 self.__remove_repo_cache(repoid = repoid) 1418 self.__save_file_mtime(settingfile, mtimefile) 1419 except (OSError, IOError,): 1420 self.__remove_repo_cache(repoid = repoid) 1421 self.__save_file_mtime(settingfile, mtimefile)

1422

Source Code for Module entropy.core