entropy.core

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework core module}. 10 11 This module contains base classes used by entropy.client, 12 entropy.server and entropy.services. 13 14 "Singleton" is a class that is inherited from singleton objects. 15 16 SystemSettings is a singleton, pluggable interface which contains 17 all the runtime settings (mostly parsed from configuration files 18 and inherited from entropy.const -- which contains almost all the 19 default values). 20 SystemSettings works as a I{dict} object. Due to limitations of 21 multiple inherittance when using the Singleton class, SystemSettings 22 ONLY mimics a I{dict} AND it's not a subclass of it. 23 24 SystemSettingsPlugin is the base class for building valid SystemSettings 25 plugin modules (see entropy.client.interfaces.client or 26 entropy.server.interfaces for working examples). 27 28 """ 29 from __future__ import with_statement 30 import os 31 from entropy.exceptions import IncorrectParameter, SystemDatabaseError 32 from entropy.const import etpConst, etpUi, etpSys, const_setup_perms, \ 33 const_secure_config_file, const_set_nice_level, \ 34 const_extract_cli_repo_params, etpCache 35 from entropy.i18n import _ 36 from threading import RLock 37

38 -class Singleton(object):

39 40 """ 41 If your class wants to become a sexy Singleton, 42 subclass this and replace __init__ with init_singleton 43 """ 44 45 __is_destroyed = False 46 __is_singleton = True

47 - def __new__(cls, *args, **kwds):

48 instance = cls.__dict__.get("__it__") 49 if instance != None: 50 if not instance.is_destroyed(): 51 return instance 52 cls.__it__ = instance = object.__new__(cls) 53 instance.init_singleton(*args, **kwds) 54 return instance

55

56 - def is_destroyed(self):

57 """ 58 In our world, Singleton instances may be destroyed, 59 this is done by setting a private bool var __is_destroyed 60 61 @rtype: bool 62 @return: instance status, if destroyed or not 63 """ 64 return self.__is_destroyed

65

66 - def is_singleton(self):

67 """ 68 Return if the instance is a singleton 69 70 @rtype: bool 71 @return: class singleton property, if singleton or not 72 """ 73 return self.__is_singleton

74

75 -class SystemSettingsPlugin:

76 77 """ 78 79 This is a plugin base class for all SystemSettings plugins. 80 It allows to add extra parsers (though metadata) to 81 SystemSettings. 82 Just inherit from this class and call add_parser to add 83 your custom parsers. 84 SystemSettings will call the parse method, as explained below. 85 86 Sample code: 87 88 >>> # load SystemSettings 89 >>> from entropy.core import SystemSettings, SystemSettingsPlugin 90 >>> system_settings = SystemSettings() 91 >>> class MyPlugin(SystemSettingsPlugin): 92 >>> pass 93 >>> my_plugin = MyPlugin('mystuff', None) 94 >>> def myparsing_function(): 95 >>> return {'abc': 1 } 96 >>> my_plugin.add_parser('parser_no_1', myparsing_function) 97 >>> system_settings.add_plugin(my_plugin) 98 >>> print(system_settings['mystuff']['parser_no_1']) 99 {'abc': 1 } 100 >>> # let's remove it 101 >>> system_settings.remove_plugin('mystuff') # through its plugin_id 102 >>> print(system_settings.get('mystuff')) 103 None 104 105 """ 106

107 - def __init__(self, plugin_id, helper_interface):

108 """ 109 SystemSettingsPlugin constructor. 110 111 @param plugin_id: plugin identifier, must be unique 112 @type plugin_id: string 113 @param helper_interface: any Python object that could 114 be of help to your parsers 115 @type handler_instance: Python object 116 @rtype: None 117 @return: None 118 """ 119 self.__parsers = [] 120 self.__plugin_id = plugin_id 121 self._helper = helper_interface 122 parser_postfix = "_parser" 123 for method in sorted(dir(self)): 124 if method == "add_parser": 125 continue 126 elif method.endswith(parser_postfix) and (method != parser_postfix): 127 parser_id = method[:len(parser_postfix)*-1] 128 self.__parsers.append((parser_id, getattr(self, method),))

129

130 - def get_id(self):

131 """ 132 Returns the unique plugin id passed at construction time. 133 134 @return: plugin identifier 135 @rtype: string 136 """ 137 return self.__plugin_id

138

139 - def add_parser(self, parser_id, parser_callable):

140 """ 141 You must call this method in order to add your custom 142 parsers to the plugin. 143 Please note, if your parser method ends with "_parser" 144 it will be automatically added this way: 145 146 method: foo_parser 147 parser_id => foo 148 method: another_fabulous_parser 149 parser_id => another_fabulous 150 151 @param parser_id: parser identifier, must be unique 152 @type parser_id: string 153 @param parser_callable: any callable function which has 154 the following signature: callable(system_settings_instance) 155 can return True to stop further parsers calls 156 @type parser_callable: callable 157 @return: None 158 @rtype: None 159 """ 160 self.__parsers.append((parser_id, parser_callable,))

161

162 - def parse(self, system_settings_instance):

163 """ 164 This method is called by SystemSettings instance 165 when building its settings metadata. 166 167 Returned data from parser will be put into the SystemSettings 168 dict using plugin_id and parser_id keys. 169 If returned data is None, SystemSettings dict won't be changed. 170 171 @param system_settings_instance: SystemSettings instance 172 @type system_settings_instance: SystemSettings instance 173 @return: None 174 @rtype: None 175 """ 176 plugin_id = self.get_id() 177 for parser_id, parser in self.__parsers: 178 data = parser(system_settings_instance) 179 if data == None: 180 continue 181 if not system_settings_instance.has_key(plugin_id): 182 system_settings_instance[plugin_id] = {} 183 system_settings_instance[plugin_id][parser_id] = data

184

185 - def post_setup(self, system_settings_instance):

186 """ 187 This method is called by SystemSettings instance 188 after having built all the SystemSettings metadata. 189 You can reimplement this and hook your refinement code 190 into this method. 191 192 @param system_settings_instance: SystemSettings instance 193 @type system_settings_instance: SystemSettings instance 194 @return: None 195 @rtype: None 196 """ 197 pass

198

199 -class SystemSettings(Singleton):

200 201 """ 202 This is the place where all the Entropy settings are stored if 203 they are not considered instance constants (etpConst). 204 For example, here we store package masking cache information and 205 settings, client-side, server-side and services settings. 206 Also, this class mimics a dictionary (even if not inheriting it 207 due to development choices). 208 209 Sample code: 210 211 >>> from entropy.core import SystemSettings 212 >>> system_settings = SystemSettings() 213 >>> system_settings.clear() 214 >>> system_settings.destroy() 215 216 """ 217 218 import entropy.tools as entropyTools

219 - def init_singleton(self):

220 221 """ 222 Replaces __init__ because SystemSettings is a Singleton. 223 see Singleton API reference for more information. 224 225 """ 226 227 from entropy.cache import EntropyCacher 228 self.__cacher = EntropyCacher() 229 self.__data = {} 230 self.__is_destroyed = False 231 self.__mutex = RLock() # reentrant lock on purpose 232 233 self.__plugins = {} 234 self.__setting_files_order = [] 235 self.__setting_files_pre_run = [] 236 self.__setting_files = {} 237 self.__mtime_files = {} 238 self.__persistent_settings = { 239 'pkg_masking_reasons': { 240 0: _('reason not available'), 241 1: _('user package.mask'), 242 2: _('system keywords'), 243 3: _('user package.unmask'), 244 4: _('user repo package.keywords (all packages)'), 245 5: _('user repo package.keywords'), 246 6: _('user package.keywords'), 247 7: _('completely masked'), 248 8: _('repository general packages.db.mask'), 249 10: _('user license.mask'), 250 11: _('user live unmask'), 251 12: _('user live mask'), 252 }, 253 'pkg_masking_reference': { 254 'reason_not_avail': 0, 255 'user_package_mask': 1, 256 'system_keyword': 2, 257 'user_package_unmask': 3, 258 'user_repo_package_keywords_all': 4, 259 'user_repo_package_keywords': 5, 260 'user_package_keywords': 6, 261 'completely_masked': 7, 262 'repository_packages_db_mask': 8, 263 'repository_in_branch_pacakges_db_mask': 9, 264 'user_license_mask': 10, 265 'user_live_unmask': 11, 266 'user_live_mask': 12, 267 }, 268 'backed_up': {}, 269 # package masking, live 270 'live_packagemasking': { 271 'unmask_matches': set(), 272 'mask_matches': set(), 273 }, 274 } 275 276 self.__setup_const() 277 self.__scan()

278

279 - def destroy(self):

280 """ 281 Overloaded method from Singleton. 282 "Destroys" the instance. 283 284 @return: None 285 @rtype: None 286 """ 287 with self.__mutex: 288 self.__is_destroyed = True

289

290 - def add_plugin(self, system_settings_plugin_instance):

291 """ 292 This method lets you add custom parsers to SystemSettings. 293 Mind that you are responsible of handling your plugin instance 294 and remove it before it is destroyed. You can remove the plugin 295 instance at any time by issuing remove_plugin. 296 Every add_plugin or remove_plugin method will also issue clear() 297 for you. This could be bad and it might be removed in future. 298 299 @param plugin_id: plugin identifier 300 @type plugin_id: string 301 @param system_settings_plugin_instance: valid SystemSettingsPlugin 302 instance 303 @type system_settings_plugin_instance: SystemSettingsPlugin instance 304 @return: None 305 @rtype: None 306 """ 307 inst = system_settings_plugin_instance 308 if not isinstance(inst,SystemSettingsPlugin): 309 raise AttributeError("SystemSettings: expected valid " + \ 310 "SystemSettingsPlugin instance") 311 with self.__mutex: 312 self.__plugins[inst.get_id()] = inst 313 self.clear()

314

315 - def remove_plugin(self, plugin_id):

316 """ 317 This method lets you remove previously added custom parsers from 318 SystemSettings through its plugin identifier. If plugin_id is not 319 available, KeyError exception will be raised. 320 Every add_plugin or remove_plugin method will also issue clear() 321 for you. This could be bad and it might be removed in future. 322 323 @param plugin_id: plugin identifier 324 @type plugin_id: basestring 325 @return: None 326 @rtype: None 327 """ 328 with self.__mutex: 329 del self.__plugins[plugin_id] 330 self.clear()

331

332 - def __setup_const(self):

333 334 """ 335 Internal method. Does constants initialization. 336 337 @return: None 338 @rtype: None 339 """ 340 341 del self.__setting_files_order[:] 342 del self.__setting_files_pre_run[:] 343 self.__setting_files.clear() 344 self.__mtime_files.clear() 345 346 self.__setting_files.update({ 347 # keywording configuration files 348 'keywords': etpConst['confpackagesdir']+"/package.keywords", 349 # unmasking configuration files 350 'unmask': etpConst['confpackagesdir']+"/package.unmask", 351 # masking configuration files 352 'mask': etpConst['confpackagesdir']+"/package.mask", 353 # satisfied packages configuration file 354 'satisfied': etpConst['confpackagesdir']+"/package.satisfied", 355 # masking configuration files 356 'license_mask': etpConst['confpackagesdir']+"/license.mask", 357 'system_mask': etpConst['confpackagesdir']+"/system.mask", 358 'system_dirs': etpConst['confdir']+"/fsdirs.conf", 359 'system_dirs_mask': etpConst['confdir']+"/fsdirsmask.conf", 360 'system_rev_symlinks': etpConst['confdir']+"/fssymlinks.conf", 361 'hw_hash': etpConst['confdir']+"/.hw.hash", 362 'socket_service': etpConst['socketconf'], 363 'system': etpConst['entropyconf'], 364 'repositories': etpConst['repositoriesconf'], 365 'system_package_sets': {}, 366 }) 367 self.__setting_files_order.extend([ 368 'keywords', 'unmask', 'mask', 'satisfied', 'license_mask', 369 'system_mask', 'system_package_sets', 'system_dirs', 370 'system_dirs_mask', 'socket_service', 'system', 371 'system_rev_symlinks', 'hw_hash' 372 ]) 373 self.__setting_files_pre_run.extend(['repositories']) 374 375 dmp_dir = etpConst['dumpstoragedir'] 376 self.__mtime_files.update({ 377 'keywords_mtime': dmp_dir+"/keywords.mtime", 378 'unmask_mtime': dmp_dir+"/unmask.mtime", 379 'mask_mtime': dmp_dir+"/mask.mtime", 380 'satisfied_mtime': dmp_dir+"/satisfied.mtime", 381 'license_mask_mtime': dmp_dir+"/license_mask.mtime", 382 'system_mask_mtime': dmp_dir+"/system_mask.mtime", 383 })

384 385

386 - def __scan(self):

387 388 """ 389 Internal method. Scan settings and fill variables. 390 391 @return: None 392 @rtype: None 393 """ 394 395 def enforce_persistent(): 396 # merge persistent settings back 397 self.__data.update(self.__persistent_settings) 398 # restore backed-up settings 399 self.__data.update(self.__persistent_settings['backed_up'].copy())

400 401 self.__parse() 402 enforce_persistent() 403 404 # plugins support 405 for plugin_id in sorted(self.__plugins): 406 self.__plugins[plugin_id].parse(self) 407 408 enforce_persistent() 409 410 # run post-SystemSettings setup, plugins hook 411 for plugin_id in sorted(self.__plugins): 412 self.__plugins[plugin_id].post_setup(self)

413

414 - def __setitem__(self, mykey, myvalue):

415 """ 416 dict method. See Python dict API reference. 417 """ 418 with self.__mutex: 419 # backup here too 420 if self.__persistent_settings.has_key(mykey): 421 self.__persistent_settings[mykey] = myvalue 422 self.__data[mykey] = myvalue

423

424 - def __getitem__(self, mykey):

425 """ 426 dict method. See Python dict API reference. 427 """ 428 with self.__mutex: 429 return self.__data[mykey]

430

431 - def __delitem__(self, mykey):

432 """ 433 dict method. See Python dict API reference. 434 """ 435 with self.__mutex: 436 del self.__data[mykey]

437

438 - def __iter__(self):

439 """ 440 dict method. See Python dict API reference. 441 """ 442 with self.__mutex: 443 return iter(self.__data)

444

445 - def __contains__(self, item):

446 """ 447 dict method. See Python dict API reference. 448 """ 449 with self.__mutex: 450 return item in self.__data

451

452 - def __cmp__(self, other):

453 """ 454 dict method. See Python dict API reference. 455 """ 456 with self.__mutex: 457 return cmp(self.__data, other)

458

459 - def __hash__(self):

460 """ 461 dict method. See Python dict API reference. 462 """ 463 with self.__mutex: 464 return hash(self.__data)

465

466 - def __len__(self):

467 """ 468 dict method. See Python dict API reference. 469 """ 470 with self.__mutex: 471 return len(self.__data)

472

473 - def get(self, mykey, alt_obj = None):

474 """ 475 dict method. See Python dict API reference. 476 """ 477 with self.__mutex: 478 return self.__data.get(mykey, alt_obj)

479

480 - def has_key(self, mykey):

481 """ 482 dict method. See Python dict API reference. 483 """ 484 with self.__mutex: 485 return self.__data.has_key(mykey)

486

487 - def copy(self):

488 """ 489 dict method. See Python dict API reference. 490 """ 491 with self.__mutex: 492 return self.__data.copy()

493

494 - def fromkeys(self, seq, val = None):

495 """ 496 dict method. See Python dict API reference. 497 """ 498 with self.__mutex: 499 return self.__data.fromkeys(seq, val)

500

501 - def items(self):

502 """ 503 dict method. See Python dict API reference. 504 """ 505 with self.__mutex: 506 return self.__data.items()

507

508 - def iteritems(self):

509 """ 510 dict method. See Python dict API reference. 511 """ 512 with self.__mutex: 513 return self.__data.iteritems()

514

515 - def iterkeys(self):

516 """ 517 dict method. See Python dict API reference. 518 """ 519 with self.__mutex: 520 return self.__data.iterkeys()

521

522 - def keys(self):

523 """ 524 dict method. See Python dict API reference. 525 """ 526 with self.__mutex: 527 return self.__data.keys()

528

529 - def pop(self, mykey, default = None):

530 """ 531 dict method. See Python dict API reference. 532 """ 533 with self.__mutex: 534 return self.__data.pop(mykey, default)

535

536 - def popitem(self):

537 """ 538 dict method. See Python dict API reference. 539 """ 540 with self.__mutex: 541 return self.__data.popitem()

542

543 - def setdefault(self, mykey, default = None):

544 """ 545 dict method. See Python dict API reference. 546 """ 547 with self.__mutex: 548 return self.__data.setdefault(mykey, default)

549

550 - def update(self, kwargs):

551 """ 552 dict method. See Python dict API reference. 553 """ 554 with self.__mutex: 555 return self.__data.update(kwargs)

556

557 - def values(self):

558 """ 559 dict method. See Python dict API reference. 560 """ 561 with self.__mutex: 562 return self.__data.values()

563

564 - def clear(self):

565 """ 566 dict method. See Python dict API reference. 567 Settings are also re-initialized here. 568 569 @return None 570 """ 571 with self.__mutex: 572 self.__data.clear() 573 self.__setup_const() 574 self.__scan()

575

576 - def set_persistent_setting(self, persistent_dict):

577 """ 578 Make metadata persistent, the input dict will be merged 579 with the base one at every reset call (clear()). 580 581 @param persistent_dict: dictionary to merge 582 @type persistent_dict: dict 583 584 @return: None 585 @rtype: None 586 """ 587 with self.__mutex: 588 self.__persistent_settings.update(persistent_dict)

589

590 - def unset_persistent_setting(self, persistent_key):

591 """ 592 Remove dict key from persistent dictionary 593 594 @param persistent_key: key to remove 595 @type persistent_dict: dict 596 597 @return: None 598 @rtype: None 599 """ 600 with self.__mutex: 601 del self.__persistent_settings[persistent_key] 602 del self.__data[persistent_key]

603

604 - def __setup_package_sets_vars(self):

605 606 """ 607 This function setups the *files* dictionary about package sets 608 that will be read and parsed afterwards by the respective 609 internal parser. 610 611 @return: None 612 @rtype: None 613 """ 614 615 # user defined package sets 616 sets_dir = etpConst['confsetsdir'] 617 pkg_set_data = {} 618 if (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 619 set_files = [x for x in os.listdir(sets_dir) if \ 620 (os.path.isfile(os.path.join(sets_dir, x)) and \ 621 os.access(os.path.join(sets_dir, x),os.R_OK))] 622 for set_file in set_files: 623 try: 624 set_file = str(set_file) 625 except (UnicodeDecodeError, UnicodeEncodeError,): 626 continue 627 pkg_set_data[set_file] = os.path.join(sets_dir, set_file) 628 self.__setting_files['system_package_sets'].update(pkg_set_data)

629

630 - def __parse(self):

631 """ 632 This is the main internal parsing method. 633 *files* and *mtimes* dictionaries are prepared and 634 parsed just a few lines later. 635 636 @return: None 637 @rtype: None 638 """ 639 # some parsers must be run BEFORE everything: 640 for item in self.__setting_files_pre_run: 641 myattr = '_%s_parser' % (item,) 642 if not hasattr(self, myattr): 643 continue 644 func = getattr(self, myattr) 645 self.__data[item] = func() 646 647 # parse main settings 648 self.__setup_package_sets_vars() 649 650 data = {} 651 for item in self.__setting_files_order: 652 myattr = '_%s_parser' % (item,) 653 if not hasattr(self, myattr): 654 continue 655 func = getattr(self, myattr) 656 self.__data[item] = func()

657

658 - def get_setting_files_data(self):

659 """ 660 Return a copy of the internal *files* dictionary. 661 This dict contains config file paths and their identifiers. 662 663 @return: dict __setting_files 664 @rtype: dict 665 """ 666 with self.__mutex: 667 return self.__setting_files.copy()

668

669 - def _keywords_parser(self):

670 """ 671 Parser returning package keyword masking metadata 672 read from package.keywords file. 673 This file contains package mask or unmask directives 674 based on package keywords. 675 676 @return: parsed metadata 677 @rtype: dict 678 """ 679 # merge universal keywords 680 data = { 681 'universal': set(), 682 'packages': {}, 683 'repositories': {}, 684 } 685 686 self.validate_entropy_cache(self.__setting_files['keywords'], 687 self.__mtime_files['keywords_mtime']) 688 content = [x.split() for x in \ 689 self.__generic_parser(self.__setting_files['keywords']) \ 690 if len(x.split()) < 4] 691 for keywordinfo in content: 692 # skip wrong lines 693 if len(keywordinfo) > 3: 694 continue 695 # inversal keywording, check if it's not repo= 696 if len(keywordinfo) == 1: 697 if keywordinfo[0].startswith("repo="): 698 continue 699 # convert into entropy format 700 if keywordinfo[0] == "**": 701 keywordinfo[0] = "" 702 data['universal'].add(keywordinfo[0]) 703 continue 704 # inversal keywording, check if it's not repo= 705 if len(keywordinfo) in (2, 3,): 706 # repo=? 707 if keywordinfo[0].startswith("repo="): 708 continue 709 # add to repo? 710 items = keywordinfo[1:] 711 # convert into entropy format 712 if keywordinfo[0] == "**": 713 keywordinfo[0] = "" 714 reponame = [x for x in items if x.startswith("repo=") \ 715 and (len(x.split("=")) == 2)] 716 if reponame: 717 reponame = reponame[0].split("=")[1] 718 if reponame not in data['repositories']: 719 data['repositories'][reponame] = {} 720 # repository unmask or package in repository unmask? 721 if keywordinfo[0] not in data['repositories'][reponame]: 722 data['repositories'][reponame][keywordinfo[0]] = set() 723 if len(items) == 1: 724 # repository unmask 725 data['repositories'][reponame][keywordinfo[0]].add('*') 726 elif "*" not in \ 727 data['repositories'][reponame][keywordinfo[0]]: 728 729 item = [x for x in items if not x.startswith("repo=")] 730 data['repositories'][reponame][keywordinfo[0]].add( 731 item[0]) 732 elif len(items) == 2: 733 # it's going to be a faulty line!!?? 734 # can't have two items and no repo= 735 continue 736 else: 737 # add keyword to packages 738 if keywordinfo[0] not in data['packages']: 739 data['packages'][keywordinfo[0]] = set() 740 data['packages'][keywordinfo[0]].add(items[0]) 741 742 # merge universal keywords 743 etpConst['keywords'].clear() 744 etpConst['keywords'].update(etpSys['keywords']) 745 for keyword in data['universal']: 746 etpConst['keywords'].add(keyword) 747 748 return data

749 750

751 - def _unmask_parser(self):

752 """ 753 Parser returning package unmasking metadata read from 754 package.unmask file. 755 This file contains package unmask directives, allowing 756 to enable experimental or *secret* packages. 757 758 @return: parsed metadata 759 @rtype: dict 760 """ 761 self.validate_entropy_cache(self.__setting_files['unmask'], 762 self.__mtime_files['unmask_mtime']) 763 return self.__generic_parser(self.__setting_files['unmask'])

764

765 - def _mask_parser(self):

766 """ 767 Parser returning package masking metadata read from 768 package.mask file. 769 This file contains package mask directives, allowing 770 to disable experimental or *secret* packages. 771 772 @return: parsed metadata 773 @rtype: dict 774 """ 775 self.validate_entropy_cache(self.__setting_files['mask'], 776 self.__mtime_files['mask_mtime']) 777 return self.__generic_parser(self.__setting_files['mask'])

778

779 - def _satisfied_parser(self):

780 """ 781 Parser returning package forced satisfaction metadata 782 read from package.satisfied file. 783 This file contains packages which updates as dependency are 784 filtered out. 785 786 @return: parsed metadata 787 @rtype: dict 788 """ 789 self.validate_entropy_cache(self.__setting_files['satisfied'], 790 self.__mtime_files['satisfied_mtime']) 791 return self.__generic_parser(self.__setting_files['satisfied'])

792

793 - def _system_mask_parser(self):

794 """ 795 Parser returning system packages mask metadata read from 796 package.system_mask file. 797 This file contains packages that should be always kept 798 installed, extending the already defined (in repository database) 799 set of atoms. 800 801 @return: parsed metadata 802 @rtype: dict 803 """ 804 self.validate_entropy_cache(self.__setting_files['system_mask'], 805 self.__mtime_files['system_mask_mtime']) 806 return self.__generic_parser(self.__setting_files['system_mask'])

807

808 - def _license_mask_parser(self):

809 """ 810 Parser returning packages masked by license metadata read from 811 license.mask file. 812 Packages shipped with licenses listed there will be masked. 813 814 @return: parsed metadata 815 @rtype: dict 816 """ 817 self.validate_entropy_cache(self.__setting_files['license_mask'], 818 self.__mtime_files['license_mask_mtime']) 819 return self.__generic_parser(self.__setting_files['license_mask'])

820

821 - def _system_package_sets_parser(self):

822 """ 823 Parser returning system defined package sets read from 824 /etc/entropy/packages/sets. 825 826 @return: parsed metadata 827 @rtype: dict 828 """ 829 data = {} 830 for set_name in self.__setting_files['system_package_sets']: 831 set_filepath = self.__setting_files['system_package_sets'][set_name] 832 set_elements = self.entropyTools.extract_packages_from_set_file( 833 set_filepath) 834 if set_elements: 835 data[set_name] = set_elements.copy() 836 return data

837

838 - def _system_dirs_parser(self):

839 """ 840 Parser returning directories considered part of the base system. 841 842 @return: parsed metadata 843 @rtype: dict 844 """ 845 return self.__generic_parser(self.__setting_files['system_dirs'])

846

847 - def _system_dirs_mask_parser(self):

848 """ 849 Parser returning directories NOT considered part of the base system. 850 Settings here overlay system_dirs_parser. 851 852 @return: parsed metadata 853 @rtype: dict 854 """ 855 return self.__generic_parser(self.__setting_files['system_dirs_mask'])

856

857 - def _hw_hash_parser(self):

858 """ 859 Hardware hash metadata parser and generator. It returns a theorically 860 unique SHA256 hash bound to the computer running this Framework. 861 862 @return: string containing SHA256 hexdigest 863 @rtype: string 864 """ 865 hw_hash_file = self.__setting_files['hw_hash'] 866 if os.access(hw_hash_file, os.R_OK | os.F_OK): 867 hash_f = open(hw_hash_file, "r") 868 hash_data = hash_f.readline().strip() 869 hash_f.close() 870 return hash_data 871 872 hash_file_dir = os.path.dirname(hw_hash_file) 873 hw_hash_exec = etpConst['etp_hw_hash_gen'] 874 if os.access(hash_file_dir, os.W_OK) and \ 875 os.access(hw_hash_exec, os.X_OK | os.F_OK | os.R_OK): 876 pipe = os.popen('{ ' + hw_hash_exec + '; } 2>&1', 'r') 877 hash_data = pipe.read().strip() 878 sts = pipe.close() 879 if sts != None: 880 return None 881 hash_f = open(hw_hash_file, "w") 882 hash_f.write(hash_data) 883 hash_f.flush() 884 hash_f.close() 885 return hash_data

886

887 - def _system_rev_symlinks_parser(self):

888 """ 889 Parser returning important system symlinks mapping. For example: 890 {'/usr/lib': ['/usr/lib64']} 891 Useful for reverse matching files belonging to packages. 892 893 @return: dict containing the mapping 894 @rtype: dict 895 """ 896 setting_file = self.__setting_files['system_rev_symlinks'] 897 raw_data = self.__generic_parser(setting_file) 898 data = {} 899 for line in raw_data: 900 line = line.split() 901 if len(line) < 2: 902 continue 903 data[line[0]] = frozenset(line[1:]) 904 return data

905

906 - def _socket_service_parser(self):

907 """ 908 Parses socket service configuration file. 909 This file contains information about Entropy remote service ports 910 and SSL. 911 912 @return: parsed metadata 913 @rtype: dict 914 """ 915 916 data = etpConst['socket_service'].copy() 917 918 sock_conf = self.__setting_files['socket_service'] 919 if not (os.path.isfile(sock_conf) and \ 920 os.access(sock_conf,os.R_OK)): 921 return data 922 923 socket_f = open(sock_conf,"r") 924 socketconf = [x.strip() for x in socket_f.readlines() if \ 925 x.strip() and not x.strip().startswith("#")] 926 socket_f.close() 927 928 for line in socketconf: 929 930 split_line = line.split("|") 931 split_line_len = len(split_line) 932 933 if line.startswith("listen|") and (split_line_len > 1): 934 935 item = split_line[1].strip() 936 if item: 937 data['hostname'] = item 938 939 elif line.startswith("listen-port|") and \ 940 (split_line_len > 1): 941 942 item = split_line[1].strip() 943 try: 944 item = int(item) 945 data['port'] = item 946 except ValueError: 947 continue 948 949 elif line.startswith("listen-timeout|") and \ 950 (split_line_len > 1): 951 952 item = split_line[1].strip() 953 try: 954 item = int(item) 955 data['timeout'] = item 956 except ValueError: 957 continue 958 959 elif line.startswith("listen-threads|") and \ 960 (split_line_len > 1): 961 962 item = split_line[1].strip() 963 try: 964 item = int(item) 965 data['threads'] = item 966 except ValueError: 967 continue 968 969 elif line.startswith("session-ttl|") and \ 970 (split_line_len > 1): 971 972 item = split_line[1].strip() 973 try: 974 item = int(item) 975 data['session_ttl'] = item 976 except ValueError: 977 continue 978 979 elif line.startswith("max-connections|") and \ 980 (split_line_len > 1): 981 982 item = split_line[1].strip() 983 try: 984 item = int(item) 985 data['max_connections'] = item 986 except ValueError: 987 continue 988 989 elif line.startswith("ssl-port|") and \ 990 (split_line_len > 1): 991 992 item = split_line[1].strip() 993 try: 994 item = int(item) 995 data['ssl_port'] = item 996 except ValueError: 997 continue 998 999 elif line.startswith("disabled-commands|") and \ 1000 (split_line_len > 1): 1001 1002 disabled_cmds = split_line[1].strip().split() 1003 for disabled_cmd in disabled_cmds: 1004 data['disabled_cmds'].add(disabled_cmd) 1005 1006 elif line.startswith("ip-blacklist|") and \ 1007 (split_line_len > 1): 1008 1009 ips_blacklist = split_line[1].strip().split() 1010 for ip_blacklist in ips_blacklist: 1011 data['ip_blacklist'].add(ip_blacklist) 1012 1013 return data

1014

1015 - def _system_parser(self):

1016 1017 """ 1018 Parses Entropy system configuration file. 1019 1020 @return: parsed metadata 1021 @rtype: dict 1022 """ 1023 1024 data = {} 1025 data['proxy'] = etpConst['proxy'].copy() 1026 data['name'] = etpConst['systemname'] 1027 data['log_level'] = etpConst['entropyloglevel'] 1028 1029 etp_conf = self.__setting_files['system'] 1030 if not os.path.isfile(etp_conf) and \ 1031 os.access(etp_conf,os.R_OK): 1032 return data 1033 1034 const_secure_config_file(etp_conf) 1035 entropy_f = open(etp_conf,"r") 1036 entropyconf = [x.strip() for x in entropy_f.readlines() if \ 1037 x.strip() and not x.strip().startswith("#")] 1038 entropy_f.close() 1039 1040 for line in entropyconf: 1041 1042 split_line = line.split("|") 1043 split_line_len = len(split_line) 1044 1045 if line.startswith("loglevel|") and \ 1046 (len(line.split("loglevel|")) == 2): 1047 1048 loglevel = line.split("loglevel|")[1] 1049 try: 1050 loglevel = int(loglevel) 1051 except ValueError: 1052 pass 1053 if (loglevel > -1) and (loglevel < 3): 1054 data['log_level'] = loglevel 1055 1056 elif line.startswith("ftp-proxy|") and \ 1057 (split_line_len == 2): 1058 1059 ftpproxy = split_line[1].strip().split() 1060 if ftpproxy: 1061 data['proxy']['ftp'] = ftpproxy[-1] 1062 1063 elif line.startswith("http-proxy|") and \ 1064 (split_line_len == 2): 1065 1066 httpproxy = split_line[1].strip().split() 1067 if httpproxy: 1068 data['proxy']['http'] = httpproxy[-1] 1069 1070 elif line.startswith("proxy-username|") and \ 1071 (split_line_len == 2): 1072 1073 httpproxy = split_line[1].strip().split() 1074 if httpproxy: 1075 data['proxy']['username'] = httpproxy[-1] 1076 1077 elif line.startswith("proxy-password|") and \ 1078 (split_line_len == 2): 1079 1080 httpproxy = split_line[1].strip().split() 1081 if httpproxy: 1082 data['proxy']['password'] = httpproxy[-1] 1083 1084 elif line.startswith("system-name|") and \ 1085 (split_line_len == 2): 1086 1087 data['name'] = split_line[1].strip() 1088 1089 elif line.startswith("nice-level|") and \ 1090 (split_line_len == 2): 1091 1092 mylevel = split_line[1].strip() 1093 try: 1094 mylevel = int(mylevel) 1095 if (mylevel >= -19) and (mylevel <= 19): 1096 const_set_nice_level(mylevel) 1097 except (ValueError,): 1098 continue 1099 1100 return data

1101

1102 - def _repositories_parser(self):

1103 1104 """ 1105 Setup Entropy Client repository settings reading them from 1106 the relative config file specified in etpConst['repositoriesconf'] 1107 1108 @return: parsed metadata 1109 @rtype: dict 1110 """ 1111 1112 data = { 1113 'available': {}, 1114 'excluded': {}, 1115 'order': [], 1116 'product': etpConst['product'], 1117 'branch': etpConst['branch'], 1118 'default_repository': etpConst['officialrepositoryid'], 1119 'transfer_limit': etpConst['downloadspeedlimit'], 1120 'security_advisories_url': etpConst['securityurl'], 1121 } 1122 1123 repo_conf = etpConst['repositoriesconf'] 1124 if not (os.path.isfile(repo_conf) and os.access(repo_conf, os.R_OK)): 1125 return data 1126 1127 repo_f = open(repo_conf,"r") 1128 repositoriesconf = [x.strip() for x in repo_f.readlines() if x.strip()] 1129 repo_f.close() 1130 1131 # setup product and branch first 1132 for line in repositoriesconf: 1133 1134 split_line = line.split("|") 1135 split_line_len = len(split_line) 1136 1137 if (line.find("product|") != -1) and \ 1138 (not line.startswith("#")) and (split_line_len == 2): 1139 1140 data['product'] = split_line[1] 1141 1142 elif (line.find("branch|") != -1) and \ 1143 (not line.startswith("#")) and (split_line_len == 2): 1144 1145 branch = split_line[1].strip() 1146 data['branch'] = branch 1147 if not os.path.isdir(etpConst['packagesbindir']+"/"+branch) \ 1148 and (etpConst['uid'] == 0): 1149 1150 try: 1151 os.makedirs(etpConst['packagesbindir']+"/"+branch) 1152 except (OSError, IOError,): 1153 continue 1154 1155 for line in repositoriesconf: 1156 1157 split_line = line.split("|") 1158 split_line_len = len(split_line) 1159 1160 # populate data['available'] 1161 if (line.find("repository|") != -1) and (split_line_len == 5): 1162 1163 excluded = False 1164 my_repodata = data['available'] 1165 if line.startswith("##"): 1166 continue 1167 elif line.startswith("#"): 1168 excluded = True 1169 my_repodata = data['excluded'] 1170 line = line[1:] 1171 1172 reponame, repodata = const_extract_cli_repo_params(line, 1173 data['branch'], data['product']) 1174 if my_repodata.has_key(reponame): 1175 1176 my_repodata[reponame]['plain_packages'].extend( 1177 repodata['plain_packages']) 1178 my_repodata[reponame]['packages'].extend( 1179 repodata['packages']) 1180 1181 if (not my_repodata[reponame]['plain_database']) and \ 1182 repodata['plain_database']: 1183 1184 my_repodata[reponame]['plain_database'] = \ 1185 repodata['plain_database'] 1186 my_repodata[reponame]['database'] = \ 1187 repodata['database'] 1188 my_repodata[reponame]['dbrevision'] = \ 1189 repodata['dbrevision'] 1190 my_repodata[reponame]['dbcformat'] = \ 1191 repodata['dbcformat'] 1192 else: 1193 1194 my_repodata[reponame] = repodata.copy() 1195 if not excluded: 1196 data['order'].append(reponame) 1197 1198 elif (line.find("officialrepositoryid|") != -1) and \ 1199 (not line.startswith("#")) and (split_line_len == 2): 1200 1201 officialreponame = split_line[1] 1202 data['default_repository'] = officialreponame 1203 1204 elif (line.find("downloadspeedlimit|") != -1) and \ 1205 (not line.startswith("#")) and (split_line_len == 2): 1206 1207 try: 1208 myval = int(split_line[1]) 1209 if myval > 0: 1210 data['transfer_limit'] = myval 1211 else: 1212 data['transfer_limit'] = None 1213 except (ValueError, IndexError,): 1214 data['transfer_limit'] = None 1215 1216 elif (line.find("securityurl|") != -1) and \ 1217 (not line.startswith("#")) and (split_line_len == 2): 1218 1219 try: 1220 data['security_advisories_url'] = split_line[1] 1221 except (IndexError, ValueError, TypeError,): 1222 continue 1223 1224 return data

1225

1226 - def _clear_repository_cache(self, repoid = None):

1227 """ 1228 Internal method, go away! 1229 """ 1230 self.__cacher.discard() 1231 self._clear_dump_cache(etpCache['world_available']) 1232 self._clear_dump_cache(etpCache['world_update']) 1233 self._clear_dump_cache(etpCache['critical_update']) 1234 self._clear_dump_cache(etpCache['check_package_update']) 1235 self._clear_dump_cache(etpCache['filter_satisfied_deps']) 1236 self._clear_dump_cache(etpCache['atomMatch']) 1237 self._clear_dump_cache(etpCache['dep_tree']) 1238 if repoid != None: 1239 self._clear_dump_cache("%s/%s%s/" % ( 1240 etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) 1241 self._clear_dump_cache("%s/%s%s/" % ( 1242 etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,))

1243

1244 - def _clear_dump_cache(self, dump_name, skip = []):

1245 """ 1246 Internal method, go away! 1247 """ 1248 dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) 1249 dump_dir = os.path.dirname(dump_path) 1250 #dump_file = os.path.basename(dump_path) 1251 for currentdir, subdirs, files in os.walk(dump_dir): 1252 path = os.path.join(dump_dir,currentdir) 1253 if skip: 1254 found = False 1255 for myskip in skip: 1256 if path.find(myskip) != -1: 1257 found = True 1258 break 1259 if found: continue 1260 for item in files: 1261 if item.endswith(etpConst['cachedumpext']): 1262 item = os.path.join(path,item) 1263 try: 1264 os.remove(item) 1265 except (OSError, IOError,): 1266 pass 1267 try: 1268 if not os.listdir(path): 1269 os.rmdir(path) 1270 except (OSError, IOError,): 1271 pass

1272

1273 - def __generic_parser(self, filepath):

1274 """ 1275 Internal method. This is the generic file parser here. 1276 1277 @param filepath: valid path 1278 @type filepath: string 1279 @return: raw text extracted from file 1280 @rtype: list 1281 """ 1282 return self.entropyTools.generic_file_content_parser(filepath)

1283

1284 - def __remove_repo_cache(self, repoid = None):

1285 """ 1286 Internal method. Remove repository cache, because not valid anymore. 1287 1288 @keyword repoid: repository identifier or None 1289 @type repoid: string or None 1290 @return: None 1291 @rtype: None 1292 """ 1293 if os.path.isdir(etpConst['dumpstoragedir']): 1294 if repoid: 1295 self._clear_repository_cache(repoid = repoid) 1296 return 1297 for repoid in self['repositories']['order']: 1298 self._clear_repository_cache(repoid = repoid) 1299 else: 1300 try: 1301 os.makedirs(etpConst['dumpstoragedir']) 1302 except IOError, e: 1303 if e.errno == 30: # readonly filesystem 1304 etpUi['pretend'] = True 1305 return 1306 except OSError: 1307 return

1308

1309 - def __save_file_mtime(self, toread, tosaveinto):

1310 """ 1311 Internal method. Save mtime of a file to another file. 1312 1313 @param toread: file path to read 1314 @type toread: string 1315 @param tosaveinto: path where to save retrieved mtime information 1316 @type tosaveinto: string 1317 @return: None 1318 @rtype: None 1319 """ 1320 if not os.path.isfile(toread): 1321 currmtime = 0.0 1322 else: 1323 currmtime = os.path.getmtime(toread) 1324 1325 if not os.path.isdir(etpConst['dumpstoragedir']): 1326 try: 1327 os.makedirs(etpConst['dumpstoragedir'], 0775) 1328 const_setup_perms(etpConst['dumpstoragedir'], 1329 etpConst['entropygid']) 1330 except IOError, e: 1331 if e.errno == 30: # readonly filesystem 1332 etpUi['pretend'] = True 1333 return 1334 except (OSError,), e: 1335 # unable to create the storage directory 1336 # useless to continue 1337 return 1338 1339 try: 1340 mtime_f = open(tosaveinto,"w") 1341 except IOError, e: # unable to write? 1342 if e.errno == 30: # readonly filesystem 1343 etpUi['pretend'] = True 1344 return 1345 else: 1346 mtime_f.write(str(currmtime)) 1347 mtime_f.flush() 1348 mtime_f.close() 1349 os.chmod(tosaveinto, 0664) 1350 if etpConst['entropygid'] != None: 1351 os.chown(tosaveinto, 0, etpConst['entropygid'])

1352 1353

1354 - def validate_entropy_cache(self, settingfile, mtimefile, repoid = None):

1355 """ 1356 Internal method. Validates Entropy Cache based on a setting file 1357 and its stored (cache bound) mtime. 1358 1359 @param settingfile: path of the setting file 1360 @type settingfile: string 1361 @param mtimefile: path where to save retrieved mtime information 1362 @type mtimefile: string 1363 @keyword repoid: repository identifier or None 1364 @type repoid: string or None 1365 @return: None 1366 @rtype: None 1367 """ 1368 1369 # can't validate if running as user, moreover 1370 # users can't make changes, so... 1371 if os.getuid() != 0: 1372 return 1373 1374 # handle on-disk cache validation 1375 # in this case, repositories cache 1376 # if file is changed, we must destroy cache 1377 if not os.path.isfile(mtimefile): 1378 # we can't know if it has been updated 1379 # remove repositories caches 1380 self.__remove_repo_cache(repoid = repoid) 1381 self.__save_file_mtime(settingfile, mtimefile) 1382 else: 1383 # check mtime 1384 try: 1385 mtime_f = open(mtimefile,"r") 1386 mtime = mtime_f.readline().strip() 1387 mtime_f.close() 1388 # compare with current mtime 1389 try: 1390 currmtime = str(os.path.getmtime(settingfile)) 1391 except OSError: 1392 currmtime = "0.0" 1393 if mtime != currmtime: 1394 self.__remove_repo_cache(repoid = repoid) 1395 self.__save_file_mtime(settingfile, mtimefile) 1396 except (OSError, IOError,): 1397 self.__remove_repo_cache(repoid = repoid) 1398 self.__save_file_mtime(settingfile, mtimefile)

1399

Source Code for Module entropy.core