entropy.server.interfaces.main

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Server Main Interfaces}. 10 11 """ 12 import sys 13 import os 14 import shutil 15 from entropy.core import Singleton 16 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ 17 SystemDatabaseError 18 from entropy.const import etpConst, etpSys, const_setup_perms, \ 19 const_create_working_dirs, const_extract_srv_repo_params, etpUi, \ 20 const_setup_file, const_get_stringtype, const_debug_write 21 from entropy.output import TextInterface, purple, red, darkgreen, \ 22 bold, brown, blue, darkred 23 from entropy.server.interfaces.mirrors import Server as MirrorsServer 24 from entropy.i18n import _ 25 from entropy.core.settings.base import SystemSettings 26 from entropy.core.settings.plugins.skel import SystemSettingsPlugin 27 from entropy.transceivers import EntropyTransceiver 28 from entropy.db import EntropyRepository 29 from entropy.db.skel import EntropyRepositoryPlugin 30 from entropy.server.interfaces.db import ServerRepositoryStatus 31 from entropy.spm.plugins.factory import get_default_instance as get_spm, \ 32 get_default_class as get_spm_class 33 from entropy.qa import QAInterfacePlugin 34 35 SERVER_QA_PLUGIN = "ServerQAInterfacePlugin" 36

37 -class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin):

38 39 import entropy.dump as dump 40 PLUGIN_ID = "__server__" 41

42 - def __init__(self, server_interface, metadata = None):

43 """ 44 Entropy server-side repository EntropyRepository Plugin class. 45 This class will be instantiated and automatically added to 46 EntropyRepository instances generated by Entropy Server. 47 48 @param server_interface: Entropy Server interface instance 49 @type server_interface: entropy.server.interfaces.Server class 50 @param metadata: any dict form metadata map (key => value) 51 @type metadata: dict 52 """ 53 EntropyRepositoryPlugin.__init__(self) 54 self.SystemSettings = SystemSettings() 55 self.srv_sys_settings_plugin = \ 56 etpConst['system_settings_plugins_ids']['server_plugin'] 57 self._server = server_interface 58 if metadata is None: 59 self._metadata = {} 60 else: 61 self._metadata = metadata 62 63 # make sure we set client_repo metadata to False, this indicates 64 # EntropyRepository that we are a server-side repository 65 # Of course, it shouldn't make any diff to not set this, but we 66 # really want to make sure it's always enforced. 67 self._metadata['client_repo'] = False

68

69 - def get_id(self):

70 return ServerEntropyRepositoryPlugin.PLUGIN_ID

71

72 - def get_metadata(self):

73 """ 74 This method should always return a direct reference to the object and 75 NOT a copy. 76 """ 77 return self._metadata

78

79 - def add_plugin_hook(self, entropy_repository_instance):

80 const_debug_write(__name__, 81 "ServerEntropyRepositoryPlugin: calling add_plugin_hook => %s" % ( 82 self,) 83 ) 84 85 use_branch = self._metadata.get('use_branch') 86 if use_branch is not None: 87 entropy_repository_instance.db_branch = use_branch 88 89 out_intf = self._metadata.get('output_interface') 90 if out_intf is not None: 91 entropy_repository_instance.updateProgress = out_intf.updateProgress 92 entropy_repository_instance.askQuestion = out_intf.askQuestion 93 94 return 0

95

96 - def close_repo_hook(self, entropy_repository_instance):

97 const_debug_write(__name__, 98 "ServerEntropyRepositoryPlugin: calling close_repo_hook => %s" % ( 99 self,) 100 ) 101 102 # this happens because close_server_repositories() might be called 103 # before setup_services() and in general, at any time, so, in this 104 # case, there is no need to print bullshit to dev. 105 if self._server.MirrorsService is None: 106 return 0 107 108 repo = entropy_repository_instance.dbname 109 dbfile = entropy_repository_instance.dbFile 110 sts = ServerRepositoryStatus() 111 if sts.is_tainted(dbfile) and not sts.is_unlock_msg(dbfile): 112 u_msg = _("Mirrors have not been unlocked. Remember to sync them.") 113 self._server.updateProgress( 114 darkgreen(u_msg), 115 importance = 1, 116 type = "info", 117 header = brown(" * ") 118 ) 119 # avoid spamming 120 sts.set_unlock_msg(dbfile) 121 122 return 0

123

124 - def commit_hook(self, entropy_repository_instance):

125 126 const_debug_write(__name__, 127 "ServerEntropyRepositoryPlugin: calling commit_hook => %s" % ( 128 self,) 129 ) 130 131 dbs = ServerRepositoryStatus() 132 dbfile = entropy_repository_instance.dbFile 133 repo = entropy_repository_instance.dbname 134 read_only = entropy_repository_instance.readOnly 135 if read_only: 136 # do not taint database 137 return 0 138 139 # taint the database status 140 taint_file = self._server.get_local_database_taint_file(repo = repo) 141 f = open(taint_file, "w") 142 f.write(etpConst['currentarch']+" database tainted\n") 143 f.flush() 144 f.close() 145 const_setup_file(taint_file, etpConst['entropygid'], 0o664) 146 dbs.set_tainted(dbfile) 147 148 if not dbs.is_bumped(dbfile): 149 # bump revision, setting DatabaseBump causes 150 # the session to just bump once 151 dbs.set_bumped(dbfile) 152 """ 153 Entropy repository revision bumping function. Every time it's called, 154 revision is incremented by 1. 155 """ 156 revision_file = self._server.get_local_database_revision_file( 157 repo = repo) 158 if not os.path.isfile(revision_file): 159 revision = 1 160 else: 161 with open(revision_file, "r") as rev_f: 162 revision = int(rev_f.readline().strip()) 163 revision += 1 164 165 tmp_revision_file = revision_file + ".tmp" 166 with open(tmp_revision_file, "w") as rev_fw: 167 rev_fw.write(str(revision)+"\n") 168 rev_fw.flush() 169 # atomic ! 170 os.rename(tmp_revision_file, revision_file) 171 172 return 0

173

174 - def _get_category_description_from_disk(self, category):

175 """ 176 Get category name description from Source Package Manager. 177 178 @param category: category name 179 @type category: string 180 @return: category description 181 @rtype: string 182 """ 183 spm = self._server.Spm() 184 return spm.get_package_category_description_metadata(category)

185

186 - def _write_rss_for_removed_package(self, repo_db, idpackage):

187 188 # setup variables we're going to use 189 srv_repo = repo_db.dbname 190 rss_revision = repo_db.retrieveRevision(idpackage) 191 rss_atom = "%s~%s" % (repo_db.retrieveAtom(idpackage), rss_revision,) 192 status = ServerRepositoryStatus() 193 srv_updates = status.get_updates_log(srv_repo) 194 rss_name = srv_repo + etpConst['rss-dump-name'] 195 196 # load metadata from on disk cache, if available 197 rss_obj = self.dump.loadobj(rss_name) 198 if rss_obj: 199 srv_updates.update(rss_obj) 200 201 # setup metadata keys, if not available 202 if 'added' not in srv_updates: 203 srv_updates['added'] = {} 204 if 'removed' not in srv_updates: 205 srv_updates['removed'] = {} 206 if 'light' not in srv_updates: 207 srv_updates['light'] = {} 208 209 # if pkgatom (rss_atom) is in the "added" metadata, drop it 210 if rss_atom in srv_updates['added']: 211 del srv_updates['added'][rss_atom] 212 # same thing for light key 213 if rss_atom in srv_updates['light']: 214 del srv_updates['light'][rss_atom] 215 216 # add metadata 217 mydict = {} 218 try: 219 mydict['description'] = repo_db.retrieveDescription(idpackage) 220 except TypeError: 221 mydict['description'] = "N/A" 222 try: 223 mydict['homepage'] = repo_db.retrieveHomepage(idpackage) 224 except TypeError: 225 mydict['homepage'] = "" 226 srv_updates['removed'][rss_atom] = mydict 227 228 # save to disk 229 self.dump.dumpobj(rss_name, srv_updates)

230

231 - def _write_rss_for_added_package(self, repo_db, package_data):

232 233 # setup variables we're going to use 234 srv_repo = repo_db.dbname 235 rss_atom = "%s~%s" % (package_data['atom'], package_data['revision'],) 236 status = ServerRepositoryStatus() 237 srv_updates = status.get_updates_log(srv_repo) 238 rss_name = srv_repo + etpConst['rss-dump-name'] 239 240 # load metadata from on disk cache, if available 241 rss_obj = self.dump.loadobj(rss_name) 242 if rss_obj: 243 srv_updates.update(rss_obj) 244 245 # setup metadata keys, if not available 246 if 'added' not in srv_updates: 247 srv_updates['added'] = {} 248 if 'removed' not in srv_updates: 249 srv_updates['removed'] = {} 250 if 'light' not in srv_updates: 251 srv_updates['light'] = {} 252 253 # if package_data['atom'] (rss_atom) is in the 254 # "removed" metadata, drop it 255 if rss_atom in srv_updates['removed']: 256 del srv_updates['removed'][rss_atom] 257 258 # add metadata 259 srv_updates['added'][rss_atom] = {} 260 srv_updates['added'][rss_atom]['description'] = \ 261 package_data['description'] 262 srv_updates['added'][rss_atom]['homepage'] = \ 263 package_data['homepage'] 264 srv_updates['light'][rss_atom] = {} 265 srv_updates['light'][rss_atom]['description'] = \ 266 package_data['description'] 267 268 # save to disk 269 self.dump.dumpobj(rss_name, srv_updates)

270

271 - def add_package_hook(self, entropy_repository_instance, idpackage, 272 package_data):

273 274 const_debug_write(__name__, 275 "ServerEntropyRepositoryPlugin: calling add_package_hook => %s" % ( 276 self,) 277 ) 278 279 # handle server-side repo RSS support 280 sys_set_plug = self.srv_sys_settings_plugin 281 if self.SystemSettings[sys_set_plug]['server']['rss']['enabled']: 282 self._write_rss_for_added_package(entropy_repository_instance, 283 package_data) 284 285 try: 286 descdata = self._get_category_description_from_disk( 287 package_data['category']) 288 entropy_repository_instance.setCategoryDescription( 289 package_data['category'], descdata) 290 except (IOError, OSError, EOFError,): 291 pass 292 293 return 0

294

295 - def remove_package_hook(self, entropy_repository_instance, idpackage, 296 from_add_package):

297 298 const_debug_write(__name__, 299 "ServerEntropyRepositoryPlugin: calling remove_package_hook => %s" % ( 300 self,) 301 ) 302 303 # handle server-side repo RSS support 304 sys_set_plug = self.srv_sys_settings_plugin 305 if self.SystemSettings[sys_set_plug]['server']['rss']['enabled'] \ 306 and (not from_add_package): 307 308 # store addPackage action 309 self._write_rss_for_removed_package(entropy_repository_instance, 310 idpackage) 311 312 return 0

313

314 - def treeupdates_move_action_hook(self, entropy_repository_instance, 315 idpackage):

316 # check for injection and warn the developer 317 injected = entropy_repository_instance.isInjected(idpackage) 318 new_atom = entropy_repository_instance.retrieveAtom(idpackage) 319 if injected: 320 mytxt = "%s: %s %s. %s !!! %s." % ( 321 bold(_("INJECT")), 322 blue(str(new_atom)), 323 red(_("has been injected")), 324 red(_("quickpkg manually to update embedded db")), 325 red(_("Repository database updated anyway")), 326 ) 327 self._server.updateProgress( 328 mytxt, 329 importance = 1, 330 type = "warning", 331 header = darkred(" * ") 332 ) 333 return 0

334

335 - def treeupdates_slot_move_action_hook(self, entropy_repository_instance, 336 idpackage):

337 return self.treeupdates_move_action_hook(entropy_repository_instance, 338 idpackage)

339

340 - def reverse_dependencies_tree_generation_hook(self, 341 entropy_repository_instance):

342 # force commit even if readonly, this will allow 343 # to automagically fix dependstable server side 344 # we don't care much about syncing the 345 # database since it's a quite trivial change 346 entropy_repository_instance.commitChanges(force = True, 347 no_plugins = True) 348 349 return 0

350 351

352 -class ServerSystemSettingsPlugin(SystemSettingsPlugin):

353

354 - def server_parser(self, sys_set):

355 356 """ 357 Parses Entropy server system configuration file. 358 359 @return dict data 360 """ 361 362 data = { 363 'repositories': etpConst['server_repositories'].copy(), 364 'qa_langs': ["en_US", "C"], 365 'default_repository_id': etpConst['defaultserverrepositoryid'], 366 'base_repository_id': None, 367 'packages_expiration_days': etpConst['packagesexpirationdays'], 368 'database_file_format': etpConst['etpdatabasefileformat'], 369 'disabled_eapis': set(), 370 'exp_based_scope': etpConst['expiration_based_scope'], 371 'sync_speed_limit': None, 372 'rss': { 373 'enabled': etpConst['rss-feed'], 374 'name': etpConst['rss-name'], 375 'base_url': etpConst['rss-base-url'], 376 'website_url': etpConst['rss-website-url'], 377 'editor': etpConst['rss-managing-editor'], 378 'max_entries': etpConst['rss-max-entries'], 379 'light_max_entries': etpConst['rss-light-max-entries'], 380 }, 381 } 382 383 fake_instance = self._helper.fake_default_repo 384 385 if not os.access(etpConst['serverconf'], os.R_OK): 386 return data 387 388 with open(etpConst['serverconf'], "r") as server_f: 389 serverconf = [x.strip() for x in server_f.readlines() if x.strip()] 390 391 default_repo_changed = False 392 393 for line in serverconf: 394 395 split_line = line.split("|") 396 split_line_len = len(split_line) 397 398 # TODO: remove this in future, supported for backward compat. 399 if (line.find("officialserverrepositoryid|") != -1) and \ 400 (not line.startswith("#")) and (split_line_len == 2): 401 402 # TODO: added for backward and mixed compat. 403 if default_repo_changed: 404 continue 405 406 if not fake_instance: 407 data['default_repository_id'] = split_line[1].strip() 408 409 elif (line.find("default-repository|") != -1) and \ 410 (not line.startswith("#")) and (split_line_len == 2): 411 412 if not fake_instance: 413 data['default_repository_id'] = split_line[1].strip() 414 default_repo_changed = True 415 416 elif line.startswith("expiration-days|") and (split_line_len == 2): 417 418 mydays = split_line[1].strip() 419 try: 420 mydays = int(mydays) 421 data['packages_expiration_days'] = mydays 422 except ValueError: 423 continue 424 425 elif line.startswith("expiration-based-scope|") and \ 426 (split_line_len == 2): 427 428 exp_opt = split_line[1].strip().lower() 429 if exp_opt in ("enable", "enabled", "true", "1", "yes"): 430 data['exp_based_scope'] = True 431 else: 432 data['exp_based_scope'] = False 433 434 elif line.startswith("disabled-eapis|") and (split_line_len == 2): 435 436 mydis = split_line[1].strip().split(",") 437 try: 438 mydis = [int(x) for x in mydis] 439 mydis = set([x for x in mydis if x in (1, 2, 3,)]) 440 except ValueError: 441 continue 442 if (len(mydis) < 3) and mydis: 443 data['disabled_eapis'] = mydis 444 445 elif line.startswith("server-basic-languages|") and \ 446 (split_line_len == 2): 447 data['qa_langs'] = split_line[1].strip().split() 448 449 elif line.startswith("repository|") and (split_line_len in (5, 6)) \ 450 and (not fake_instance): 451 452 repoid, repodata = const_extract_srv_repo_params(line, 453 product = sys_set['repositories']['product']) 454 if repoid in data['repositories']: 455 # just update mirrors 456 data['repositories'][repoid]['mirrors'].extend( 457 repodata['mirrors']) 458 else: 459 data['repositories'][repoid] = repodata.copy() 460 461 # base_repository_id support 462 if data['base_repository_id'] is None: 463 data['base_repository_id'] = repoid 464 465 elif line.startswith("database-format|") and (split_line_len == 2): 466 467 fmt = split_line[1] 468 if fmt in etpConst['etpdatabasesupportedcformats']: 469 data['database_file_format'] = fmt 470 471 elif line.startswith("syncspeedlimit|") and (split_line_len == 2): 472 473 try: 474 speed_limit = int(split_line[1]) 475 except ValueError: 476 speed_limit = None 477 data['sync_speed_limit'] = speed_limit 478 479 elif line.startswith("rss-feed|") and (split_line_len == 2): 480 481 feed = split_line[1] 482 if feed in ("enable", "enabled", "true", "1"): 483 data['rss']['enabled'] = True 484 elif feed in ("disable", "disabled", "false", "0", "no",): 485 data['rss']['enabled'] = False 486 487 elif line.startswith("rss-name|") and (split_line_len == 2): 488 489 feedname = line.split("rss-name|")[1].strip() 490 data['rss']['name'] = feedname 491 492 elif line.startswith("rss-base-url|") and (split_line_len == 2): 493 494 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip() 495 if not data['rss']['base_url'][-1] == "/": 496 data['rss']['base_url'] += "/" 497 498 elif line.startswith("rss-website-url|") and (split_line_len == 2): 499 500 data['rss']['website_url'] = split_line[1].strip() 501 502 elif line.startswith("managing-editor|") and (split_line_len == 2): 503 504 data['rss']['editor'] = split_line[1].strip() 505 506 elif line.startswith("max-rss-entries|") and (split_line_len == 2): 507 508 try: 509 entries = int(split_line[1].strip()) 510 data['rss']['max_entries'] = entries 511 except (ValueError, IndexError,): 512 continue 513 514 elif line.startswith("max-rss-light-entries|") and \ 515 (split_line_len == 2): 516 517 try: 518 entries = int(split_line[1].strip()) 519 data['rss']['light_max_entries'] = entries 520 except (ValueError, IndexError,): 521 continue 522 523 # add system database if community repository mode is enabled 524 if self._helper.community_repo: 525 data['repositories'][etpConst['clientserverrepoid']] = {} 526 mydata = {} 527 mydata['description'] = "Community Repositories System Database" 528 mydata['mirrors'] = [] 529 mydata['community'] = False 530 data['repositories'][etpConst['clientserverrepoid']].update(mydata) 531 # installed packages repository is now the base repository 532 data['base_repository_id'] = etpConst['clientserverrepoid'] 533 534 # expand paths 535 for repoid in data['repositories']: 536 data['repositories'][repoid]['packages_dir'] = \ 537 os.path.join( etpConst['entropyworkdir'], 538 "server", 539 repoid, 540 "packages", 541 etpSys['arch'] 542 ) 543 data['repositories'][repoid]['store_dir'] = \ 544 os.path.join( etpConst['entropyworkdir'], 545 "server", 546 repoid, 547 "store", 548 etpSys['arch'] 549 ) 550 data['repositories'][repoid]['upload_dir'] = \ 551 os.path.join( etpConst['entropyworkdir'], 552 "server", 553 repoid, 554 "upload", 555 etpSys['arch'] 556 ) 557 data['repositories'][repoid]['database_dir'] = \ 558 os.path.join( etpConst['entropyworkdir'], 559 "server", 560 repoid, 561 "database", 562 etpSys['arch'] 563 ) 564 data['repositories'][repoid]['packages_relative_path'] = \ 565 os.path.join( sys_set['repositories']['product'], 566 repoid, 567 "packages", 568 etpSys['arch'] 569 )+"/" 570 data['repositories'][repoid]['database_relative_path'] = \ 571 os.path.join( sys_set['repositories']['product'], 572 repoid, 573 "database", 574 etpSys['arch'] 575 )+"/" 576 577 # Support for shell variables 578 shell_repoid = os.getenv('ETP_REPO') 579 if shell_repoid: 580 data['default_repository_id'] = shell_repoid 581 582 expiration_days = os.getenv('ETP_EXPIRATION_DAYS') 583 if expiration_days: 584 try: 585 expiration_days = int(expiration_days) 586 data['packages_expiration_days'] = expiration_days 587 except ValueError: 588 pass 589 590 return data

591

592 -class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

593 594 import entropy.tools as entropyTools 595

596 - def repos_parser(self, sys_set):

597 598 data = {} 599 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 600 # if support is not enabled, don't waste time scanning files 601 srv_parser_data = sys_set[srv_plug_id]['server'] 602 if not srv_parser_data['exp_based_scope']: 603 return data 604 605 # get expiration-based packages removal data from config files 606 for repoid in srv_parser_data['repositories']: 607 608 # filter out system repository if community repository 609 # mode is enabled 610 if repoid == etpConst['clientserverrepoid']: 611 continue 612 613 idpackages = set() 614 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file( 615 repo = repoid) 616 dbconn = self._helper.open_server_repository( 617 just_reading = True, repo = repoid) 618 619 if os.access(exp_fp, os.R_OK) and os.path.isfile(exp_fp): 620 pkgs = self.entropyTools.generic_file_content_parser(exp_fp) 621 if '*' in pkgs: # wildcard support 622 idpackages.add(-1) 623 else: 624 for pkg in pkgs: 625 idpackage, rc_match = dbconn.atomMatch(pkg) 626 if rc_match: 627 continue 628 idpackages.add(idpackage) 629 630 data[repoid] = idpackages 631 632 return data

633

634 -class ServerQAInterfacePlugin(QAInterfacePlugin):

635

636 - def __check_package_using_spm(self, package_path):

637 638 spm_class = get_spm_class() 639 spm_rc, spm_msg = spm_class.execute_qa_tests(package_path) 640 641 if spm_rc == 0: 642 return True 643 sys.stderr.write("QA Error: " + spm_msg + "\n") 644 sys.stderr.flush() 645 return False

646

647 - def get_tests(self):

648 return [self.__check_package_using_spm]

649

650 - def get_id(self):

651 return SERVER_QA_PLUGIN

652 653

654 -class Server(Singleton, TextInterface):

655 656 _memory_db_instances = {} 657 _treeupdates_repos = set() 658

659 - def init_singleton(self, default_repository = None, save_repository = False, 660 community_repo = False, fake_default_repo = False, 661 fake_default_repo_id = '::fake::', 662 fake_default_repo_desc = 'this is a fake repository'):

663 664 self.__instance_destroyed = False 665 if etpConst['uid'] != 0: 666 mytxt = _("Entropy Server interface must be run as root") 667 import warnings 668 warnings.warn(mytxt) 669 670 # settings 671 self.__server_dbcache = {} 672 self.dbcache_load_metadata = {} 673 self.SystemSettings = SystemSettings() 674 self.community_repo = community_repo 675 from entropy.db import dbapi2 676 self.dbapi2 = dbapi2 # export for third parties 677 etpSys['serverside'] = True 678 self.fake_default_repo = fake_default_repo 679 self.indexing = False 680 self.xcache = False 681 self.MirrorsService = None 682 self.__settings_to_backup = [] 683 self.__do_save_repository = save_repository 684 self.__sync_lock_cache = set() 685 self.rssMessages = { 686 'added': {}, 687 'removed': {}, 688 'commitmessage': "", 689 'light': {}, 690 } 691 692 self.sys_settings_fatscope_plugin_id = \ 693 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 694 self.sys_settings_plugin_id = \ 695 etpConst['system_settings_plugins_ids']['server_plugin'] 696 697 # create our SystemSettings plugin 698 self.sys_settings_plugin = ServerSystemSettingsPlugin( 699 self.sys_settings_plugin_id, self) 700 self.SystemSettings.add_plugin(self.sys_settings_plugin) 701 702 # Fatscope support SystemSettings plugin 703 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin( 704 self.sys_settings_fatscope_plugin_id, self) 705 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin) 706 707 # setup fake repository 708 if fake_default_repo: 709 default_repository = fake_default_repo_id 710 self.init_generic_memory_server_repository(fake_default_repo_id, 711 fake_default_repo_desc, set_as_default = True) 712 713 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 714 self.default_repository = default_repository 715 if self.default_repository is None: 716 self.default_repository = srv_set['default_repository_id'] 717 718 if self.default_repository in srv_set['repositories']: 719 self.ensure_paths(self.default_repository) 720 721 if self.default_repository not in srv_set['repositories']: 722 raise PermissionDenied("PermissionDenied: %s %s" % ( 723 self.default_repository, 724 _("repository not configured"), 725 ) 726 ) 727 if etpConst['clientserverrepoid'] == self.default_repository: 728 raise PermissionDenied("PermissionDenied: %s %s" % ( 729 etpConst['clientserverrepoid'], 730 _("protected repository id, can't use this, sorry dude..."), 731 ) 732 ) 733 734 self.switch_default_repository(self.default_repository)

735

736 - def destroy(self):

737 self.__instance_destroyed = True 738 if hasattr(self, 'ClientService'): 739 self.ClientService.destroy() 740 if hasattr(self, 'sys_settings_server_plugin'): 741 try: 742 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id) 743 except KeyError: 744 pass 745 if hasattr(self, 'sys_settings_fatscope_plugin'): 746 try: 747 self.SystemSettings.remove_plugin( 748 self.sys_settings_fatscope_plugin) 749 except KeyError: 750 pass 751 self.close_server_databases()

752

753 - def is_destroyed(self):

754 return self.__instance_destroyed

755

756 - def __del__(self):

757 self.destroy()

758

759 - def ensure_paths(self, repo):

760 upload_dir = os.path.join(self.get_local_upload_directory(repo), 761 self.SystemSettings['repositories']['branch']) 762 db_dir = self.get_local_database_dir(repo) 763 for mydir in [upload_dir, db_dir]: 764 if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)): 765 os.makedirs(mydir) 766 const_setup_perms(mydir, etpConst['entropygid'])

767

768 - def setup_services(self):

769 self.setup_entropy_settings() 770 cs_name = 'ClientService' 771 if hasattr(self, cs_name): 772 obj = getattr(self, cs_name) 773 # no need to close, remove, whatever QA plugins, it's automagically 774 # arranged 775 obj.destroy() 776 from entropy.client.interfaces import Client 777 self.ClientService = Client( 778 indexing = self.indexing, 779 xcache = self.xcache, 780 repo_validation = False, 781 noclientdb = 1 782 ) 783 from entropy.cache import EntropyCacher 784 self.Cacher = EntropyCacher() 785 self.ClientService.updateProgress = self.updateProgress 786 self.validRepositories = self.ClientService.validRepositories 787 self.entropyTools = self.ClientService.entropyTools 788 self.dumpTools = self.ClientService.dumpTools 789 self.backup_entropy_settings() 790 791 self.MirrorsService = MirrorsServer(self)

792

793 - def QA(self):

794 """ 795 Get Entropy QA Interface instance. 796 797 @return: Entropy QA Interface instance 798 @rtype: entropy.qa.QAInterface instance 799 """ 800 qa_plugin = ServerQAInterfacePlugin() 801 qa = self.ClientService.QA() 802 qa.add_plugin(qa_plugin) 803 return qa

804

805 - def Spm(self):

806 """ 807 Get Source Package Manager interface instance. 808 809 @return: Source Package Manager interface instance 810 @rtype: entropy.spm.plugins.skel.SpmPlugin based instance 811 """ 812 return get_spm(self)

813

814 - def Spm_class(self):

815 """ 816 Get Source Package Manager interface class. 817 """ 818 return get_spm_class()

819

820 - def Transceiver(self, uri):

821 """ 822 Get EntropyTransceiver interface instance. 823 824 @param uri: EntropyTransceiver URI 825 @type uri: string 826 @return: EntropyTransceiver instance 827 @rtype: entropy.transceivers.EntropyTransceiver 828 """ 829 txc = EntropyTransceiver(uri) 830 txc.set_output_interface(self) 831 return txc

832

833 - def setup_entropy_settings(self, repo = None):

834 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 835 backup_list = [ 836 'etpdatabaseclientfilepath', 837 'clientdbid', 838 {'server': srv_set.copy()}, 839 ] 840 for setting in backup_list: 841 if setting not in self.__settings_to_backup: 842 self.__settings_to_backup.append(setting) 843 # setup client database 844 if not self.community_repo: 845 etpConst['etpdatabaseclientfilepath'] = \ 846 self.get_local_database_file(repo) 847 etpConst['clientdbid'] = etpConst['serverdbid'] 848 const_create_working_dirs()

849

850 - def close_server_databases(self):

851 for item in self.__server_dbcache: 852 try: 853 self.__server_dbcache[item].closeDB() 854 except self.dbapi2.ProgrammingError: # already closed? 855 pass 856 self.__server_dbcache.clear()

857

858 - def close_server_database(self, dbinstance):

859 found = None 860 for item in self.__server_dbcache: 861 if dbinstance == self.__server_dbcache[item]: 862 found = item 863 break 864 if found: 865 instance = self.__server_dbcache.pop(found) 866 instance.closeDB()

867

868 - def get_available_repositories(self):

869 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 870 return srv_set['repositories'].copy()

871

872 - def switch_default_repository(self, repoid, save = None, 873 handle_uninitialized = True):

874 875 # avoid setting __default__ as default server repo 876 if repoid == etpConst['clientserverrepoid']: 877 return 878 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 879 880 if save is None: 881 save = self.__do_save_repository 882 if repoid not in srv_set['repositories']: 883 raise PermissionDenied("PermissionDenied: %s %s" % ( 884 repoid, 885 _("repository not configured"), 886 ) 887 ) 888 self.close_server_databases() 889 srv_set['default_repository_id'] = repoid 890 self.default_repository = repoid 891 self.setup_services() 892 if save: 893 self.save_default_repository(repoid) 894 895 self.setup_community_repositories_settings() 896 self.show_interface_status() 897 if handle_uninitialized and etpUi['warn']: 898 self.handle_uninitialized_repository(repoid)

899

900 - def setup_community_repositories_settings(self):

901 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 902 if self.community_repo: 903 for repoid in srv_set['repositories']: 904 srv_set['repositories'][repoid]['community'] = True

905 906

907 - def handle_uninitialized_repository(self, repoid):

908 909 if not self.is_repository_initialized(repoid): 910 mytxt = blue("%s.") % ( 911 _("Your default repository is not initialized"),) 912 self.updateProgress( 913 "[%s:%s] %s" % ( 914 brown("repo"), 915 purple(repoid), 916 mytxt, 917 ), 918 importance = 1, 919 type = "warning", 920 header = darkred(" !!! ") 921 ) 922 answer = self.askQuestion( 923 _("Do you want to initialize your default repository ?")) 924 if answer == _("No"): 925 mytxt = red("%s.") % ( 926 _("Continuing with an uninitialized repository"),) 927 self.updateProgress( 928 "[%s:%s] %s" % ( 929 brown("repo"), 930 purple(repoid), 931 mytxt, 932 ), 933 importance = 1, 934 type = "warning", 935 header = darkred(" !!! ") 936 ) 937 else: 938 # move empty database for security sake 939 dbfile = self.get_local_database_file(repoid) 940 if os.path.isfile(dbfile): 941 shutil.move(dbfile, dbfile+".backup") 942 self.initialize_server_database(empty = True, 943 repo = repoid, warnings = False)

944 945

946 - def show_interface_status(self):

947 type_txt = _("server-side repository") 948 if self.community_repo: 949 type_txt = _("community repository") 950 # ..on repository: <repository_name> 951 mytxt = _("Entropy Server Interface Instance on repository") 952 self.updateProgress( 953 blue("%s: %s, %s: %s (%s: %s)" % ( 954 mytxt, 955 red(self.default_repository), 956 _("current branch"), 957 darkgreen(self.SystemSettings['repositories']['branch']), 958 purple(_("type")), 959 bold(type_txt), 960 ) 961 ), 962 importance = 2, 963 type = "info", 964 header = red(" @@ ") 965 ) 966 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 967 repos = list(srv_set['repositories'].keys()) 968 mytxt = blue("%s:") % (_("Currently configured repositories"),) 969 self.updateProgress( 970 mytxt, 971 importance = 1, 972 type = "info", 973 header = red(" @@ ") 974 ) 975 for repo in repos: 976 self.updateProgress( 977 darkgreen(repo), 978 importance = 0, 979 type = "info", 980 header = brown(" # ") 981 )

982 983

984 - def save_default_repository(self, repoid):

985 986 # avoid setting __default__ as default server repo 987 if repoid == etpConst['clientserverrepoid']: 988 return 989 990 if os.path.isfile(etpConst['serverconf']): 991 f_srv = open(etpConst['serverconf'], "r") 992 content = f_srv.readlines() 993 f_srv.close() 994 content = [x.strip() for x in content] 995 found = False 996 new_content = [] 997 for line in content: 998 if line.strip().startswith("default-repository|"): 999 line = "default-repository|%s" % (repoid,) 1000 found = True 1001 new_content.append(line) 1002 if not found: 1003 new_content.append("default-repository|%s" % (repoid,)) 1004 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp", "w") 1005 for line in new_content: 1006 f_srv_t.write(line+"\n") 1007 f_srv_t.flush() 1008 f_srv_t.close() 1009 os.rename(etpConst['serverconf']+".save_default_repo_tmp", 1010 etpConst['serverconf']) 1011 else: 1012 f_srv = open(etpConst['serverconf'], "w") 1013 f_srv.write("default-repository|%s\n" % (repoid,)) 1014 f_srv.flush() 1015 f_srv.close()

1016

1017 - def toggle_repository(self, repoid, enable = True):

1018 1019 # avoid setting __default__ as default server repo 1020 if repoid == etpConst['clientserverrepoid']: 1021 return False 1022 1023 if not os.path.isfile(etpConst['serverconf']): 1024 return None 1025 f_srv = open(etpConst['serverconf']) 1026 tmpfile = etpConst['serverconf']+".switch" 1027 mycontent = [x.strip() for x in f_srv.readlines()] 1028 f_srv.close() 1029 f_tmp = open(tmpfile, "w") 1030 st = "repository|%s" % (repoid,) 1031 status = False 1032 for line in mycontent: 1033 if enable: 1034 if (line.find(st) != -1) and line.startswith("#") and \ 1035 (len(line.split("|")) == 5): 1036 line = line[1:] 1037 status = True 1038 else: 1039 if (line.find(st) != -1) and not line.startswith("#") and \ 1040 (len(line.split("|")) == 5): 1041 line = "#"+line 1042 status = True 1043 f_tmp.write(line+"\n") 1044 f_tmp.flush() 1045 f_tmp.close() 1046 shutil.move(tmpfile, etpConst['serverconf']) 1047 if status: 1048 self.close_server_databases() 1049 self.SystemSettings.clear() 1050 self.setup_services() 1051 self.show_interface_status() 1052 return status

1053

1054 - def backup_entropy_settings(self):

1055 for setting in self.__settings_to_backup: 1056 if isinstance(setting, const_get_stringtype()): 1057 self.ClientService.backup_constant(setting) 1058 elif isinstance(setting, dict): 1059 self.SystemSettings.set_persistent_setting(setting)

1060

1061 - def is_repository_initialized(self, repo):

1062 1063 def do_validate(dbc): 1064 try: 1065 dbc.validateDatabase() 1066 return True 1067 except SystemDatabaseError: 1068 return False

1069 1070 dbc = self.open_server_repository(just_reading = True, repo = repo) 1071 valid = do_validate(dbc) 1072 self.close_server_database(dbc) 1073 if not valid: # check online? 1074 dbc = self.open_server_repository(read_only = False, 1075 no_upload = True, repo = repo, is_new = True) 1076 valid = do_validate(dbc) 1077 self.close_server_database(dbc) 1078 1079 return valid

1080

1081 - def do_server_repository_sync_lock(self, repo, no_upload):

1082 1083 if repo is None: 1084 repo = self.default_repository 1085 1086 # check if the database is locked locally 1087 lock_file = self.MirrorsService.get_database_lockfile(repo) 1088 if os.path.isfile(lock_file): 1089 self.updateProgress( 1090 red(_("Entropy database is already locked by you :-)")), 1091 importance = 1, 1092 type = "info", 1093 header = red(" * ") 1094 ) 1095 else: 1096 # check if the database is locked REMOTELY 1097 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),) 1098 self.updateProgress( 1099 red(mytxt), 1100 importance = 1, 1101 type = "info", 1102 header = red(" * "), 1103 back = True 1104 ) 1105 for uri in self.get_remote_mirrors(repo): 1106 1107 crippled_uri = EntropyTransceiver.get_uri_name(uri) 1108 1109 given_up = self.MirrorsService.mirror_lock_check(uri, 1110 repo = repo) 1111 if given_up: 1112 mytxt = "%s:" % (_("Mirrors status table"),) 1113 self.updateProgress( 1114 darkgreen(mytxt), 1115 importance = 1, 1116 type = "info", 1117 header = brown(" * ") 1118 ) 1119 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo) 1120 for db_uri, db_st1, db_st2 in dbstatus: 1121 db_st1_info = darkgreen(_("Unlocked")) 1122 if db_st1: 1123 db_st1_info = red(_("Locked")) 1124 db_st2_info = darkgreen(_("Unlocked")) 1125 if db_st2: 1126 db_st2_info = red(_("Locked")) 1127 1128 crippled_uri = EntropyTransceiver.get_uri_name(db_uri) 1129 self.updateProgress( 1130 "%s: [%s: %s] [%s: %s]" % ( 1131 bold(crippled_uri), 1132 brown(_("database")), 1133 db_st1_info, 1134 brown(_("download")), 1135 db_st2_info, 1136 ), 1137 importance = 1, 1138 type = "info", 1139 header = "\t" 1140 ) 1141 1142 raise OnlineMirrorError("OnlineMirrorError: %s %s" % ( 1143 _("cannot lock mirror"), 1144 crippled_uri, 1145 ) 1146 ) 1147 1148 # if we arrive here, it is because all the mirrors are unlocked 1149 self.MirrorsService.lock_mirrors(True, repo = repo) 1150 self.MirrorsService.sync_databases(no_upload, repo = repo)

1151 1152

1153 - def init_generic_memory_server_repository(self, repoid, description, 1154 mirrors = None, community_repo = False, service_url = None, 1155 set_as_default = False):

1156 1157 if mirrors is None: 1158 mirrors = [] 1159 dbc = self._open_memory_database(repoid) 1160 Server._memory_db_instances[repoid] = dbc 1161 1162 eapi3_port = int(etpConst['socket_service']['port']) 1163 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port']) 1164 # add to settings 1165 repodata = { 1166 'repoid': repoid, 1167 'description': description, 1168 'mirrors': mirrors, 1169 'community': community_repo, 1170 'service_port': eapi3_port, 1171 'ssl_service_port': eapi3_ssl_port, 1172 'service_url': service_url, 1173 'handler': '', # not supported 1174 'in_memory': True, 1175 } 1176 1177 etpConst['server_repositories'][repoid] = repodata 1178 if set_as_default: 1179 etpConst['defaultserverrepositoryid'] = repoid 1180 sys_set = SystemSettings() 1181 sys_set.clear() 1182 1183 etp_repo_meta = { 1184 'lock_remote': False, 1185 'no_upload': True, 1186 'use_branch': None, 1187 'output_interface': self, 1188 } 1189 srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta) 1190 dbc.add_plugin(srv_plug) 1191 return dbc

1192

1193 - def _open_memory_database(self, repo):

1194 """ 1195 Open in-memory EntropyRepository interface. 1196 1197 @keyword dbname: database name 1198 @type dbname: string 1199 @keyword output_interface: entropy.output.TextInterface based instance 1200 @type output_interface: entropy.output.TextInterface based instance 1201 """ 1202 conn = EntropyRepository( 1203 readOnly = False, 1204 dbFile = ':memory:', 1205 dbname = repo, 1206 xcache = False, 1207 indexing = False, 1208 skipChecks = True 1209 ) 1210 conn.initializeDatabase() 1211 return conn

1212

1213 - def open_repository(self, repoid):

1214 """ 1215 This method aims to improve class usability by providing an easier 1216 method to open Entropy Server-side repositories like it happens 1217 with Entropy Client-side ones. If you just want open a read-only 1218 repository, feel free to use this wrapping method. 1219 1220 @param repoid: repository identifier 1221 @type repoid: string 1222 @return: EntropyRepository instance 1223 @rtype: entropy.db.EntropyRepository 1224 """ 1225 return self.open_server_repository(repo = repoid, just_reading = True, 1226 do_treeupdates = False)

1227

1228 - def open_server_repository( 1229 self, 1230 read_only = True, 1231 no_upload = True, 1232 just_reading = False, 1233 repo = None, 1234 indexing = True, 1235 warnings = True, 1236 do_cache = True, 1237 use_branch = None, 1238 lock_remote = True, 1239 is_new = False, 1240 do_treeupdates = True 1241 ):

1242 1243 if repo is None: 1244 repo = self.default_repository 1245 1246 if repo == etpConst['clientserverrepoid'] and self.community_repo: 1247 return self.ClientService.clientDbconn 1248 1249 # in-memory server repos 1250 if repo in Server._memory_db_instances: 1251 return Server._memory_db_instances.get(repo) 1252 1253 if just_reading: 1254 read_only = True 1255 no_upload = True 1256 1257 local_dbfile = self.get_local_database_file(repo, use_branch) 1258 if do_cache: 1259 cached = self.__server_dbcache.get( 1260 (repo, etpConst['systemroot'], local_dbfile, read_only, 1261 no_upload, just_reading, use_branch, lock_remote,) 1262 ) 1263 if cached != None: 1264 return cached 1265 1266 local_dbfile_dir = os.path.dirname(local_dbfile) 1267 if not os.path.isdir(local_dbfile_dir): 1268 os.makedirs(local_dbfile_dir) 1269 1270 if (not read_only) and (lock_remote) and \ 1271 (repo not in self.__sync_lock_cache): 1272 self.do_server_repository_sync_lock(repo, no_upload) 1273 self.__sync_lock_cache.add(repo) 1274 1275 local_dbfile_exists = os.path.lexists(local_dbfile) 1276 conn = EntropyRepository( 1277 readOnly = read_only, 1278 dbFile = local_dbfile, 1279 dbname = repo, 1280 xcache = False # always set to False, if you want to enable 1281 # you need to make sure that client-side and server-side caches 1282 # don't collide due to sharing EntropyRepository.dbname 1283 ) 1284 etp_repo_meta = { 1285 'lock_remote': lock_remote, 1286 'no_upload': no_upload, 1287 'use_branch': use_branch, 1288 'output_interface': self, 1289 } 1290 srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta) 1291 conn.add_plugin(srv_plug) 1292 """ 1293 FIXME: remove this once we have a pluggable EntropyRepository class 1294 Setup server repository status information for newly created repos. 1295 NOTE: This is a temp. workaround waiting for real pluggable 1296 EntropyRepository interface calls. 1297 """ 1298 taint_file = self.get_local_database_taint_file(repo) 1299 if os.path.isfile(taint_file): 1300 dbs = ServerRepositoryStatus() 1301 dbs.set_tainted(local_dbfile) 1302 dbs.set_bumped(local_dbfile) 1303 1304 if not local_dbfile_exists: 1305 # better than having a completely broken db 1306 conn.readOnly = False 1307 conn.initializeDatabase() 1308 conn.commitChanges() 1309 1310 valid = True 1311 try: 1312 conn.validateDatabase() 1313 except SystemDatabaseError: 1314 valid = False 1315 1316 # verify if we need to update the database to sync 1317 # with portage updates, we just ignore being readonly in the case 1318 if (repo not in Server._treeupdates_repos) and \ 1319 (not just_reading): 1320 # sometimes, when filling a new server db 1321 # we need to avoid tree updates 1322 if valid: 1323 if do_treeupdates: 1324 self.repository_packages_spm_sync(conn, 1325 branch = use_branch, repo = repo) 1326 elif warnings and not is_new: 1327 mytxt = _("Entropy database is corrupted!") 1328 self.updateProgress( 1329 darkred(mytxt), 1330 importance = 1, 1331 type = "warning", 1332 header = bold(" !!! ") 1333 ) 1334 1335 if not read_only and valid and indexing: 1336 1337 self.updateProgress( 1338 "[repo:%s|%s] %s" % ( 1339 blue(repo), 1340 red(_("database")), 1341 blue(_("indexing database")), 1342 ), 1343 importance = 1, 1344 type = "info", 1345 header = brown(" @@ ") 1346 ) 1347 conn.createAllIndexes() 1348 1349 if do_cache: 1350 # !!! also cache just_reading otherwise there will be 1351 # real issues if the connection is opened several times 1352 self.__server_dbcache[ 1353 (repo, etpConst['systemroot'], local_dbfile, read_only, 1354 no_upload, just_reading, use_branch, lock_remote,)] = conn 1355 1356 # auto-update package sets 1357 if (not read_only) and (not is_new): 1358 cur_sets = conn.retrievePackageSets() 1359 sys_sets = self.get_configured_package_sets(repo) 1360 if cur_sets != sys_sets: 1361 self.update_database_package_sets(repo, dbconn = conn) 1362 conn.commitChanges() 1363 1364 return conn

1365

1366 - def repository_packages_spm_sync(self, repo_db, branch = None, repo = None):

1367 """ 1368 Service method used to sync package names with Source Package Manager. 1369 Source Package Manager can change package names, categories or slot 1370 and Entropy repositories must be kept in sync. 1371 """ 1372 if branch is None: 1373 branch = self.SystemSettings['repositories']['branch'] 1374 if repo is None: 1375 repo = self.default_repository 1376 Server._treeupdates_repos.add(repo) 1377 self.Spm().package_names_update(repo_db, repo, self, branch)

1378

1379 - def deps_tester(self, default_repo = None):

1380 1381 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1382 server_repos = list(sys_set['repositories'].keys()) 1383 installed_packages = set() 1384 # if a default repository is passed, we will just test against it 1385 if default_repo: 1386 server_repos = [default_repo] 1387 1388 for repo in server_repos: 1389 dbconn = self.open_server_repository(read_only = True, 1390 no_upload = True, repo = repo, do_treeupdates = False) 1391 installed_packages |= set([(x, repo) for x in \ 1392 dbconn.listAllIdpackages()]) 1393 1394 1395 deps_not_satisfied = set() 1396 length = str((len(installed_packages))) 1397 count = 0 1398 mytxt = _("Checking") 1399 1400 for idpackage, repo in installed_packages: 1401 count += 1 1402 dbconn = self.open_server_repository(read_only = True, 1403 no_upload = True, repo = repo, do_treeupdates = False) 1404 1405 if (count%150 == 0) or (count == length) or (count == 1): 1406 atom = dbconn.retrieveAtom(idpackage) 1407 self.updateProgress( 1408 darkgreen(mytxt)+" "+bold(atom), 1409 importance = 0, 1410 type = "info", 1411 back = True, 1412 count = (count, length), 1413 header = darkred(" @@ ") 1414 ) 1415 1416 xdeps = dbconn.retrieveDependencies(idpackage) 1417 for xdep in xdeps: 1418 xid, xuseless = self.atom_match(xdep) 1419 if xid == -1: 1420 deps_not_satisfied.add(xdep) 1421 1422 return deps_not_satisfied

1423

1424 - def dependencies_test(self, repo = None):

1425 1426 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("...")) 1427 self.updateProgress( 1428 mytxt, 1429 importance = 2, 1430 type = "info", 1431 header = red(" @@ ") 1432 ) 1433 1434 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1435 server_repos = list(srv_set['repositories'].keys()) 1436 deps_not_matched = self.deps_tester(repo) 1437 1438 if deps_not_matched: 1439 1440 crying_atoms = {} 1441 for atom in deps_not_matched: 1442 for repo in server_repos: 1443 dbconn = self.open_server_repository(just_reading = True, 1444 repo = repo, do_treeupdates = False) 1445 riddep = dbconn.searchDependency(atom) 1446 if riddep == -1: 1447 continue 1448 ridpackages = dbconn.searchIdpackageFromIddependency(riddep) 1449 for i in ridpackages: 1450 iatom = dbconn.retrieveAtom(i) 1451 if atom not in crying_atoms: 1452 crying_atoms[atom] = set() 1453 crying_atoms[atom].add((iatom, repo)) 1454 1455 mytxt = blue("%s:") % (_("These are the dependencies not found"),) 1456 self.updateProgress( 1457 mytxt, 1458 importance = 1, 1459 type = "info", 1460 header = red(" @@ ") 1461 ) 1462 mytxt = "%s:" % (_("Needed by"),) 1463 for atom in deps_not_matched: 1464 self.updateProgress( 1465 red(atom), 1466 importance = 1, 1467 type = "info", 1468 header = blue(" # ") 1469 ) 1470 if atom in crying_atoms: 1471 self.updateProgress( 1472 red(mytxt), 1473 importance = 0, 1474 type = "info", 1475 header = blue(" # ") 1476 ) 1477 for my_dep, myrepo in crying_atoms[atom]: 1478 self.updateProgress( 1479 "[%s:%s] %s" % ( 1480 blue(_("by repo")), 1481 darkred(myrepo), 1482 darkgreen(my_dep), 1483 ), 1484 importance = 0, 1485 type = "info", 1486 header = blue(" # ") 1487 ) 1488 else: 1489 1490 mytxt = blue(_("Every dependency is satisfied. It's all fine.")) 1491 self.updateProgress( 1492 mytxt, 1493 importance = 2, 1494 type = "info", 1495 header = red(" @@ ") 1496 ) 1497 1498 return deps_not_matched

1499

1500 - def test_shared_objects(self, get_files = False, repo = None, 1501 dump_results_to_file = False):

1502 1503 pkg_list_path = None 1504 if dump_results_to_file: 1505 tmp_dir = os.path.dirname(self.entropyTools.get_random_temp_file()) 1506 pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt") 1507 dmp_data = [ 1508 (_("Broken and matched packages list"), pkg_list_path,), 1509 ] 1510 mytxt = "%s:" % (purple(_("Dumping results into these files")),) 1511 self.updateProgress( 1512 mytxt, 1513 importance = 1, 1514 type = "info", 1515 header = blue(" @@ ") 1516 ) 1517 for txt, path in dmp_data: 1518 mytxt = "%s: %s" % (blue(txt), path,) 1519 self.updateProgress( 1520 mytxt, 1521 importance = 0, 1522 type = "info", 1523 header = darkgreen(" ## ") 1524 ) 1525 1526 1527 # load db 1528 dbconn = self.open_server_repository(read_only = True, 1529 no_upload = True, repo = repo) 1530 QA = self.QA() 1531 packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn, 1532 broken_symbols = True, dump_results_to_file = dump_results_to_file) 1533 if status != 0: 1534 return 1, None 1535 1536 if get_files: 1537 return 0, brokenexecs 1538 1539 if (not brokenexecs) and (not packages_matched): 1540 mytxt = "%s." % (_("System is healthy"),) 1541 self.updateProgress( 1542 blue(mytxt), 1543 importance = 2, 1544 type = "info", 1545 header = red(" @@ ") 1546 ) 1547 return 0, None 1548 1549 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),) 1550 self.updateProgress( 1551 blue(mytxt), 1552 importance = 1, 1553 type = "info", 1554 header = red(" @@ ") 1555 ) 1556 1557 real_brokenexecs = [os.path.realpath(x) for x in brokenexecs if \ 1558 x != os.path.realpath(x)] 1559 brokenexecs.update(real_brokenexecs) 1560 packages = self.Spm().search_paths_owners(brokenexecs) 1561 1562 if packages: 1563 mytxt = "%s:" % (_("These are the matched packages"),) 1564 self.updateProgress( 1565 red(mytxt), 1566 importance = 1, 1567 type = "info", 1568 header = red(" @@ ") 1569 ) 1570 for my_atom, my_elf_id in packages: 1571 package_slot = my_atom, my_elf_id 1572 self.updateProgress( 1573 "%s [elf:%s]" % ( 1574 purple(my_atom), 1575 darkgreen(str(my_elf_id)), 1576 ), 1577 importance = 0, 1578 type = "info", 1579 header = red(" # ") 1580 ) 1581 for filename in sorted(packages[package_slot]): 1582 self.updateProgress( 1583 darkgreen(filename), 1584 importance = 0, 1585 type = "info", 1586 header = brown(" => ") 1587 ) 1588 1589 pkgstring_list = sorted(["%s:%s" % ( 1590 self.entropyTools.dep_getkey(x[0]), x[1],) for x \ 1591 in sorted(packages)]) 1592 if pkg_list_path is not None: 1593 with open(pkg_list_path, "w") as pkg_f: 1594 for pkgstr in pkgstring_list: 1595 pkg_f.write(pkgstr + "\n") 1596 pkg_f.flush() 1597 pkgstring = ' '.join(pkgstring_list) 1598 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,) 1599 self.updateProgress( 1600 mytxt, 1601 importance = 1, 1602 type = "info", 1603 header = red(" @@ ") 1604 ) 1605 else: 1606 self.updateProgress( 1607 red(_("No matched packages")), 1608 importance = 1, 1609 type = "info", 1610 header = red(" @@ ") 1611 ) 1612 1613 return 0, packages

1614

1615 - def orphaned_spm_packages_test(self):

1616 1617 mytxt = "%s %s" % ( 1618 blue(_("Running orphaned SPM packages test")), red("..."),) 1619 self.updateProgress( 1620 mytxt, 1621 importance = 2, 1622 type = "info", 1623 header = red(" @@ ") 1624 ) 1625 installed_packages = self.Spm().get_installed_packages() 1626 length = len(installed_packages) 1627 not_found = {} 1628 count = 0 1629 for installed_package in installed_packages: 1630 count += 1 1631 self.updateProgress( 1632 "%s: %s" % ( 1633 darkgreen(_("Scanning package")), 1634 brown(installed_package),), 1635 importance = 0, 1636 type = "info", 1637 back = True, 1638 count = (count, length), 1639 header = darkred(" @@ ") 1640 ) 1641 key, slot = (self.entropyTools.dep_getkey(installed_package), 1642 self.Spm().get_installed_package_metadata(installed_package, 1643 "SLOT"),) 1644 pkg_atom = "%s:%s" % (key, slot,) 1645 tree_atom = self.Spm().match_package(pkg_atom) 1646 if not tree_atom: 1647 not_found[installed_package] = pkg_atom 1648 self.updateProgress( 1649 "%s: %s" % ( 1650 blue(pkg_atom), 1651 darkred(_("not found anymore")), 1652 ), 1653 importance = 0, 1654 type = "warning", 1655 count = (count, length), 1656 header = darkred(" @@ ") 1657 ) 1658 1659 if not_found: 1660 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)]) 1661 self.updateProgress( 1662 "%s: %s" % ( 1663 blue(_("Packages string")), 1664 not_found_list, 1665 ), 1666 importance = 0, 1667 type = "warning", 1668 count = (count, length), 1669 header = darkred(" @@ ") 1670 ) 1671 1672 return not_found

1673

1674 - def depends_table_initialize(self, repo = None):

1675 dbconn = self.open_server_repository(read_only = False, 1676 no_upload = True, repo = repo) 1677 dbconn.generateReverseDependenciesMetadata() 1678 self.taint_database(repo = repo) 1679 dbconn.commitChanges()

1680

1681 - def create_empty_database(self, dbpath = None, repo = None):

1682 if dbpath is None: 1683 dbpath = self.get_local_database_file(repo) 1684 1685 dbdir = os.path.dirname(dbpath) 1686 if not os.path.isdir(dbdir): 1687 os.makedirs(dbdir) 1688 1689 mytxt = red("%s ...") % (_("Initializing an empty database"),) 1690 self.updateProgress( 1691 mytxt, 1692 importance = 1, 1693 type = "info", 1694 header = darkgreen(" * "), 1695 back = True 1696 ) 1697 dbconn = self.ClientService.open_generic_database(dbpath) 1698 dbconn.initializeDatabase() 1699 dbconn.commitChanges() 1700 dbconn.closeDB() 1701 mytxt = "%s %s %s." % ( 1702 red(_("Entropy database file")), 1703 bold(dbpath), 1704 red(_("successfully initialized")), 1705 ) 1706 self.updateProgress( 1707 mytxt, 1708 importance = 1, 1709 type = "info", 1710 header = darkgreen(" * ") 1711 )

1712

1713 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

1714 1715 # check package_tag "no spaces" 1716 1717 try: 1718 package_tag = str(package_tag) 1719 if " " in package_tag: 1720 raise ValueError 1721 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,): 1722 self.updateProgress( 1723 "%s: %s" % ( 1724 blue(_("Invalid tag specified")), 1725 package_tag, 1726 ), 1727 importance = 1, type = "error", header = darkred(" !! ") 1728 ) 1729 return 1, package_tag 1730 1731 if repo is None: 1732 repo = self.default_repository 1733 1734 # sanity check 1735 invalid_atoms = [] 1736 dbconn = self.open_server_repository(read_only = True, 1737 no_upload = True, repo = repo) 1738 for idpackage in idpackages: 1739 ver_tag = dbconn.retrieveVersionTag(idpackage) 1740 if ver_tag: 1741 invalid_atoms.append(dbconn.retrieveAtom(idpackage)) 1742 1743 if invalid_atoms: 1744 self.updateProgress( 1745 "%s: %s" % ( 1746 blue(_("Packages already tagged, action aborted")), 1747 ', '.join([darkred(str(x)) for x in invalid_atoms]), 1748 ), 1749 importance = 1, type = "error", header = darkred(" !! ") 1750 ) 1751 return 2, invalid_atoms 1752 1753 matches = [(x, repo) for x in idpackages] 1754 status = 0 1755 data = self.move_packages( 1756 matches, to_repo = repo, from_repo = repo, ask = ask, 1757 do_copy = True, new_tag = package_tag 1758 ) 1759 return status, data

1760

1761 - def flushback_packages(self, from_branches, repo = None, ask = True):

1762 """ 1763 When creating a new branch, for space reasons, packages are not 1764 moved to a new location. This works fine until old branch is removed. 1765 To avoid inconsistences, before deciding to do that, all the packages 1766 in the old branch should be flushed back to the the currently configured 1767 branch. 1768 1769 @param from_branches -- list of branches to move packages from 1770 @type from_branches -- list 1771 @param repo -- repository to work on 1772 @type repo -- str 1773 @param ask -- user interactivity 1774 @type ask -- bool 1775 1776 @return status 1777 """ 1778 1779 status = True 1780 if repo is None: 1781 repo = self.default_repository 1782 branch = self.SystemSettings['repositories']['branch'] 1783 1784 if branch in from_branches: 1785 from_branches = [x for x in from_branches if x != branch] 1786 1787 self.updateProgress( 1788 "[%s=>%s|%s] %s" % ( 1789 darkgreen(', '.join(from_branches)), 1790 darkred(branch), 1791 brown(repo), 1792 blue(_("flushing back selected packages from branches")), 1793 ), 1794 importance = 2, 1795 type = "info", 1796 header = red(" @@ ") 1797 ) 1798 1799 dbconn = self.open_server_repository(read_only = True, 1800 no_upload = True, repo = repo) 1801 1802 idpackage_map = dict(((x, [],) for x in from_branches)) 1803 idpackages = dbconn.listAllIdpackages(order_by = 'atom') 1804 for idpackage in idpackages: 1805 download_url = dbconn.retrieveDownloadURL(idpackage) 1806 url_br = self.ClientService.get_branch_from_download_relative_uri( 1807 download_url) 1808 if url_br in from_branches: 1809 idpackage_map[url_br].append(idpackage) 1810 1811 mapped_branches = [x for x in idpackage_map if idpackage_map[x]] 1812 if not mapped_branches: 1813 self.updateProgress( 1814 "[%s=>%s|%s] %s !" % ( 1815 darkgreen(', '.join(from_branches)), 1816 darkred(branch), 1817 brown(repo), 1818 blue(_("nothing to do")), 1819 ), 1820 importance = 0, 1821 type = "warning", 1822 header = blue(" @@ ") 1823 ) 1824 return status 1825 1826 1827 all_fine = True 1828 tmp_down_dir = self.entropyTools.get_random_temp_file() 1829 os.makedirs(tmp_down_dir) 1830 1831 download_queue = {} 1832 local_up_dir = self.get_local_upload_directory(repo) 1833 local_basedir = os.path.join(local_up_dir, branch) 1834 dbconn = self.open_server_repository(read_only = False, 1835 no_upload = True, repo = repo) 1836 1837 def generate_queue(branch, repo, from_branch, down_q, idpackage_map): 1838 1839 self.updateProgress( 1840 "[%s=>%s|%s] %s" % ( 1841 darkgreen(from_branch), 1842 darkred(branch), 1843 brown(repo), 1844 brown(_("these are the packages that will be flushed")), 1845 ), 1846 importance = 1, 1847 type = "info", 1848 header = brown(" @@ ") 1849 ) 1850 1851 1852 for idpackage in idpackage_map[from_branch]: 1853 atom = dbconn.retrieveAtom(idpackage) 1854 self.updateProgress( 1855 "[%s=>%s|%s] %s" % ( 1856 darkgreen(from_branch), 1857 darkred(branch), 1858 brown(repo), 1859 purple(atom), 1860 ), 1861 importance = 0, 1862 type = "info", 1863 header = blue(" # ") 1864 ) 1865 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) 1866 pkg_fp = os.path.join(tmp_down_dir, pkg_fp) 1867 down_q.append((pkg_fp, idpackage,))

1868 1869 1870 for from_branch in sorted(mapped_branches): 1871 1872 download_queue[from_branch] = [] 1873 all_fine = False 1874 generate_queue(branch, repo, from_branch, 1875 download_queue[from_branch], idpackage_map) 1876 1877 if ask: 1878 rc_question = self.askQuestion( 1879 _("Would you like to continue ?")) 1880 if rc_question == _("No"): 1881 continue 1882 1883 remote_relative_path = self.get_remote_packages_relative_path(repo) 1884 1885 for uri in self.get_remote_mirrors(repo): 1886 1887 crippled_uri = EntropyTransceiver.get_uri_name(uri) 1888 basedir = os.path.join(remote_relative_path, from_branch) 1889 1890 downloader_queue = [x[0] for x in download_queue[from_branch]] 1891 downloader = self.MirrorsService.TransceiverServerHandler( 1892 self, 1893 [uri], 1894 downloader_queue, 1895 critical_files = downloader_queue, 1896 txc_basedir = basedir, 1897 local_basedir = tmp_down_dir, 1898 download = True, 1899 repo = repo 1900 ) 1901 1902 errors, m_fine_uris, m_broken_uris = downloader.go() 1903 1904 if not errors: 1905 for downloaded_path, idpackage in \ 1906 download_queue[from_branch]: 1907 1908 self.updateProgress( 1909 "[%s=>%s|%s|%s] %s: %s" % ( 1910 darkgreen(from_branch), 1911 darkred(branch), 1912 brown(repo), 1913 dbconn.retrieveAtom(idpackage), 1914 blue(_("checking package hash")), 1915 darkgreen(os.path.basename(downloaded_path)), 1916 ), 1917 importance = 0, 1918 type = "info", 1919 header = brown(" "), 1920 back = True 1921 ) 1922 1923 md5hash = self.entropyTools.md5sum(downloaded_path) 1924 db_md5hash = dbconn.retrieveDigest(idpackage) 1925 if md5hash != db_md5hash: 1926 errors = True 1927 self.updateProgress( 1928 "[%s=>%s|%s|%s] %s: %s" % ( 1929 darkgreen(from_branch), 1930 darkred(branch), 1931 brown(repo), 1932 dbconn.retrieveAtom(idpackage), 1933 blue(_("hash does not match for")), 1934 darkgreen(os.path.basename(downloaded_path)), 1935 ), 1936 importance = 0, 1937 type = "error", 1938 header = brown(" ") 1939 ) 1940 continue 1941 1942 1943 1944 if errors: 1945 reason = _("wrong md5") 1946 if m_broken_uris: 1947 my_broken_uris = [ 1948 (EntropyTransceiver.get_uri_name(x), y,) \ 1949 for x, y in m_broken_uris] 1950 reason = my_broken_uris[0][1] 1951 1952 self.updateProgress( 1953 "[%s=>%s|%s] %s, %s: %s" % ( 1954 darkgreen(from_branch), 1955 darkred(branch), 1956 brown(repo), 1957 blue(_("download errors")), 1958 blue(_("reason")), 1959 reason, 1960 ), 1961 importance = 1, 1962 type = "error", 1963 header = darkred(" !!! ") 1964 ) 1965 # continuing if possible 1966 continue 1967 1968 all_fine = True 1969 1970 self.updateProgress( 1971 "[%s=>%s|%s] %s: %s" % ( 1972 darkgreen(from_branch), 1973 darkred(branch), 1974 brown(repo), 1975 blue(_("download completed successfully")), 1976 darkgreen(crippled_uri), 1977 ), 1978 importance = 1, 1979 type = "info", 1980 header = darkgreen(" * ") 1981 ) 1982 1983 if not all_fine: 1984 self.updateProgress( 1985 "[%s=>%s|%s] %s" % ( 1986 darkgreen(', '.join(from_branches)), 1987 darkred(branch), 1988 brown(repo), 1989 blue(_("error downloading packages from mirrors")), 1990 ), 1991 importance = 2, 1992 type = "error", 1993 header = darkred(" !!! ") 1994 ) 1995 return False 1996 1997 for from_branch in sorted(mapped_branches): 1998 1999 self.updateProgress( 2000 "[%s=>%s|%s] %s: %s" % ( 2001 darkgreen(from_branch), 2002 darkred(branch), 2003 brown(repo), 2004 blue(_("working on branch")), 2005 darkgreen(from_branch), 2006 ), 2007 importance = 1, 2008 type = "info", 2009 header = brown(" @@ ") 2010 ) 2011 2012 down_queue = download_queue[from_branch] 2013 for package_path, idpackage in down_queue: 2014 2015 self.updateProgress( 2016 "[%s=>%s|%s] %s: %s" % ( 2017 darkgreen(from_branch), 2018 darkred(branch), 2019 brown(repo), 2020 blue(_("updating package")), 2021 darkgreen(os.path.basename(package_path)), 2022 ), 2023 importance = 1, 2024 type = "info", 2025 header = brown(" "), 2026 back = True 2027 ) 2028 2029 # move files to upload 2030 package_name = os.path.basename(package_path) 2031 new_package_path = os.path.join(local_basedir, package_name) 2032 try: 2033 os.rename(package_path, new_package_path) 2034 except OSError: 2035 shutil.move(package_path, new_package_path) 2036 2037 # create md5 checksum 2038 self.entropyTools.create_md5_file(new_package_path) 2039 2040 # update database 2041 download_url = dbconn.retrieveDownloadURL(idpackage) 2042 download_url = \ 2043 self.ClientService.swap_branch_in_download_relative_uri( 2044 branch, download_url) 2045 dbconn.setDownloadURL(idpackage, download_url) 2046 dbconn.switchBranch(idpackage, branch) 2047 dbconn.commitChanges() 2048 2049 self.updateProgress( 2050 "[%s=>%s|%s] %s: %s" % ( 2051 darkgreen(from_branch), 2052 darkred(branch), 2053 brown(repo), 2054 blue(_("package flushed")), 2055 darkgreen(os.path.basename(package_path)), 2056 ), 2057 importance = 1, 2058 type = "info", 2059 header = brown(" ") 2060 ) 2061 2062 try: 2063 os.rmdir(tmp_down_dir) 2064 except OSError: 2065 pass 2066 2067 return True 2068

2069 - def move_packages(self, matches, to_repo, from_repo = None, ask = True, 2070 do_copy = False, new_tag = None, pull_deps = False):

2071 2072 if from_repo is None: 2073 from_repo = self.default_repository 2074 switched = set() 2075 2076 my_matches = list(matches) 2077 2078 # avoid setting __default__ as default server repo 2079 if etpConst['clientserverrepoid'] in (to_repo, from_repo): 2080 self.updateProgress( 2081 "%s: %s" % ( 2082 blue(_("Cannot touch system database")), 2083 red(etpConst['clientserverrepoid']), 2084 ), 2085 importance = 2, type = "warning", header = darkred(" @@ ") 2086 ) 2087 return switched 2088 2089 if not my_matches and from_repo: 2090 dbconn = self.open_server_repository(read_only = True, 2091 no_upload = True, repo = from_repo) 2092 my_matches = set( \ 2093 [(x, from_repo) for x in \ 2094 dbconn.listAllIdpackages()] 2095 ) 2096 2097 mytxt = _("Preparing to move selected packages to") 2098 if do_copy: 2099 mytxt = _("Preparing to copy selected packages to") 2100 self.updateProgress( 2101 "%s %s:" % ( 2102 blue(mytxt), 2103 red(to_repo), 2104 ), 2105 importance = 2, 2106 type = "info", 2107 header = red(" @@ ") 2108 ) 2109 self.updateProgress( 2110 "%s: %s" % ( 2111 bold(_("Note")), 2112 red(_("all old packages with conflicting scope will be " \ 2113 "removed from destination repo unless injected")), 2114 ), 2115 importance = 1, 2116 type = "info", 2117 header = red(" @@ ") 2118 ) 2119 2120 new_tag_string = '' 2121 if new_tag != None: 2122 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")), 2123 brown(new_tag),) 2124 2125 my_qa = self.QA() 2126 branch = self.SystemSettings['repositories']['branch'] 2127 pull_deps_matches = [] 2128 for idpackage, repo in my_matches: 2129 dbconn = self.open_server_repository(read_only = True, 2130 no_upload = True, repo = repo) 2131 self.updateProgress( 2132 "[%s=>%s|%s] %s " % ( 2133 darkgreen(repo), 2134 darkred(to_repo), 2135 brown(branch), 2136 blue(dbconn.retrieveAtom(idpackage)), 2137 ) + new_tag_string, 2138 importance = 0, 2139 type = "info", 2140 header = brown(" # ") 2141 ) 2142 # do we want to pull in also package dependencies? 2143 if pull_deps: 2144 dep_idpackages = my_qa.get_deep_dependency_list(dbconn, 2145 idpackage) 2146 for dep_idpackage in dep_idpackages: 2147 2148 my_dep_match = (dep_idpackage, repo,) 2149 if my_dep_match in pull_deps_matches: 2150 continue 2151 if my_dep_match in my_matches: 2152 continue 2153 2154 pull_deps_matches.append(my_dep_match) 2155 self.updateProgress( 2156 "[%s|%s] %s" % ( 2157 brown(branch), 2158 blue(_("dependency")), 2159 purple(dbconn.retrieveAtom(dep_idpackage)), 2160 ), 2161 importance = 0, 2162 type = "info", 2163 header = purple(" >> ") 2164 ) 2165 2166 if pull_deps: 2167 # put deps first! 2168 my_matches = pull_deps_matches + [x for x in my_matches if x not \ 2169 in pull_deps_matches] 2170 2171 if ask: 2172 rc_question = self.askQuestion(_("Would you like to continue ?")) 2173 if rc_question == _("No"): 2174 return switched 2175 2176 for idpackage, repo in my_matches: 2177 dbconn = self.open_server_repository(read_only = False, 2178 no_upload = True, repo = repo) 2179 match_branch = dbconn.retrieveBranch(idpackage) 2180 match_atom = dbconn.retrieveAtom(idpackage) 2181 package_filename = os.path.basename( 2182 dbconn.retrieveDownloadURL(idpackage)) 2183 self.updateProgress( 2184 "[%s=>%s|%s] %s: %s" % ( 2185 darkgreen(repo), 2186 darkred(to_repo), 2187 brown(branch), 2188 blue(_("switching")), 2189 darkgreen(match_atom), 2190 ), 2191 importance = 0, 2192 type = "info", 2193 header = red(" @@ "), 2194 back = True 2195 ) 2196 # move binary file 2197 from_file = os.path.join(self.get_local_packages_directory(repo), 2198 match_branch, package_filename) 2199 if not os.path.isfile(from_file): 2200 from_file = os.path.join(self.get_local_upload_directory(repo), 2201 match_branch, package_filename) 2202 if not os.path.isfile(from_file): 2203 self.updateProgress( 2204 "[%s=>%s|%s] %s: %s -> %s" % ( 2205 darkgreen(repo), 2206 darkred(to_repo), 2207 brown(branch), 2208 bold(_("cannot switch, package not found, skipping")), 2209 darkgreen(match_atom), 2210 red(from_file), 2211 ), 2212 importance = 1, 2213 type = "warning", 2214 header = darkred(" !!! ") 2215 ) 2216 continue 2217 2218 if new_tag != None: 2219 match_category = dbconn.retrieveCategory(idpackage) 2220 match_name = dbconn.retrieveName(idpackage) 2221 match_version = dbconn.retrieveVersion(idpackage) 2222 tagged_package_filename = \ 2223 self.entropyTools.create_package_filename( 2224 match_category, match_name, match_version, new_tag) 2225 to_file = os.path.join(self.get_local_upload_directory(to_repo), 2226 match_branch, tagged_package_filename) 2227 else: 2228 to_file = os.path.join(self.get_local_upload_directory(to_repo), 2229 match_branch, package_filename) 2230 if not os.path.isdir(os.path.dirname(to_file)): 2231 os.makedirs(os.path.dirname(to_file)) 2232 2233 copy_data = [ 2234 (from_file, to_file,), 2235 (from_file + etpConst['packagesmd5fileext'], 2236 to_file + etpConst['packagesmd5fileext'],), 2237 (from_file + etpConst['packagesexpirationfileext'], 2238 to_file + etpConst['packagesexpirationfileext'],) 2239 ] 2240 2241 for from_item, to_item in copy_data: 2242 self.updateProgress( 2243 "[%s=>%s|%s] %s: %s" % ( 2244 darkgreen(repo), 2245 darkred(to_repo), 2246 brown(branch), 2247 blue(_("moving file")), 2248 darkgreen(os.path.basename(from_item)), 2249 ), 2250 importance = 0, 2251 type = "info", 2252 header = red(" @@ "), 2253 back = True 2254 ) 2255 if os.path.isfile(from_item): 2256 shutil.copy2(from_item, to_item) 2257 2258 self.updateProgress( 2259 "[%s=>%s|%s] %s: %s" % ( 2260 darkgreen(repo), 2261 darkred(to_repo), 2262 brown(branch), 2263 blue(_("loading data from source database")), 2264 darkgreen(repo), 2265 ), 2266 importance = 0, 2267 type = "info", 2268 header = red(" @@ "), 2269 back = True 2270 ) 2271 # install package into destination db 2272 data = dbconn.getPackageData(idpackage) 2273 if new_tag != None: 2274 data['versiontag'] = new_tag 2275 2276 todbconn = self.open_server_repository(read_only = False, 2277 no_upload = True, repo = to_repo) 2278 2279 self.updateProgress( 2280 "[%s=>%s|%s] %s: %s" % ( 2281 darkgreen(repo), 2282 darkred(to_repo), 2283 brown(branch), 2284 blue(_("injecting data to destination database")), 2285 darkgreen(to_repo), 2286 ), 2287 importance = 0, 2288 type = "info", 2289 header = red(" @@ "), 2290 back = True 2291 ) 2292 new_idpackage, new_revision, new_data = todbconn.handlePackage(data) 2293 del data 2294 todbconn.commitChanges() 2295 2296 if not do_copy: 2297 self.updateProgress( 2298 "[%s=>%s|%s] %s: %s" % ( 2299 darkgreen(repo), 2300 darkred(to_repo), 2301 brown(branch), 2302 blue(_("removing entry from source database")), 2303 darkgreen(repo), 2304 ), 2305 importance = 0, 2306 type = "info", 2307 header = red(" @@ "), 2308 back = True 2309 ) 2310 2311 # remove package from old db 2312 dbconn.removePackage(idpackage) 2313 dbconn.commitChanges() 2314 2315 self.updateProgress( 2316 "[%s=>%s|%s] %s: %s" % ( 2317 darkgreen(repo), 2318 darkred(to_repo), 2319 brown(branch), 2320 blue(_("successfully handled atom")), 2321 darkgreen(match_atom), 2322 ), 2323 importance = 0, 2324 type = "info", 2325 header = blue(" @@ ") 2326 ) 2327 switched.add((idpackage, repo,)) 2328 2329 # just run this to make dev aware 2330 self.dependencies_test(to_repo) 2331 2332 return switched

2333 2334

2335 - def package_injector(self, package_file, inject = False, repo = None):

2336 2337 if repo is None: 2338 repo = self.default_repository 2339 2340 upload_dir = os.path.join(self.get_local_upload_directory(repo), 2341 self.SystemSettings['repositories']['branch']) 2342 if not os.path.isdir(upload_dir): 2343 os.makedirs(upload_dir) 2344 2345 dbconn = self.open_server_repository(read_only = False, 2346 no_upload = True, repo = repo) 2347 self.updateProgress( 2348 red("[repo: %s] %s: %s" % ( 2349 darkgreen(repo), 2350 _("adding package"), 2351 bold(os.path.basename(package_file)), 2352 ) 2353 ), 2354 importance = 1, 2355 type = "info", 2356 header = brown(" * "), 2357 back = True 2358 ) 2359 mydata = self.Spm().extract_package_metadata(package_file) 2360 mydata['injected'] = inject 2361 idpackage, revision, mydata = dbconn.handlePackage(mydata) 2362 2363 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2364 myserver_repos = list(srv_set['repositories'].keys()) 2365 2366 ### since we are handling more repositories, we need to make sure 2367 ### that there are no packages in other repositories with same atom 2368 ### and greater revision 2369 rev_test_atom = mydata['atom'] 2370 max_rev = -1 2371 for myrepo in myserver_repos: 2372 2373 # not myself 2374 if myrepo == repo: 2375 continue 2376 2377 mydbconn = self.open_server_repository(read_only = True, 2378 no_upload = True, repo = myrepo) 2379 2380 myrepo_idpackages = mydbconn.getIDPackages(rev_test_atom) 2381 for myrepo_idpackage in myrepo_idpackages: 2382 myrev = mydbconn.retrieveRevision(myrepo_idpackage) 2383 if myrev > max_rev: 2384 max_rev = myrev 2385 2386 if max_rev >= revision: 2387 max_rev += 1 2388 revision = max_rev 2389 mydata['revision'] = revision 2390 # update revision for pkg now 2391 dbconn.setRevision(idpackage, revision) 2392 2393 2394 # set trashed counters 2395 trashing_counters = set() 2396 2397 for myrepo in myserver_repos: 2398 mydbconn = self.open_server_repository(read_only = True, 2399 no_upload = True, repo = myrepo) 2400 mylist = mydbconn.retrieve_packages_to_remove( 2401 mydata['name'], 2402 mydata['category'], 2403 mydata['slot'], 2404 mydata['injected'] 2405 ) 2406 for myitem in mylist: 2407 trashing_counters.add(mydbconn.retrieveSpmUid(myitem)) 2408 2409 for mycounter in trashing_counters: 2410 dbconn.setTrashedUid(mycounter) 2411 2412 # add package info to our current server repository 2413 dbconn.dropInstalledPackageFromStore(idpackage) 2414 dbconn.storeInstalledPackage(idpackage, repo) 2415 atom = dbconn.retrieveAtom(idpackage) 2416 2417 self.updateProgress( 2418 "[repo:%s] %s: %s %s: %s" % ( 2419 darkgreen(repo), 2420 blue(_("added package")), 2421 darkgreen(atom), 2422 blue(_("rev")), # as in revision 2423 bold(str(revision)), 2424 ), 2425 importance = 1, 2426 type = "info", 2427 header = red(" @@ ") 2428 ) 2429 2430 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage)) 2431 if manual_deps: 2432 self.updateProgress( 2433 "[repo:%s] %s: %s" % ( 2434 darkgreen(repo), 2435 blue(_("manual dependencies for")), 2436 darkgreen(atom), 2437 ), 2438 importance = 1, 2439 type = "warning", 2440 header = darkgreen(" ## ") 2441 ) 2442 for m_dep in manual_deps: 2443 self.updateProgress( 2444 brown(m_dep), 2445 importance = 1, 2446 type = "warning", 2447 header = darkred(" # ") 2448 ) 2449 2450 download_url = self._setup_repository_package_filename(idpackage, 2451 repo = repo) 2452 downloadfile = os.path.basename(download_url) 2453 destination_path = os.path.join(upload_dir, downloadfile) 2454 try: 2455 os.rename(package_file, destination_path) 2456 except OSError: 2457 shutil.move(package_file, destination_path) 2458 2459 dbconn.commitChanges() 2460 return idpackage, destination_path

2461 2462 # this function changes the final repository package filename

2463 - def _setup_repository_package_filename(self, idpackage, repo = None):

2464 2465 dbconn = self.open_server_repository(read_only = False, 2466 no_upload = True, repo = repo) 2467 2468 downloadurl = dbconn.retrieveDownloadURL(idpackage) 2469 packagerev = dbconn.retrieveRevision(idpackage) 2470 downloaddir = os.path.dirname(downloadurl) 2471 downloadfile = os.path.basename(downloadurl) 2472 # add revision 2473 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev, 2474 etpConst['packagesext'],) 2475 downloadurl = os.path.join(downloaddir, downloadfile) 2476 2477 # update url 2478 dbconn.setDownloadURL(idpackage, downloadurl) 2479 2480 return downloadurl

2481

2482 - def add_packages_to_repository(self, packages_data, ask = True, 2483 repo = None):

2484 2485 if repo is None: 2486 repo = self.default_repository 2487 2488 mycount = 0 2489 maxcount = len(packages_data) 2490 idpackages_added = set() 2491 to_be_injected = set() 2492 my_qa = self.QA() 2493 missing_deps_taint = False 2494 for package_filepath, inject in packages_data: 2495 2496 mycount += 1 2497 self.updateProgress( 2498 "[repo:%s] %s: %s" % ( 2499 darkgreen(repo), 2500 blue(_("adding package")), 2501 darkgreen(os.path.basename(package_filepath)), 2502 ), 2503 importance = 1, 2504 type = "info", 2505 header = blue(" @@ "), 2506 count = (mycount, maxcount,) 2507 ) 2508 2509 try: 2510 # add to database 2511 idpackage, destination_path = self.package_injector( 2512 package_filepath, 2513 inject = inject, 2514 repo = repo 2515 ) 2516 idpackages_added.add(idpackage) 2517 to_be_injected.add((idpackage, destination_path)) 2518 except Exception as err: 2519 self.entropyTools.print_traceback() 2520 self.updateProgress( 2521 "[repo:%s] %s: %s" % ( 2522 darkgreen(repo), 2523 darkred(_("Exception caught, closing tasks")), 2524 darkgreen(str(err)), 2525 ), 2526 importance = 1, 2527 type = "error", 2528 header = bold(" !!! "), 2529 count = (mycount, maxcount,) 2530 ) 2531 # reinit depends table 2532 self.depends_table_initialize(repo) 2533 # reinit librarypathsidpackage table 2534 if idpackages_added: 2535 dbconn = self.open_server_repository(read_only = False, 2536 no_upload = True, repo = repo) 2537 missing_deps_taint = my_qa.test_missing_dependencies( 2538 idpackages_added, 2539 dbconn, 2540 ask = ask, 2541 repo = repo, 2542 self_check = True, 2543 black_list = \ 2544 self.get_missing_dependencies_blacklist( 2545 repo = repo), 2546 black_list_adder = \ 2547 self.add_missing_dependencies_blacklist_items 2548 ) 2549 my_qa.test_depends_linking(idpackages_added, dbconn, 2550 repo = repo) 2551 if to_be_injected: 2552 self.inject_database_into_packages(to_be_injected, 2553 repo = repo) 2554 # reinit depends table 2555 if missing_deps_taint: 2556 self.depends_table_initialize(repo) 2557 self.close_server_databases() 2558 raise 2559 2560 # reinit depends table 2561 self.depends_table_initialize(repo) 2562 2563 if idpackages_added: 2564 dbconn = self.open_server_repository(read_only = False, 2565 no_upload = True, repo = repo) 2566 missing_deps_taint = my_qa.test_missing_dependencies( 2567 idpackages_added, 2568 dbconn, 2569 ask = ask, 2570 repo = repo, 2571 self_check = True, 2572 black_list = \ 2573 self.get_missing_dependencies_blacklist(repo = repo), 2574 black_list_adder = \ 2575 self.add_missing_dependencies_blacklist_items 2576 ) 2577 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo) 2578 2579 # reinit depends table 2580 if missing_deps_taint: 2581 self.depends_table_initialize(repo) 2582 2583 # inject database into packages 2584 self.inject_database_into_packages(to_be_injected, repo = repo) 2585 2586 return idpackages_added

2587 2588

2589 - def inject_database_into_packages(self, injection_data, repo = None):

2590 2591 if repo is None: 2592 repo = self.default_repository 2593 2594 # now inject metadata into tbz2 packages 2595 self.updateProgress( 2596 "[repo:%s] %s:" % ( 2597 darkgreen(repo), 2598 blue(_("Injecting entropy metadata into built packages")), 2599 ), 2600 importance = 1, 2601 type = "info", 2602 header = red(" @@ ") 2603 ) 2604 2605 dbconn = self.open_server_repository(read_only = False, 2606 no_upload = True, repo = repo) 2607 for idpackage, package_path in injection_data: 2608 self.updateProgress( 2609 "[repo:%s|%s] %s: %s" % ( 2610 darkgreen(repo), 2611 brown(str(idpackage)), 2612 blue(_("injecting entropy metadata")), 2613 darkgreen(os.path.basename(package_path)), 2614 ), 2615 importance = 1, 2616 type = "info", 2617 header = blue(" @@ "), 2618 back = True 2619 ) 2620 data = dbconn.getPackageData(idpackage) 2621 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2622 dbpath = self.ClientService.inject_entropy_database_into_package( 2623 package_path, data, treeupdates_actions) 2624 digest = self.entropyTools.md5sum(package_path) 2625 # update digest 2626 dbconn.setDigest(idpackage, digest) 2627 # update signatures 2628 signatures = data['signatures'].copy() 2629 for hash_key in sorted(signatures): 2630 hash_func = getattr(self.entropyTools, hash_key) 2631 signatures[hash_key] = hash_func(package_path) 2632 dbconn.setSignatures(idpackage, signatures['sha1'], 2633 signatures['sha256'], signatures['sha512']) 2634 self.entropyTools.create_md5_file(package_path) 2635 const_setup_file(package_path, etpConst['entropygid'], 0o664) 2636 # remove garbage 2637 os.remove(dbpath) 2638 self.updateProgress( 2639 "[repo:%s|%s] %s: %s" % ( 2640 darkgreen(repo), 2641 brown(str(idpackage)), 2642 blue(_("injection complete")), 2643 darkgreen(os.path.basename(package_path)), 2644 ), 2645 importance = 1, 2646 type = "info", 2647 header = red(" @@ ") 2648 ) 2649 dbconn.commitChanges()

2650

2651 - def check_config_file_updates(self):

2652 self.updateProgress( 2653 "[%s] %s" % ( 2654 red(_("config files")), # something short please 2655 blue(_("checking system")), 2656 ), 2657 importance = 1, 2658 type = "info", 2659 header = blue(" @@ "), 2660 back = True 2661 ) 2662 # scanning for config files not updated 2663 scandata = self.ClientService.FileUpdates.scanfs(dcache = False) 2664 if scandata: 2665 self.updateProgress( 2666 "[%s] %s" % ( 2667 red(_("config files")), # something short please 2668 blue(_("there are configuration files not updated yet")), 2669 ), 2670 importance = 1, 2671 type = "error", 2672 header = darkred(" @@ ") 2673 ) 2674 for key in scandata: 2675 self.updateProgress( 2676 "%s" % (brown(etpConst['systemroot'] + \ 2677 scandata[key]['destination'])), 2678 importance = 1, 2679 type = "info", 2680 header = "\t" 2681 ) 2682 return True 2683 return False

2684

2685 - def remove_packages(self, idpackages, repo = None):

2686 2687 if repo is None: 2688 repo = self.default_repository 2689 2690 dbconn = self.open_server_repository(read_only = False, 2691 no_upload = True, repo = repo) 2692 for idpackage in idpackages: 2693 atom = dbconn.retrieveAtom(idpackage) 2694 self.updateProgress( 2695 "[repo:%s] %s: %s" % ( 2696 darkgreen(repo), 2697 blue(_("removing package")), 2698 darkgreen(atom), 2699 ), 2700 importance = 1, 2701 type = "info", 2702 header = brown(" @@ ") 2703 ) 2704 dbconn.removePackage(idpackage) 2705 self.close_server_database(dbconn) 2706 self.updateProgress( 2707 "[repo:%s] %s" % ( 2708 darkgreen(repo), 2709 blue(_("removal complete")), 2710 ), 2711 importance = 1, 2712 type = "info", 2713 header = brown(" @@ ") 2714 )

2715

2716 - def taint_database(self, repo = None):

2717 if repo is None: 2718 repo = self.default_repository 2719 2720 # taint the database status 2721 db_file = self.get_local_database_file(repo = repo) 2722 taint_file = self.get_local_database_taint_file(repo = repo) 2723 f = open(taint_file, "w") 2724 f.write("database tainted\n") 2725 f.flush() 2726 f.close() 2727 const_setup_file(taint_file, etpConst['entropygid'], 0o664) 2728 ServerRepositoryStatus().set_tainted(db_file)

2729

2730 - def bump_database(self, repo = None):

2731 dbconn = self.open_server_repository(read_only = False, 2732 no_upload = True, repo = repo) 2733 self.taint_database(repo = repo) 2734 self.close_server_database(dbconn)

2735

2736 - def get_remote_mirrors(self, repo = None):

2737 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2738 if repo is None: 2739 repo = self.default_repository 2740 return srv_set['repositories'][repo]['mirrors'][:]

2741

2742 - def get_remote_packages_relative_path(self, repo = None):

2743 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2744 if repo is None: 2745 repo = self.default_repository 2746 return srv_set['repositories'][repo]['packages_relative_path']

2747

2748 - def get_remote_database_relative_path(self, repo = None):

2749 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2750 if repo is None: 2751 repo = self.default_repository 2752 return srv_set['repositories'][repo]['database_relative_path']

2753

2754 - def get_local_database_file(self, repo = None, branch = None):

2755 if repo is None: 2756 repo = self.default_repository 2757 return os.path.join(self.get_local_database_dir(repo, branch), 2758 etpConst['etpdatabasefile'])

2759

2760 - def get_local_store_directory(self, repo = None):

2761 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2762 if repo is None: 2763 repo = self.default_repository 2764 return srv_set['repositories'][repo]['store_dir']

2765

2766 - def get_local_upload_directory(self, repo = None):

2767 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2768 if repo is None: 2769 repo = self.default_repository 2770 return srv_set['repositories'][repo]['upload_dir']

2771

2772 - def get_local_packages_directory(self, repo = None):

2773 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2774 if repo is None: 2775 repo = self.default_repository 2776 return srv_set['repositories'][repo]['packages_dir']

2777

2778 - def get_local_database_taint_file(self, repo = None, branch = None):

2779 if repo is None: 2780 repo = self.default_repository 2781 return os.path.join(self.get_local_database_dir(repo, branch), 2782 etpConst['etpdatabasetaintfile'])

2783

2784 - def get_local_database_revision_file(self, repo = None, branch = None):

2785 if repo is None: 2786 repo = self.default_repository 2787 return os.path.join(self.get_local_database_dir(repo, branch), 2788 etpConst['etpdatabaserevisionfile'])

2789

2790 - def get_local_database_timestamp_file(self, repo = None, branch = None):

2791 if repo is None: 2792 repo = self.default_repository 2793 return os.path.join(self.get_local_database_dir(repo, branch), 2794 etpConst['etpdatabasetimestampfile'])

2795

2796 - def get_local_database_ca_cert_file(self, repo = None, branch = None):

2797 if repo is None: 2798 repo = self.default_repository 2799 return os.path.join(self.get_local_database_dir(repo, branch), 2800 etpConst['etpdatabasecacertfile'])

2801

2802 - def get_local_database_server_cert_file(self, repo = None, branch = None):

2803 if repo is None: 2804 repo = self.default_repository 2805 return os.path.join(self.get_local_database_dir(repo, branch), 2806 etpConst['etpdatabaseservercertfile'])

2807

2808 - def get_local_database_mask_file(self, repo = None, branch = None):

2809 if repo is None: 2810 repo = self.default_repository 2811 return os.path.join(self.get_local_database_dir(repo, branch), 2812 etpConst['etpdatabasemaskfile'])

2813

2814 - def get_local_database_system_mask_file(self, repo = None, branch = None):

2815 if repo is None: 2816 repo = self.default_repository 2817 return os.path.join(self.get_local_database_dir(repo, branch), 2818 etpConst['etpdatabasesytemmaskfile'])

2819

2820 - def get_local_database_confl_tagged_file(self, repo = None, branch = None):

2821 if repo is None: 2822 repo = self.default_repository 2823 return os.path.join(self.get_local_database_dir(repo, branch), 2824 etpConst['etpdatabaseconflictingtaggedfile'])

2825

2826 - def get_local_database_licensewhitelist_file(self, repo = None, 2827 branch = None):

2828 2829 if repo is None: 2830 repo = self.default_repository 2831 return os.path.join(self.get_local_database_dir(repo, branch), 2832 etpConst['etpdatabaselicwhitelistfile'])

2833

2834 - def get_local_database_rss_file(self, repo = None, branch = None):

2835 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2836 if repo is None: 2837 repo = self.default_repository 2838 return os.path.join(self.get_local_database_dir(repo, branch), 2839 srv_set['rss']['name'])

2840

2841 - def get_local_database_rsslight_file(self, repo = None, branch = None):

2842 if repo is None: 2843 repo = self.default_repository 2844 return os.path.join(self.get_local_database_dir(repo, branch), 2845 etpConst['rss-light-name'])

2846

2847 - def get_local_database_notice_board_file(self, repo = None, branch = None):

2848 if repo is None: 2849 repo = self.default_repository 2850 return os.path.join(self.get_local_database_dir(repo, branch), 2851 etpConst['rss-notice-board'])

2852

2853 - def get_local_database_treeupdates_file(self, repo = None, branch = None):

2854 if repo is None: 2855 repo = self.default_repository 2856 return os.path.join(self.get_local_database_dir(repo, branch), 2857 etpConst['etpdatabaseupdatefile'])

2858

2859 - def get_local_database_compressed_metafiles_file(self, repo = None, 2860 branch = None):

2861 2862 if repo is None: 2863 repo = self.default_repository 2864 return os.path.join(self.get_local_database_dir(repo, branch), 2865 etpConst['etpdatabasemetafilesfile'])

2866

2867 - def get_local_database_metafiles_not_found_file(self, repo = None, 2868 branch = None):

2869 2870 if repo is None: 2871 repo = self.default_repository 2872 return os.path.join(self.get_local_database_dir(repo, branch), 2873 etpConst['etpdatabasemetafilesnotfound'])

2874

2875 - def get_local_exp_based_pkgs_rm_whitelist_file(self, repo = None, 2876 branch = None):

2877 if repo is None: 2878 repo = self.default_repository 2879 return os.path.join(self.get_local_database_dir(repo, branch), 2880 etpConst['etpdatabaseexpbasedpkgsrm'])

2881

2882 - def get_local_pkglist_file(self, repo = None, branch = None):

2883 if repo is None: 2884 repo = self.default_repository 2885 return os.path.join(self.get_local_database_dir(repo, branch), 2886 etpConst['etpdatabasepkglist'])

2887

2888 - def get_local_database_sets_dir(self, repo = None, branch = None):

2889 if repo is None: 2890 repo = self.default_repository 2891 return os.path.join(self.get_local_database_dir(repo, branch), 2892 etpConst['confsetsdirname'])

2893

2894 - def get_local_post_branch_mig_script(self, repo = None, branch = None):

2895 if repo is None: 2896 repo = self.default_repository 2897 return os.path.join(self.get_local_database_dir(repo, branch), 2898 etpConst['etp_post_branch_hop_script'])

2899

2900 - def get_local_post_branch_upg_script(self, repo = None, branch = None):

2901 if repo is None: 2902 repo = self.default_repository 2903 return os.path.join(self.get_local_database_dir(repo, branch), 2904 etpConst['etp_post_branch_upgrade_script'])

2905

2906 - def get_local_post_repo_update_script(self, repo = None, branch = None):

2907 if repo is None: 2908 repo = self.default_repository 2909 return os.path.join(self.get_local_database_dir(repo, branch), 2910 etpConst['etp_post_repo_update_script'])

2911

2912 - def get_local_critical_updates_file(self, repo = None, branch = None):

2913 if repo is None: 2914 repo = self.default_repository 2915 return os.path.join(self.get_local_database_dir(repo, branch), 2916 etpConst['etpdatabasecriticalfile'])

2917

2918 - def get_local_database_keywords_file(self, repo = None, branch = None):

2919 if repo is None: 2920 repo = self.default_repository 2921 return os.path.join(self.get_local_database_dir(repo, branch), 2922 etpConst['etpdatabasekeywordsfile'])

2923

2924 - def get_local_database_dir(self, repo = None, branch = None):

2925 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2926 if repo is None: 2927 repo = self.default_repository 2928 if branch is None: 2929 branch = self.SystemSettings['repositories']['branch'] 2930 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2931 branch)

2932

2933 - def get_missing_dependencies_blacklist_file(self, repo = None, 2934 branch = None):

2935 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2936 if repo is None: 2937 repo = self.default_repository 2938 if branch is None: 2939 branch = self.SystemSettings['repositories']['branch'] 2940 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2941 branch, etpConst['etpdatabasemissingdepsblfile'])

2942

2943 - def get_missing_dependencies_blacklist(self, repo = None, branch = None):

2944 if repo is None: 2945 repo = self.default_repository 2946 if branch is None: 2947 branch = self.SystemSettings['repositories']['branch'] 2948 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2949 wl_data = [] 2950 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK): 2951 f_wl = open(wl_file, "r") 2952 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \ 2953 not x.strip().startswith("#")] 2954 f_wl.close() 2955 return set(wl_data)

2956

2957 - def add_missing_dependencies_blacklist_items(self, items, repo = None, 2958 branch = None):

2959 2960 if repo is None: 2961 repo = self.default_repository 2962 if branch is None: 2963 branch = self.SystemSettings['repositories']['branch'] 2964 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2965 wl_dir = os.path.dirname(wl_file) 2966 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)): 2967 return 2968 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK): 2969 return 2970 f_wl = open(wl_file, "a+") 2971 f_wl.write('\n'.join(items)+'\n') 2972 f_wl.flush() 2973 f_wl.close()

2974

2975 - def get_local_database_revision(self, repo = None):

2976 2977 if repo is None: 2978 repo = self.default_repository 2979 2980 dbrev_file = self.get_local_database_revision_file(repo) 2981 if os.path.isfile(dbrev_file): 2982 f_rev = open(dbrev_file) 2983 rev = f_rev.readline().strip() 2984 f_rev.close() 2985 try: 2986 rev = int(rev) 2987 except ValueError: 2988 self.updateProgress( 2989 "[repo:%s] %s: %s - %s" % ( 2990 darkgreen(repo), 2991 blue(_("invalid database revision")), 2992 bold(rev), 2993 blue(_("defaulting to 0")), 2994 ), 2995 importance = 2, 2996 type = "error", 2997 header = darkred(" !!! ") 2998 ) 2999 rev = 0 3000 return rev 3001 else: 3002 return 0

3003

3004 - def get_remote_database_revision(self, repo = None):

3005 3006 if repo is None: 3007 repo = self.default_repository 3008 3009 remote_status = self.MirrorsService.get_remote_databases_status(repo) 3010 if not [x for x in remote_status if x[1]]: 3011 remote_revision = 0 3012 else: 3013 remote_revision = max([x[1] for x in remote_status]) 3014 3015 return remote_revision

3016

3017 - def get_branch_from_download_relative_uri(self, mypath):

3018 return self.ClientService.get_branch_from_download_relative_uri(mypath)

3019

3020 - def get_current_timestamp(self):

3021 from datetime import datetime 3022 import time 3023 return "%s" % (datetime.fromtimestamp(time.time()),)

3024

3025 - def create_repository_pkglist(self, repo = None, branch = None):

3026 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch) 3027 3028 tmp_pkglist_file = pkglist_file + ".tmp" 3029 dbconn = self.open_server_repository(repo = repo, just_reading = True, 3030 do_treeupdates = False) 3031 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True) 3032 3033 with open(tmp_pkglist_file, "w") as pkg_f: 3034 for pkg in pkglist: 3035 pkg_f.write(pkg + "\n") 3036 pkg_f.flush() 3037 3038 os.rename(tmp_pkglist_file, pkglist_file)

3039

3040 - def package_set_list(self, *args, **kwargs):

3041 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3042 repos = list(srv_set['repositories'].keys()) 3043 kwargs['server_repos'] = repos 3044 kwargs['serverInstance'] = self 3045 return self.ClientService.package_set_list(*args, **kwargs)

3046

3047 - def package_set_search(self, *args, **kwargs):

3048 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3049 repos = list(srv_set['repositories'].keys()) 3050 kwargs['server_repos'] = repos 3051 kwargs['serverInstance'] = self 3052 return self.ClientService.package_set_search(*args, **kwargs)

3053

3054 - def package_set_match(self, *args, **kwargs):

3055 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3056 repos = list(srv_set['repositories'].keys()) 3057 kwargs['server_repos'] = repos 3058 kwargs['serverInstance'] = self 3059 return self.ClientService.package_set_match(*args, **kwargs)

3060

3061 - def atom_match(self, *args, **kwargs):

3062 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3063 repos = list(srv_set['repositories'].keys()) 3064 kwargs['server_repos'] = repos 3065 kwargs['serverInstance'] = self 3066 return self.ClientService.atom_match(*args, **kwargs)

3067

3068 - def scan_package_changes(self):

3069 3070 spm = self.Spm() 3071 3072 spm_packages = spm.get_installed_packages() 3073 installed_packages = [] 3074 for spm_package in spm_packages: 3075 pkg_counter = spm.get_installed_package_metadata(spm_package, 3076 "COUNTER") 3077 try: 3078 pkg_counter = int(pkg_counter) 3079 except ValueError: 3080 continue 3081 installed_packages.append((spm_package, pkg_counter,)) 3082 3083 installed_counters = set() 3084 to_be_added = set() 3085 to_be_removed = set() 3086 to_be_injected = set() 3087 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3088 exp_based_scope = my_settings['exp_based_scope'] 3089 3090 server_repos = list(my_settings['repositories'].keys()) 3091 3092 # packages to be added 3093 for spm_atom, spm_counter in installed_packages: 3094 found = False 3095 for server_repo in server_repos: 3096 installed_counters.add(spm_counter) 3097 server_dbconn = self.open_server_repository(read_only = True, 3098 no_upload = True, repo = server_repo) 3099 counter = server_dbconn.isSpmUidAvailable(spm_counter) 3100 if counter: 3101 found = True 3102 break 3103 if not found: 3104 to_be_added.add((spm_atom, spm_counter,)) 3105 3106 # packages to be removed from the database 3107 database_counters = {} 3108 for server_repo in server_repos: 3109 server_dbconn = self.open_server_repository(read_only = True, 3110 no_upload = True, repo = server_repo) 3111 database_counters[server_repo] = server_dbconn.listAllSpmUids() 3112 3113 ordered_counters = set() 3114 for server_repo in database_counters: 3115 for data in database_counters[server_repo]: 3116 ordered_counters.add((data, server_repo)) 3117 database_counters = ordered_counters 3118 3119 for (counter, idpackage,), xrepo in database_counters: 3120 3121 if counter < 0: 3122 continue # skip packages without valid counter 3123 3124 if counter in installed_counters: 3125 continue 3126 3127 dbconn = self.open_server_repository(read_only = True, 3128 no_upload = True, repo = xrepo) 3129 3130 dorm = True 3131 # check if the package is in to_be_added 3132 if to_be_added: 3133 3134 dorm = False 3135 atom = dbconn.retrieveAtom(idpackage) 3136 atomkey = self.entropyTools.dep_getkey(atom) 3137 atomtag = self.entropyTools.dep_gettag(atom) 3138 atomslot = dbconn.retrieveSlot(idpackage) 3139 3140 add = True 3141 for spm_atom, spm_counter in to_be_added: 3142 addslot = self.Spm().get_installed_package_metadata( 3143 spm_atom, "SLOT") 3144 addkey = self.entropyTools.dep_getkey(spm_atom) 3145 # workaround for ebuilds not having slot 3146 if addslot is None: 3147 addslot = '0' 3148 # atomtag != None is for handling tagged pkgs correctly 3149 if (atomkey == addkey) and \ 3150 ((str(atomslot) == str(addslot)) or (atomtag != None)): 3151 # do not add to to_be_removed 3152 add = False 3153 break 3154 3155 if not add: 3156 continue 3157 dorm = True 3158 3159 # checking if we are allowed to remove stuff on this repo 3160 # it xrepo is not the default one, we MUST skip this to 3161 # avoid touching what developer doesn't expect 3162 if dorm and (xrepo == self.default_repository): 3163 trashed = self.is_counter_trashed(counter) 3164 if trashed: 3165 # search into portage then 3166 try: 3167 key, slot = dbconn.retrieveKeySlot(idpackage) 3168 trashed = self.Spm().match_installed_package( 3169 key+":"+slot) 3170 except TypeError: # referred to retrieveKeySlot 3171 trashed = True 3172 if not trashed: 3173 3174 dbtag = dbconn.retrieveVersionTag(idpackage) 3175 if dbtag: 3176 is_injected = dbconn.isInjected(idpackage) 3177 if not is_injected: 3178 to_be_injected.add((idpackage, xrepo)) 3179 3180 elif exp_based_scope: 3181 3182 # check if support for this is set 3183 plg_id = self.sys_settings_fatscope_plugin_id 3184 exp_data = self.SystemSettings[plg_id]['repos'].get( 3185 xrepo, set()) 3186 3187 # only some packages are set, check if our is 3188 # in the list 3189 if (idpackage not in exp_data) and (-1 not in exp_data): 3190 to_be_removed.add((idpackage, xrepo)) 3191 continue 3192 3193 idpackage_expired = self.is_match_expired((idpackage, 3194 xrepo,)) 3195 3196 if idpackage_expired: 3197 # expired !!! 3198 # add this and its depends (reverse deps) 3199 to_be_removed.add((idpackage, xrepo)) 3200 for my_id in dbconn.retrieveReverseDependencies( 3201 idpackage): 3202 to_be_removed.add((my_id, xrepo)) 3203 3204 else: 3205 to_be_removed.add((idpackage, xrepo)) 3206 3207 return to_be_added, to_be_removed, to_be_injected

3208

3209 - def is_match_expired(self, match):

3210 3211 idpackage, repoid = match 3212 dbconn = self.open_server_repository(repo = repoid, just_reading = True) 3213 # 3600 * 24 = 86400 3214 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3215 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400 3216 cur_unix_time = self.entropyTools.get_current_unix_time() 3217 # if packages removal is triggered by expiration 3218 # we will have to check if our package is really 3219 # expired and remove its reverse deps too 3220 mydate = dbconn.retrieveCreationDate(idpackage) 3221 # cross fingers hoping that time is set correctly 3222 mydelta = cur_unix_time - float(mydate) 3223 if mydelta > pkg_exp_secs: 3224 return True 3225 return False

3226

3227 - def is_counter_trashed(self, counter):

3228 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3229 server_repos = list(srv_set['repositories'].keys()) 3230 for repo in server_repos: 3231 dbconn = self.open_server_repository(read_only = True, 3232 no_upload = True, repo = repo) 3233 if dbconn.isSpmUidTrashed(counter): 3234 return True 3235 return False

3236

3237 - def transform_package_into_injected(self, idpackage, repo = None):

3238 dbconn = self.open_server_repository(read_only = False, 3239 no_upload = True, repo = repo) 3240 counter = dbconn.getNewNegativeSpmUid() 3241 dbconn.setSpmUid(idpackage, counter) 3242 dbconn.setInjected(idpackage)

3243

3244 - def initialize_server_database(self, empty = True, repo = None, 3245 warnings = True):

3246 3247 if repo is None: 3248 repo = self.default_repository 3249 3250 self.close_server_databases() 3251 revisions_match = {} 3252 treeupdates_actions = [] 3253 injected_packages = set() 3254 idpackages = set() 3255 idpackages_added = set() 3256 3257 mytxt = red("%s ...") % (_("Initializing Entropy database"),) 3258 self.updateProgress( 3259 mytxt, importance = 1, 3260 type = "info", header = darkgreen(" * "), 3261 back = True 3262 ) 3263 3264 if os.path.isfile(self.get_local_database_file(repo)): 3265 3266 dbconn = self.open_server_repository(read_only = True, 3267 no_upload = True, repo = repo, warnings = warnings) 3268 3269 try: 3270 dbconn.validateDatabase() 3271 idpackages = dbconn.listAllIdpackages() 3272 except SystemDatabaseError: 3273 pass 3274 3275 try: 3276 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 3277 except dbconn.dbapi2.Error: 3278 pass 3279 3280 # save list of injected packages 3281 try: 3282 injected_packages = dbconn.listAllInjectedPackages( 3283 just_files = True) 3284 injected_packages = set([os.path.basename(x) for x \ 3285 in injected_packages]) 3286 except dbconn.dbapi2.Error: 3287 pass 3288 3289 for idpackage in idpackages: 3290 url = dbconn.retrieveDownloadURL(idpackage) 3291 package = os.path.basename(url) 3292 branch = dbconn.retrieveBranch(idpackage) 3293 revision = dbconn.retrieveRevision(idpackage) 3294 revisions_match[package] = (branch, revision,) 3295 3296 self.close_server_database(dbconn) 3297 3298 mytxt = "%s: %s: %s" % ( 3299 bold(_("WARNING")), 3300 red(_("database already exists")), 3301 self.get_local_database_file(repo), 3302 ) 3303 self.updateProgress( 3304 mytxt, 3305 importance = 1, 3306 type = "warning", 3307 header = darkred(" !!! ") 3308 ) 3309 3310 rc_question = self.askQuestion(_("Do you want to continue ?")) 3311 if rc_question == _("No"): 3312 return 3313 try: 3314 os.remove(self.get_local_database_file(repo)) 3315 except OSError: 3316 pass 3317 3318 3319 # initialize 3320 dbconn = self.open_server_repository(read_only = False, 3321 no_upload = True, repo = repo, is_new = True) 3322 dbconn.initializeDatabase() 3323 3324 if not empty: 3325 3326 revisions_file = "/entropy-revisions-dump.txt" 3327 # dump revisions - as a backup 3328 if revisions_match: 3329 self.updateProgress( 3330 "%s: %s" % ( 3331 red(_("Dumping current revisions to file")), 3332 darkgreen(revisions_file), 3333 ), 3334 importance = 1, 3335 type = "info", 3336 header = darkgreen(" * ") 3337 ) 3338 f_rev = open(revisions_file, "w") 3339 f_rev.write(str(revisions_match)) 3340 f_rev.flush() 3341 f_rev.close() 3342 3343 # dump treeupdates - as a backup 3344 treeupdates_file = "/entropy-treeupdates-dump.txt" 3345 if treeupdates_actions: 3346 self.updateProgress( 3347 "%s: %s" % ( 3348 # do not translate treeupdates 3349 red(_("Dumping current 'treeupdates' actions to file")), 3350 bold(treeupdates_file), 3351 ), 3352 importance = 1, 3353 type = "info", 3354 header = darkgreen(" * ") 3355 ) 3356 f_tree = open(treeupdates_file, "w") 3357 f_tree.write(str(treeupdates_actions)) 3358 f_tree.flush() 3359 f_tree.close() 3360 3361 rc_question = self.askQuestion( 3362 _("Would you like to sync packages first (important!) ?")) 3363 if rc_question == _("Yes"): 3364 self.MirrorsService.sync_packages(repo = repo) 3365 3366 # fill tree updates actions 3367 if treeupdates_actions: 3368 dbconn.bumpTreeUpdatesActions(treeupdates_actions) 3369 3370 # now fill the database 3371 pkg_branch_dir = os.path.join( 3372 self.get_local_packages_directory(repo), 3373 self.SystemSettings['repositories']['branch']) 3374 pkglist = os.listdir(pkg_branch_dir) 3375 # filter .md5 and .expired packages 3376 pkg_ext_len = len(etpConst['packagesext']) 3377 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \ 3378 etpConst['packagesext']) and not \ 3379 os.path.isfile(os.path.join(pkg_branch_dir, 3380 x + etpConst['packagesexpirationfileext']))] 3381 3382 if pkglist: 3383 self.updateProgress( 3384 "%s '%s' %s %s" % ( 3385 red(_("Reinitializing Entropy database for branch")), 3386 bold(self.SystemSettings['repositories']['branch']), 3387 red(_("using Packages in the repository")), 3388 red("..."), 3389 ), 3390 importance = 1, 3391 type = "info", 3392 header = darkgreen(" * ") 3393 ) 3394 3395 counter = 0 3396 maxcount = len(pkglist) 3397 branch = self.SystemSettings['repositories']['branch'] 3398 for pkg in pkglist: 3399 counter += 1 3400 3401 self.updateProgress( 3402 "[repo:%s|%s] %s: %s" % ( 3403 darkgreen(repo), 3404 brown(branch), 3405 blue(_("analyzing")), 3406 bold(pkg), 3407 ), 3408 importance = 1, 3409 type = "info", 3410 header = " ", 3411 back = True, 3412 count = (counter, maxcount,) 3413 ) 3414 3415 doinject = False 3416 if pkg in injected_packages: 3417 doinject = True 3418 3419 pkg_path = os.path.join(self.get_local_packages_directory(repo), 3420 branch, pkg) 3421 mydata = self.Spm().extract_package_metadata(pkg_path) 3422 mydata['injected'] = doinject 3423 3424 # get previous revision 3425 revision_avail = revisions_match.get(pkg) 3426 add_revision = 0 3427 if (revision_avail != None): 3428 if branch == revision_avail[0]: 3429 add_revision = revision_avail[1] 3430 3431 idpackage, revision, mydata_upd = dbconn.addPackage(mydata, 3432 revision = add_revision) 3433 idpackages_added.add(idpackage) 3434 3435 self.updateProgress( 3436 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % ( 3437 repo, 3438 brown(branch), 3439 darkgreen(str(counter)), 3440 blue(str(maxcount)), 3441 red(_("added package")), 3442 darkgreen(pkg), 3443 red(_("revision")), 3444 brown(str(revision)), 3445 ), 3446 importance = 1, 3447 type = "info", 3448 header = " ", 3449 back = True 3450 ) 3451 3452 self.depends_table_initialize(repo) 3453 3454 my_qa = self.QA() 3455 3456 if idpackages_added: 3457 dbconn = self.open_server_repository(read_only = False, 3458 no_upload = True, repo = repo) 3459 my_qa.test_missing_dependencies( 3460 idpackages_added, dbconn, ask = True, 3461 repo = repo, self_check = True, 3462 black_list = \ 3463 self.get_missing_dependencies_blacklist(repo = repo), 3464 black_list_adder = \ 3465 self.add_missing_dependencies_blacklist_items 3466 ) 3467 3468 dbconn.commitChanges() 3469 self.close_server_databases() 3470 3471 return 0

3472

3473 - def match_packages(self, packages, repo = None):

3474 3475 dbconn = self.open_server_repository(read_only = True, 3476 no_upload = True, repo = repo) 3477 if ("world" in packages) or not packages: 3478 return dbconn.listAllIdpackages(), True 3479 else: 3480 idpackages = set() 3481 for package in packages: 3482 matches = dbconn.atomMatch(package, multiMatch = True) 3483 if matches[1] == 0: 3484 idpackages |= matches[0] 3485 else: 3486 mytxt = "%s: %s: %s" % ( 3487 red(_("Attention")), 3488 blue(_("cannot match")), 3489 bold(package), 3490 ) 3491 self.updateProgress( 3492 mytxt, 3493 importance = 1, 3494 type = "warning", 3495 header = darkred(" !!! ") 3496 ) 3497 return idpackages, False

3498

3499 - def verify_remote_packages(self, packages, ask = True, repo = None):

3500 3501 if repo is None: 3502 repo = self.default_repository 3503 3504 self.updateProgress( 3505 "[%s] %s:" % ( 3506 red("remote"), 3507 blue(_("Integrity verification of the selected packages")), 3508 ), 3509 importance = 1, 3510 type = "info", 3511 header = blue(" @@ ") 3512 ) 3513 3514 idpackages, world = self.match_packages(packages) 3515 dbconn = self.open_server_repository(read_only = True, no_upload = True, 3516 repo = repo) 3517 branch = self.SystemSettings['repositories']['branch'] 3518 3519 if world: 3520 self.updateProgress( 3521 blue( 3522 _("All the packages in repository will be checked.")), 3523 importance = 1, 3524 type = "info", 3525 header = " " 3526 ) 3527 else: 3528 mytxt = red("%s:") % ( 3529 _("This is the list of the packages that would be checked"),) 3530 self.updateProgress( 3531 mytxt, 3532 importance = 1, 3533 type = "info", 3534 header = " " 3535 ) 3536 for idpackage in idpackages: 3537 pkgatom = dbconn.retrieveAtom(idpackage) 3538 down_url = dbconn.retrieveDownloadURL(idpackage) 3539 pkgfile = os.path.basename(down_url) 3540 self.updateProgress( 3541 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)), 3542 importance = 1, 3543 type = "info", 3544 header = darkgreen(" - ") 3545 ) 3546 3547 if ask: 3548 rc_question = self.askQuestion( 3549 _("Would you like to continue ?")) 3550 if rc_question == _("No"): 3551 return set(), set(), {} 3552 3553 match = set() 3554 not_match = set() 3555 broken_packages = {} 3556 3557 for uri in self.get_remote_mirrors(repo): 3558 3559 crippled_uri = EntropyTransceiver.get_uri_name(uri) 3560 self.updateProgress( 3561 "[repo:%s] %s: %s" % ( 3562 darkgreen(repo), 3563 blue(_("Working on mirror")), 3564 brown(crippled_uri), 3565 ), 3566 importance = 1, 3567 type = "info", 3568 header = red(" @@ ") 3569 ) 3570 3571 3572 totalcounter = len(idpackages) 3573 currentcounter = 0 3574 3575 txc = self.Transceiver(uri) 3576 txc.set_verbosity(False) 3577 with txc as handler: 3578 3579 for idpackage in idpackages: 3580 3581 currentcounter += 1 3582 pkgfile = dbconn.retrieveDownloadURL(idpackage) 3583 pkghash = dbconn.retrieveDigest(idpackage) 3584 3585 self.updateProgress( 3586 "[%s] %s: %s" % ( 3587 brown(crippled_uri), 3588 blue(_("checking hash")), 3589 darkgreen(pkgfile), 3590 ), 3591 importance = 1, 3592 type = "info", 3593 header = blue(" @@ "), 3594 back = True, 3595 count = (currentcounter, totalcounter,) 3596 ) 3597 3598 pkg_rel_path = '/'.join(pkgfile.split("/")[2:]) 3599 3600 pkgfile = os.path.join( 3601 self.get_remote_packages_relative_path(repo), 3602 pkg_rel_path) 3603 3604 ck_remote = handler.get_md5(pkgfile) 3605 if ck_remote is None: 3606 self.updateProgress( 3607 "[%s] %s: %s %s" % ( 3608 brown(crippled_uri), 3609 blue(_("digest verification of")), 3610 bold(pkgfile), 3611 blue(_("not supported")), 3612 ), 3613 importance = 1, 3614 type = "info", 3615 header = blue(" @@ "), 3616 count = (currentcounter, totalcounter,) 3617 ) 3618 continue 3619 3620 if ck_remote == pkghash: 3621 match.add(idpackage) 3622 else: 3623 not_match.add(idpackage) 3624 self.updateProgress( 3625 "[%s] %s: %s %s" % ( 3626 brown(crippled_uri), 3627 blue(_("package")), 3628 bold(pkgfile), 3629 red(_("NOT healthy")), 3630 ), 3631 importance = 1, 3632 type = "warning", 3633 header = darkred(" !!! "), 3634 count = (currentcounter, totalcounter,) 3635 ) 3636 if crippled_uri not in broken_packages: 3637 broken_packages[crippled_uri] = [] 3638 broken_packages[crippled_uri].append(pkgfile) 3639 3640 if broken_packages: 3641 mytxt = blue("%s:") % ( 3642 _("This is the list of broken packages"),) 3643 self.updateProgress( 3644 mytxt, 3645 importance = 1, 3646 type = "info", 3647 header = red(" * ") 3648 ) 3649 for mirror in list(broken_packages.keys()): 3650 mytxt = "%s: %s" % ( 3651 brown(_("Mirror")), 3652 bold(mirror), 3653 ) 3654 self.updateProgress( 3655 mytxt, 3656 importance = 1, 3657 type = "info", 3658 header = red(" <> ") 3659 ) 3660 for broken_package in broken_packages[mirror]: 3661 self.updateProgress( 3662 blue(broken_package), 3663 importance = 1, 3664 type = "info", 3665 header = red(" - ") 3666 ) 3667 3668 self.updateProgress( 3669 "%s:" % ( 3670 blue(_("Statistics")), 3671 ), 3672 importance = 1, 3673 type = "info", 3674 header = red(" @@ ") 3675 ) 3676 self.updateProgress( 3677 "[%s] %s:\t%s" % ( 3678 red(crippled_uri), 3679 brown(_("Number of checked packages")), 3680 brown(str(len(match) + len(not_match))), 3681 ), 3682 importance = 1, 3683 type = "info", 3684 header = brown(" # ") 3685 ) 3686 self.updateProgress( 3687 "[%s] %s:\t%s" % ( 3688 red(crippled_uri), 3689 darkgreen(_("Number of healthy packages")), 3690 darkgreen(str(len(match))), 3691 ), 3692 importance = 1, 3693 type = "info", 3694 header = brown(" # ") 3695 ) 3696 self.updateProgress( 3697 "[%s] %s:\t%s" % ( 3698 red(crippled_uri), 3699 darkred(_("Number of broken packages")), 3700 darkred(str(len(not_match))), 3701 ), 3702 importance = 1, 3703 type = "info", 3704 header = brown(" # ") 3705 ) 3706 3707 return match, not_match, broken_packages

3708 3709

3710 - def verify_local_packages(self, packages, ask = True, repo = None):

3711 3712 if repo is None: 3713 repo = self.default_repository 3714 3715 self.updateProgress( 3716 "[%s] %s:" % ( 3717 red(_("local")), 3718 blue(_("Integrity verification of the selected packages")), 3719 ), 3720 importance = 1, 3721 type = "info", 3722 header = darkgreen(" * ") 3723 ) 3724 3725 idpackages, world = self.match_packages(packages) 3726 dbconn = self.open_server_repository(read_only = True, 3727 no_upload = True, repo = repo) 3728 3729 if world: 3730 self.updateProgress( 3731 blue(_("All the packages in repository will be checked.")), 3732 importance = 1, 3733 type = "info", 3734 header = " " 3735 ) 3736 3737 to_download = set() 3738 available = set() 3739 for idpackage in idpackages: 3740 3741 pkgatom = dbconn.retrieveAtom(idpackage) 3742 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3743 pkg_rel_path = '/'.join(pkg_path.split("/")[2:]) 3744 3745 bindir_path = os.path.join(self.get_local_packages_directory(repo), 3746 pkg_rel_path) 3747 uploaddir_path = os.path.join(self.get_local_upload_directory(repo), 3748 pkg_rel_path) 3749 3750 if os.path.isfile(bindir_path): 3751 if not world: 3752 self.updateProgress( 3753 "[%s] %s :: %s" % ( 3754 darkgreen(_("available")), 3755 blue(pkgatom), 3756 darkgreen(pkg_rel_path), 3757 ), 3758 importance = 0, 3759 type = "info", 3760 header = darkgreen(" # ") 3761 ) 3762 available.add(idpackage) 3763 elif os.path.isfile(uploaddir_path): 3764 if not world: 3765 self.updateProgress( 3766 "[%s] %s :: %s" % ( 3767 darkred(_("upload/ignored")), 3768 blue(pkgatom), 3769 darkgreen(pkg_rel_path), 3770 ), 3771 importance = 0, 3772 type = "info", 3773 header = darkgreen(" # ") 3774 ) 3775 else: 3776 self.updateProgress( 3777 "[%s] %s :: %s" % ( 3778 brown(_("download")), 3779 blue(pkgatom), 3780 darkgreen(pkg_rel_path), 3781 ), 3782 importance = 0, 3783 type = "info", 3784 header = darkgreen(" # ") 3785 ) 3786 to_download.add((idpackage, pkg_path,)) 3787 3788 if ask: 3789 rc_question = self.askQuestion(_("Would you like to continue ?")) 3790 if rc_question == _("No"): 3791 return set(), set(), set(), set() 3792 3793 3794 fine = set() 3795 failed = set() 3796 downloaded_fine = set() 3797 downloaded_errors = set() 3798 3799 if to_download: 3800 3801 not_downloaded = set() 3802 mytxt = blue("%s ...") % (_("Starting to download missing files"),) 3803 self.updateProgress( 3804 mytxt, 3805 importance = 1, 3806 type = "info", 3807 header = " " 3808 ) 3809 for uri in self.get_remote_mirrors(repo): 3810 3811 if not_downloaded: 3812 mytxt = blue("%s ...") % ( 3813 _("Searching missing/broken files on another mirror"),) 3814 self.updateProgress( 3815 mytxt, 3816 importance = 1, 3817 type = "info", 3818 header = " " 3819 ) 3820 to_download = not_downloaded.copy() 3821 not_downloaded = set() 3822 3823 for idpackage, pkg_path in to_download: 3824 rc_down = self.MirrorsService.download_package(uri, 3825 pkg_path, repo = repo) 3826 if rc_down: 3827 downloaded_fine.add(idpackage) 3828 available.add(idpackage) 3829 else: 3830 not_downloaded.add(pkg_path) 3831 3832 if not not_downloaded: 3833 self.updateProgress( 3834 red(_("Binary packages downloaded successfully.")), 3835 importance = 1, 3836 type = "info", 3837 header = " " 3838 ) 3839 break 3840 3841 if not_downloaded: 3842 mytxt = blue("%s:") % ( 3843 _("These are the packages that cannot be found online"),) 3844 self.updateProgress( 3845 mytxt, 3846 importance = 1, 3847 type = "info", 3848 header = " " 3849 ) 3850 for pkg_path in not_downloaded: 3851 downloaded_errors.add(pkg_path) 3852 self.updateProgress( 3853 brown(pkg_path), 3854 importance = 1, 3855 type = "warning", 3856 header = red(" * ") 3857 ) 3858 downloaded_errors |= not_downloaded 3859 mytxt = "%s." % (_("They won't be checked"),) 3860 self.updateProgress( 3861 mytxt, 3862 importance = 1, 3863 type = "warning", 3864 header = " " 3865 ) 3866 3867 my_qa = self.QA() 3868 3869 totalcounter = str(len(available)) 3870 currentcounter = 0 3871 for idpackage in available: 3872 currentcounter += 1 3873 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3874 orig_branch = self.get_branch_from_download_relative_uri(pkg_path) 3875 pkgfile = os.path.basename(pkg_path) 3876 3877 self.updateProgress( 3878 "[branch:%s] %s %s" % ( 3879 brown(orig_branch), 3880 blue(_("checking status of")), 3881 darkgreen(pkgfile), 3882 ), 3883 importance = 1, 3884 type = "info", 3885 header = " ", 3886 back = True, 3887 count = (currentcounter, totalcounter,) 3888 ) 3889 3890 storedmd5 = dbconn.retrieveDigest(idpackage) 3891 pkgpath = os.path.join(self.get_local_packages_directory(repo), 3892 orig_branch, pkgfile) 3893 result = self.entropyTools.compare_md5(pkgpath, storedmd5) 3894 qa_fine = my_qa.entropy_package_checks(pkgpath) 3895 if result and qa_fine: 3896 fine.add(idpackage) 3897 else: 3898 failed.add(idpackage) 3899 self.updateProgress( 3900 "[branch:%s] %s %s %s: %s" % ( 3901 brown(orig_branch), 3902 blue(_("package")), 3903 darkgreen(pkg_path), 3904 blue(_("is corrupted, stored checksum")), 3905 brown(storedmd5), 3906 ), 3907 importance = 1, 3908 type = "info", 3909 header = " ", 3910 count = (currentcounter, totalcounter,) 3911 ) 3912 3913 if failed: 3914 mytxt = blue("%s:") % (_("This is the list of broken packages"),) 3915 self.updateProgress( 3916 mytxt, 3917 importance = 1, 3918 type = "warning", 3919 header = darkred(" # ") 3920 ) 3921 for idpackage in failed: 3922 atom = dbconn.retrieveAtom(idpackage) 3923 down_p = dbconn.retrieveDownloadURL(idpackage) 3924 self.updateProgress( 3925 blue("[atom:%s] %s" % (atom, down_p,)), 3926 importance = 0, 3927 type = "warning", 3928 header = brown(" # ") 3929 ) 3930 3931 # print stats 3932 self.updateProgress( 3933 red("Statistics:"), 3934 importance = 1, 3935 type = "info", 3936 header = blue(" * ") 3937 ) 3938 self.updateProgress( 3939 brown("%s => %s" % ( 3940 len(fine) + len(failed), 3941 _("checked packages"), 3942 ) 3943 ), 3944 importance = 0, 3945 type = "info", 3946 header = brown(" # ") 3947 ) 3948 self.updateProgress( 3949 darkgreen("%s => %s" % ( 3950 len(fine), 3951 _("healthy packages"), 3952 ) 3953 ), 3954 importance = 0, 3955 type = "info", 3956 header = brown(" # ") 3957 ) 3958 self.updateProgress( 3959 darkred("%s => %s" % ( 3960 len(failed), 3961 _("broken packages"), 3962 ) 3963 ), 3964 importance = 0, 3965 type = "info", 3966 header = brown(" # ") 3967 ) 3968 self.updateProgress( 3969 blue("%s => %s" % ( 3970 len(downloaded_fine), 3971 _("downloaded packages"), 3972 ) 3973 ), 3974 importance = 0, 3975 type = "info", 3976 header = brown(" # ") 3977 ) 3978 self.updateProgress( 3979 bold("%s => %s" % ( 3980 len(downloaded_errors), 3981 _("failed downloads"), 3982 ) 3983 ), 3984 importance = 0, 3985 type = "info", 3986 header = brown(" # ") 3987 ) 3988 3989 self.close_server_database(dbconn) 3990 return fine, failed, downloaded_fine, downloaded_errors

3991 3992

3993 - def switch_packages_branch(self, from_branch, to_branch, repo = None):

3994 3995 if repo is None: 3996 repo = self.default_repository 3997 3998 if to_branch != self.SystemSettings['repositories']['branch']: 3999 mytxt = "%s: %s %s" % ( 4000 blue(_("Please setup your branch to")), 4001 bold(to_branch), 4002 blue(_("and retry")), 4003 ) 4004 self.updateProgress( 4005 mytxt, 4006 importance = 1, 4007 type = "error", 4008 header = darkred(" !! ") 4009 ) 4010 return None 4011 4012 mytxt = red("%s ...") % (_("Copying database (if not exists)"),) 4013 self.updateProgress( 4014 mytxt, 4015 importance = 1, 4016 type = "info", 4017 header = darkgreen(" @@ ") 4018 ) 4019 branch_dbdir = self.get_local_database_dir(repo) 4020 old_branch_dbdir = self.get_local_database_dir(repo, from_branch) 4021 4022 # close all our databases 4023 self.close_server_databases() 4024 4025 # if database file did not exist got created as an empty file 4026 # we can just rm -rf it 4027 branch_dbfile = self.get_local_database_file(repo) 4028 if os.path.isfile(branch_dbfile): 4029 if self.entropyTools.get_file_size(branch_dbfile) == 0: 4030 shutil.rmtree(branch_dbdir, True) 4031 4032 if os.path.isdir(branch_dbdir): 4033 4034 while True: 4035 rnd_num = self.entropyTools.get_random_number() 4036 backup_dbdir = branch_dbdir + str(rnd_num) 4037 if not os.path.isdir(backup_dbdir): 4038 break 4039 os.rename(branch_dbdir, backup_dbdir) 4040 4041 if os.path.isdir(old_branch_dbdir): 4042 shutil.copytree(old_branch_dbdir, branch_dbdir) 4043 4044 mytxt = red("%s ...") % (_("Switching packages"),) 4045 self.updateProgress( 4046 mytxt, 4047 importance = 1, 4048 type = "info", 4049 header = darkgreen(" @@ ") 4050 ) 4051 4052 dbconn = self.open_server_repository(read_only = False, 4053 no_upload = True, repo = repo, lock_remote = False) 4054 try: 4055 dbconn.validateDatabase() 4056 except SystemDatabaseError: 4057 self.handle_uninitialized_repository(repo) 4058 dbconn = self.open_server_repository(read_only = False, 4059 no_upload = True, repo = repo, lock_remote = False) 4060 4061 idpackages = dbconn.listAllIdpackages() 4062 already_switched = set() 4063 not_found = set() 4064 switched = set() 4065 ignored = set() 4066 no_checksum = set() 4067 4068 maxcount = len(idpackages) 4069 count = 0 4070 for idpackage in idpackages: 4071 count += 1 4072 4073 cur_branch = dbconn.retrieveBranch(idpackage) 4074 atom = dbconn.retrieveAtom(idpackage) 4075 if cur_branch == to_branch: 4076 already_switched.add(idpackage) 4077 self.updateProgress( 4078 red("%s %s, %s %s" % ( 4079 _("Ignoring"), 4080 bold(atom), 4081 _("already in branch"), 4082 cur_branch, 4083 ) 4084 ), 4085 importance = 0, 4086 type = "info", 4087 header = darkgreen(" @@ "), 4088 count = (count, maxcount,) 4089 ) 4090 ignored.add(idpackage) 4091 continue 4092 4093 self.updateProgress( 4094 "[%s=>%s] %s" % ( 4095 brown(cur_branch), 4096 bold(to_branch), 4097 darkgreen(atom), 4098 ), 4099 importance = 0, 4100 type = "info", 4101 header = darkgreen(" @@ "), 4102 back = True, 4103 count = (count, maxcount,) 4104 ) 4105 dbconn.switchBranch(idpackage, to_branch) 4106 dbconn.commitChanges() 4107 switched.add(idpackage) 4108 4109 dbconn.commitChanges() 4110 4111 # now migrate counters 4112 dbconn.moveSpmUidsToBranch(to_branch, from_branch = from_branch) 4113 4114 self.close_server_database(dbconn) 4115 mytxt = blue("%s.") % (_("migration loop completed"),) 4116 self.updateProgress( 4117 "[%s=>%s] %s" % ( 4118 brown(from_branch), 4119 bold(to_branch), 4120 mytxt, 4121 ), 4122 importance = 1, 4123 type = "info", 4124 header = darkgreen(" * ") 4125 ) 4126 4127 return switched, already_switched, ignored, not_found, no_checksum

4128

4129 - def get_entropy_sets(self, repo = None, branch = None):

4130 4131 if branch is None: 4132 branch = self.SystemSettings['repositories']['branch'] 4133 if repo is None: 4134 repo = self.default_repository 4135 4136 sets_dir = self.get_local_database_sets_dir(repo, branch) 4137 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 4138 return {} 4139 4140 mydata = {} 4141 items = os.listdir(sets_dir) 4142 for item in items: 4143 4144 try: 4145 item_clean = str(item) 4146 except (UnicodeEncodeError, UnicodeDecodeError,): 4147 continue 4148 item_path = os.path.join(sets_dir, item) 4149 if not (os.path.isfile(item_path) and \ 4150 os.access(item_path, os.R_OK)): 4151 continue 4152 item_elements = self.entropyTools.extract_packages_from_set_file( 4153 item_path) 4154 if item_elements: 4155 mydata[item_clean] = item_elements.copy() 4156 4157 return mydata

4158

4159 - def get_configured_package_sets(self, repo = None, branch = None, 4160 validate = True):

4161 4162 if branch is None: 4163 branch = self.SystemSettings['repositories']['branch'] 4164 if repo is None: 4165 repo = self.default_repository 4166 4167 # portage sets 4168 sets_data = self.Spm().get_package_sets(False) 4169 sets_data.update(self.get_entropy_sets(repo, branch)) 4170 4171 if validate: 4172 invalid_sets = set() 4173 # validate 4174 for setname in sets_data: 4175 good = True 4176 for atom in sets_data[setname]: 4177 dbconn = self.open_server_repository(just_reading = True, 4178 repo = repo) 4179 match = dbconn.atomMatch(atom) 4180 if match[0] == -1: 4181 good = False 4182 break 4183 if not good: 4184 invalid_sets.add(setname) 4185 for invalid_set in invalid_sets: 4186 del sets_data[invalid_set] 4187 4188 return sets_data

4189

4190 - def update_database_package_sets(self, repo = None, dbconn = None):

4191 4192 if repo is None: 4193 repo = self.default_repository 4194 package_sets = self.get_configured_package_sets(repo) 4195 if dbconn is None: 4196 dbconn = self.open_server_repository( 4197 read_only = False, no_upload = True, repo = repo, 4198 do_treeupdates = False) 4199 dbconn.clearPackageSets() 4200 if package_sets: 4201 dbconn.insertPackageSets(package_sets) 4202 dbconn.commitChanges()

4203

Source Code for Module entropy.server.interfaces.main