entropy.server.interfaces.main

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Server Main Interfaces}. 10 11 """ 12 13 from __future__ import with_statement 14 import os 15 import shutil 16 from entropy.core import Singleton 17 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ 18 SystemDatabaseError 19 from entropy.const import etpConst, etpSys, const_setup_perms, \ 20 const_create_working_dirs, const_extract_srv_repo_params, etpUi 21 from entropy.output import TextInterface, purple, red, darkgreen, \ 22 bold, brown, blue, darkred 23 from entropy.server.interfaces.mirrors import Server as MirrorsServer 24 from entropy.i18n import _ 25 from entropy.core.settings.base import SystemSettings 26 from entropy.core.settings.plugins.skel import SystemSettingsPlugin 27 from entropy.transceivers import FtpInterface 28 from entropy.db import EntropyRepository 29 from entropy.spm.plugins.factory import get_default_instance as get_spm 30

31 -class ServerSystemSettingsPlugin(SystemSettingsPlugin):

32

33 - def server_parser(self, sys_set):

34 35 """ 36 Parses Entropy server system configuration file. 37 38 @return dict data 39 """ 40 41 data = { 42 'repositories': etpConst['server_repositories'].copy(), 43 'default_repository_id': etpConst['officialserverrepositoryid'], 44 'packages_expiration_days': etpConst['packagesexpirationdays'], 45 'database_file_format': etpConst['etpdatabasefileformat'], 46 'disabled_eapis': set(), 47 'exp_based_scope': etpConst['expiration_based_scope'], 48 'rss': { 49 'enabled': etpConst['rss-feed'], 50 'name': etpConst['rss-name'], 51 'base_url': etpConst['rss-base-url'], 52 'website_url': etpConst['rss-website-url'], 53 'editor': etpConst['rss-managing-editor'], 54 'max_entries': etpConst['rss-max-entries'], 55 'light_max_entries': etpConst['rss-light-max-entries'], 56 }, 57 } 58 59 fake_instance = self._helper.fake_default_repo 60 61 if not os.access(etpConst['serverconf'], os.R_OK): 62 return data 63 64 with open(etpConst['serverconf'],"r") as server_f: 65 serverconf = [x.strip() for x in server_f.readlines() if x.strip()] 66 67 for line in serverconf: 68 69 split_line = line.split("|") 70 split_line_len = len(split_line) 71 72 if (line.find("officialserverrepositoryid|") != -1) and \ 73 (not line.startswith("#")) and (split_line_len == 2): 74 75 if not fake_instance: 76 data['default_repository_id'] = split_line[1].strip() 77 78 elif line.startswith("expiration-days|") and (split_line_len == 2): 79 80 mydays = split_line[1].strip() 81 try: 82 mydays = int(mydays) 83 data['packages_expiration_days'] = mydays 84 except ValueError: 85 continue 86 87 elif line.startswith("expiration-based-scope|") and \ 88 (split_line_len == 2): 89 90 exp_opt = split_line[1].strip().lower() 91 if exp_opt in ("enable", "enabled", "true", "1", "yes"): 92 data['exp_based_scope'] = True 93 else: 94 data['exp_based_scope'] = False 95 96 elif line.startswith("disabled-eapis|") and (split_line_len == 2): 97 98 mydis = split_line[1].strip().split(",") 99 try: 100 mydis = [int(x) for x in mydis] 101 mydis = set([x for x in mydis if x in (1, 2, 3,)]) 102 except ValueError: 103 continue 104 if (len(mydis) < 3) and mydis: 105 data['disabled_eapis'] = mydis 106 107 108 elif line.startswith("repository|") and (split_line_len in [5, 6]) \ 109 and (not fake_instance): 110 111 repoid, repodata = const_extract_srv_repo_params(line, 112 product = sys_set['repositories']['product']) 113 if repoid in data['repositories']: 114 # just update mirrors 115 data['repositories'][repoid]['mirrors'].extend( 116 repodata['mirrors']) 117 else: 118 data['repositories'][repoid] = repodata.copy() 119 120 elif line.startswith("database-format|") and (split_line_len == 2): 121 122 fmt = split_line[1] 123 if fmt in etpConst['etpdatabasesupportedcformats']: 124 data['database_file_format'] = fmt 125 126 elif line.startswith("rss-feed|") and (split_line_len == 2): 127 128 feed = split_line[1] 129 if feed in ("enable", "enabled", "true", "1"): 130 data['rss']['enabled'] = True 131 elif feed in ("disable", "disabled", "false", "0", "no",): 132 data['rss']['enabled'] = False 133 134 elif line.startswith("rss-name|") and (split_line_len == 2): 135 136 feedname = line.split("rss-name|")[1].strip() 137 data['rss']['name'] = feedname 138 139 elif line.startswith("rss-base-url|") and (split_line_len == 2): 140 141 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip() 142 if not data['rss']['base_url'][-1] == "/": 143 data['rss']['base_url'] += "/" 144 145 elif line.startswith("rss-website-url|") and (split_line_len == 2): 146 147 data['rss']['website_url'] = split_line[1].strip() 148 149 elif line.startswith("managing-editor|") and (split_line_len == 2): 150 151 data['rss']['editor'] = split_line[1].strip() 152 153 elif line.startswith("max-rss-entries|") and (split_line_len == 2): 154 155 try: 156 entries = int(split_line[1].strip()) 157 data['rss']['max_entries'] = entries 158 except (ValueError, IndexError,): 159 continue 160 161 elif line.startswith("max-rss-light-entries|") and \ 162 (split_line_len == 2): 163 164 try: 165 entries = int(split_line[1].strip()) 166 data['rss']['light_max_entries'] = entries 167 except (ValueError, IndexError,): 168 continue 169 170 # add system database if community repository mode is enabled 171 if self._helper.community_repo: 172 data['repositories'][etpConst['clientserverrepoid']] = {} 173 mydata = {} 174 mydata['description'] = "Community Repositories System Database" 175 mydata['mirrors'] = [] 176 mydata['community'] = False 177 data['repositories'][etpConst['clientserverrepoid']].update(mydata) 178 179 # expand paths 180 for repoid in data['repositories']: 181 data['repositories'][repoid]['packages_dir'] = \ 182 os.path.join( etpConst['entropyworkdir'], 183 "server", 184 repoid, 185 "packages", 186 etpSys['arch'] 187 ) 188 data['repositories'][repoid]['store_dir'] = \ 189 os.path.join( etpConst['entropyworkdir'], 190 "server", 191 repoid, 192 "store", 193 etpSys['arch'] 194 ) 195 data['repositories'][repoid]['upload_dir'] = \ 196 os.path.join( etpConst['entropyworkdir'], 197 "server", 198 repoid, 199 "upload", 200 etpSys['arch'] 201 ) 202 data['repositories'][repoid]['database_dir'] = \ 203 os.path.join( etpConst['entropyworkdir'], 204 "server", 205 repoid, 206 "database", 207 etpSys['arch'] 208 ) 209 data['repositories'][repoid]['packages_relative_path'] = \ 210 os.path.join( sys_set['repositories']['product'], 211 repoid, 212 "packages", 213 etpSys['arch'] 214 )+"/" 215 data['repositories'][repoid]['database_relative_path'] = \ 216 os.path.join( sys_set['repositories']['product'], 217 repoid, 218 "database", 219 etpSys['arch'] 220 )+"/" 221 222 # Support for shell variables 223 shell_repoid = os.getenv('ETP_REPO') 224 if shell_repoid: 225 data['default_repository_id'] = shell_repoid 226 227 expiration_days = os.getenv('ETP_EXPIRATION_DAYS') 228 if expiration_days: 229 try: 230 expiration_days = int(expiration_days) 231 data['packages_expiration_days'] = expiration_days 232 except ValueError: 233 pass 234 235 return data

236

237 -class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

238 239 import entropy.tools as entropyTools 240

241 - def repos_parser(self, sys_set):

242 243 data = {} 244 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 245 # if support is not enabled, don't waste time scanning files 246 srv_parser_data = sys_set[srv_plug_id]['server'] 247 if not srv_parser_data['exp_based_scope']: 248 return data 249 250 # get expiration-based packages removal data from config files 251 for repoid in srv_parser_data['repositories']: 252 253 # filter out system repository if community repository 254 # mode is enabled 255 if repoid == etpConst['clientserverrepoid']: 256 continue 257 258 idpackages = set() 259 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file( 260 repo = repoid) 261 dbconn = self._helper.open_server_repository( 262 just_reading = True, repo = repoid) 263 264 if os.access(exp_fp, os.R_OK | os.F_OK): 265 pkgs = self.entropyTools.generic_file_content_parser(exp_fp) 266 if '*' in pkgs: # wildcard support 267 idpackages.add(-1) 268 else: 269 for pkg in pkgs: 270 idpackage, rc_match = dbconn.atomMatch(pkg) 271 if rc_match: 272 continue 273 idpackages.add(idpackage) 274 275 data[repoid] = idpackages 276 277 return data

278 279

280 -class Server(Singleton, TextInterface):

281

282 - def init_singleton(self, default_repository = None, save_repository = False, 283 community_repo = False, fake_default_repo = False, 284 fake_default_repo_id = '::fake::', 285 fake_default_repo_desc = 'this is a fake repository'):

286 287 self.__instance_destroyed = False 288 if etpConst['uid'] != 0: 289 mytxt = _("Entropy Server interface must be run as root") 290 raise PermissionDenied("PermissionDenied: %s" % (mytxt,)) 291 292 # settings 293 self.SystemSettings = SystemSettings() 294 self.community_repo = community_repo 295 from entropy.db import dbapi2 296 self.dbapi2 = dbapi2 # export for third parties 297 etpSys['serverside'] = True 298 self._memory_db_instances = {} 299 self.fake_default_repo = fake_default_repo 300 self.indexing = False 301 self.xcache = False 302 self.MirrorsService = None 303 self.__server_dbcache = {} 304 self.repository_treeupdate_digests = {} 305 self.__settings_to_backup = [] 306 self.__do_save_repository = save_repository 307 self.__sync_lock_cache = set() 308 self.rssMessages = { 309 'added': {}, 310 'removed': {}, 311 'commitmessage': "", 312 'light': {}, 313 } 314 315 if fake_default_repo: 316 default_repository = fake_default_repo_id 317 etpConst['officialserverrepositoryid'] = fake_default_repo_id 318 self.init_generic_memory_server_repository(fake_default_repo_id, 319 fake_default_repo_desc) 320 321 # create our SystemSettings plugin 322 self.sys_settings_plugin_id = \ 323 etpConst['system_settings_plugins_ids']['server_plugin'] 324 self.sys_settings_plugin = ServerSystemSettingsPlugin( 325 self.sys_settings_plugin_id, self) 326 self.SystemSettings.add_plugin(self.sys_settings_plugin) 327 328 # Fatscope support SystemSettings plugin 329 self.sys_settings_fatscope_plugin_id = \ 330 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 331 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin( 332 self.sys_settings_fatscope_plugin_id, self) 333 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin) 334 335 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 336 self.default_repository = default_repository 337 if self.default_repository == None: 338 self.default_repository = srv_set['default_repository_id'] 339 340 if self.default_repository in srv_set['repositories']: 341 self.ensure_paths(self.default_repository) 342 343 # NOW DISABLED, deprecated! 344 # self.migrate_repository_databases_to_new_branched_path() 345 346 if self.default_repository not in srv_set['repositories']: 347 raise PermissionDenied("PermissionDenied: %s %s" % ( 348 self.default_repository, 349 _("repository not configured"), 350 ) 351 ) 352 if etpConst['clientserverrepoid'] == self.default_repository: 353 raise PermissionDenied("PermissionDenied: %s %s" % ( 354 etpConst['clientserverrepoid'], 355 _("protected repository id, can't use this, sorry dude..."), 356 ) 357 ) 358 359 self.switch_default_repository(self.default_repository)

360

361 - def destroy(self):

362 self.__instance_destroyed = True 363 if hasattr(self,'ClientService'): 364 self.ClientService.destroy() 365 if hasattr(self,'sys_settings_server_plugin'): 366 try: 367 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id) 368 except KeyError: 369 pass 370 if hasattr(self,'sys_settings_fatscope_plugin'): 371 try: 372 self.SystemSettings.remove_plugin( 373 self.sys_settings_fatscope_plugin) 374 except KeyError: 375 pass 376 self.close_server_databases()

377

378 - def is_destroyed(self):

379 return self.__instance_destroyed

380

381 - def __del__(self):

382 self.destroy()

383

384 - def ensure_paths(self, repo):

385 upload_dir = os.path.join(self.get_local_upload_directory(repo), 386 self.SystemSettings['repositories']['branch']) 387 db_dir = self.get_local_database_dir(repo) 388 for mydir in [upload_dir, db_dir]: 389 if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)): 390 os.makedirs(mydir) 391 const_setup_perms(mydir, etpConst['entropygid'])

392 393

394 - def migrate_repository_databases_to_new_branched_path(self):

395 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 396 migrated_filename = '.branch_migrated' 397 for repoid in srv_set['repositories'].keys(): 398 399 if repoid == etpConst['clientserverrepoid']: continue 400 mydir = srv_set['repositories'][repoid]['database_dir'] 401 if not os.path.isdir(mydir): # empty ? 402 continue 403 404 migrated_filepath = os.path.join(mydir, migrated_filename) 405 if os.path.isfile(migrated_filepath): 406 continue 407 408 my_branched_dir = self.get_local_database_dir(repoid) 409 if os.path.isdir(my_branched_dir): # wtf? do not touch 410 continue 411 412 self.updateProgress( 413 "[%s:%s] %s: %s, %s: %s" % ( 414 brown("repo"), 415 purple(repoid), 416 _("migrating database path from"), 417 brown(mydir), 418 _('to'), 419 brown(my_branched_dir), 420 ), 421 importance = 1, 422 type = "info", 423 header = bold(" @@ ") 424 ) 425 426 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \ 427 (os.path.isfile(os.path.join(mydir, x)) and \ 428 os.access(os.path.join(mydir, x), os.W_OK)) 429 ] 430 os.makedirs(my_branched_dir) 431 const_setup_perms(my_branched_dir, etpConst['entropygid']) 432 433 for repo_file in repo_files: 434 repo_filename = os.path.basename(repo_file) 435 shutil.move(repo_file, 436 os.path.join(my_branched_dir, repo_filename)) 437 438 f_migrated = open(migrated_filepath,"w") 439 f_migrated.write("done\n") 440 f_migrated.flush() 441 f_migrated.close()

442

443 - def setup_services(self):

444 self.setup_entropy_settings() 445 cs_name = 'ClientService' 446 if hasattr(self, cs_name): 447 obj = getattr(self, cs_name) 448 obj.destroy() 449 from entropy.client.interfaces import Client 450 self.ClientService = Client( 451 indexing = self.indexing, 452 xcache = self.xcache, 453 repo_validation = False, 454 noclientdb = 1 455 ) 456 from entropy.cache import EntropyCacher 457 self.Cacher = EntropyCacher() 458 self.ClientService.updateProgress = self.updateProgress 459 self.validRepositories = self.ClientService.validRepositories 460 self.entropyTools = self.ClientService.entropyTools 461 self.dumpTools = self.ClientService.dumpTools 462 self.QA = self.ClientService.QA 463 self.backup_entropy_settings() 464 465 self.MirrorsService = MirrorsServer(self)

466

467 - def Spm(self):

468 """ 469 Get Source Package Manager interface instance. 470 471 @return: Source Package Manager interface instance 472 @rtype: entropy.spm.plugins.skel.SpmPlugin based instance 473 """ 474 return get_spm(self)

475

476 - def setup_entropy_settings(self, repo = None):

477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 478 backup_list = [ 479 'etpdatabaseclientfilepath', 480 'clientdbid', 481 {'server': srv_set.copy()}, 482 ] 483 for setting in backup_list: 484 if setting not in self.__settings_to_backup: 485 self.__settings_to_backup.append(setting) 486 # setup client database 487 if not self.community_repo: 488 etpConst['etpdatabaseclientfilepath'] = \ 489 self.get_local_database_file(repo) 490 etpConst['clientdbid'] = etpConst['serverdbid'] 491 const_create_working_dirs()

492

493 - def close_server_databases(self):

494 if hasattr(self,'serverDbCache'): 495 for item in self.__server_dbcache: 496 try: 497 self.__server_dbcache[item].closeDB() 498 except self.dbapi2.ProgrammingError: # already closed? 499 pass 500 self.__server_dbcache.clear()

501

502 - def close_server_database(self, dbinstance):

503 found = None 504 for item in self.__server_dbcache: 505 if dbinstance == self.__server_dbcache[item]: 506 found = item 507 break 508 if found: 509 instance = self.__server_dbcache.pop(found) 510 instance.closeDB()

511

512 - def get_available_repositories(self):

513 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 514 return srv_set['repositories'].copy()

515

516 - def switch_default_repository(self, repoid, save = None, 517 handle_uninitialized = True):

518 519 # avoid setting __default__ as default server repo 520 if repoid == etpConst['clientserverrepoid']: 521 return 522 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 523 524 if save == None: 525 save = self.__do_save_repository 526 if repoid not in srv_set['repositories']: 527 raise PermissionDenied("PermissionDenied: %s %s" % ( 528 repoid, 529 _("repository not configured"), 530 ) 531 ) 532 self.close_server_databases() 533 srv_set['default_repository_id'] = repoid 534 self.default_repository = repoid 535 self.setup_services() 536 if save: 537 self.save_default_repository(repoid) 538 539 self.setup_community_repositories_settings() 540 self.show_interface_status() 541 if handle_uninitialized and etpUi['warn']: 542 self.handle_uninitialized_repository(repoid)

543

544 - def setup_community_repositories_settings(self):

545 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 546 if self.community_repo: 547 for repoid in srv_set['repositories']: 548 srv_set['repositories'][repoid]['community'] = True

549 550

551 - def handle_uninitialized_repository(self, repoid):

552 553 if not self.is_repository_initialized(repoid): 554 mytxt = blue("%s.") % ( 555 _("Your default repository is not initialized"),) 556 self.updateProgress( 557 "[%s:%s] %s" % ( 558 brown("repo"), 559 purple(repoid), 560 mytxt, 561 ), 562 importance = 1, 563 type = "warning", 564 header = darkred(" !!! ") 565 ) 566 answer = self.askQuestion( 567 _("Do you want to initialize your default repository ?")) 568 if answer == "No": 569 mytxt = red("%s.") % ( 570 _("Continuing with an uninitialized repository"),) 571 self.updateProgress( 572 "[%s:%s] %s" % ( 573 brown("repo"), 574 purple(repoid), 575 mytxt, 576 ), 577 importance = 1, 578 type = "warning", 579 header = darkred(" !!! ") 580 ) 581 else: 582 # move empty database for security sake 583 dbfile = self.get_local_database_file(repoid) 584 if os.path.isfile(dbfile): 585 shutil.move(dbfile, dbfile+".backup") 586 self.initialize_server_database(empty = True, 587 repo = repoid, warnings = False)

588 589

590 - def show_interface_status(self):

591 type_txt = _("server-side repository") 592 if self.community_repo: 593 type_txt = _("community repository") 594 # ..on repository: <repository_name> 595 mytxt = _("Entropy Server Interface Instance on repository") 596 self.updateProgress( 597 blue("%s: %s, %s: %s (%s: %s)" % ( 598 mytxt, 599 red(self.default_repository), 600 _("current branch"), 601 darkgreen(self.SystemSettings['repositories']['branch']), 602 purple(_("type")), 603 bold(type_txt), 604 ) 605 ), 606 importance = 2, 607 type = "info", 608 header = red(" @@ ") 609 ) 610 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 611 repos = srv_set['repositories'].keys() 612 mytxt = blue("%s:") % (_("Currently configured repositories"),) 613 self.updateProgress( 614 mytxt, 615 importance = 1, 616 type = "info", 617 header = red(" @@ ") 618 ) 619 for repo in repos: 620 self.updateProgress( 621 darkgreen(repo), 622 importance = 0, 623 type = "info", 624 header = brown(" # ") 625 )

626 627

628 - def save_default_repository(self, repoid):

629 630 # avoid setting __default__ as default server repo 631 if repoid == etpConst['clientserverrepoid']: 632 return 633 634 if os.path.isfile(etpConst['serverconf']): 635 f_srv = open(etpConst['serverconf'],"r") 636 content = f_srv.readlines() 637 f_srv.close() 638 content = [x.strip() for x in content] 639 found = False 640 new_content = [] 641 for line in content: 642 if line.strip().startswith("officialserverrepositoryid|"): 643 line = "officialserverrepositoryid|%s" % (repoid,) 644 found = True 645 new_content.append(line) 646 if not found: 647 new_content.append("officialserverrepositoryid|%s" % (repoid,)) 648 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w") 649 for line in new_content: 650 f_srv_t.write(line+"\n") 651 f_srv_t.flush() 652 f_srv_t.close() 653 os.rename(etpConst['serverconf']+".save_default_repo_tmp", 654 etpConst['serverconf']) 655 else: 656 f_srv = open(etpConst['serverconf'],"w") 657 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,)) 658 f_srv.flush() 659 f_srv.close()

660

661 - def toggle_repository(self, repoid, enable = True):

662 663 # avoid setting __default__ as default server repo 664 if repoid == etpConst['clientserverrepoid']: 665 return False 666 667 if not os.path.isfile(etpConst['serverconf']): 668 return None 669 f_srv = open(etpConst['serverconf']) 670 tmpfile = etpConst['serverconf']+".switch" 671 mycontent = [x.strip() for x in f_srv.readlines()] 672 f_srv.close() 673 f_tmp = open(tmpfile,"w") 674 st = "repository|%s" % (repoid,) 675 status = False 676 for line in mycontent: 677 if enable: 678 if (line.find(st) != -1) and line.startswith("#") and \ 679 (len(line.split("|")) == 5): 680 line = line[1:] 681 status = True 682 else: 683 if (line.find(st) != -1) and not line.startswith("#") and \ 684 (len(line.split("|")) == 5): 685 line = "#"+line 686 status = True 687 f_tmp.write(line+"\n") 688 f_tmp.flush() 689 f_tmp.close() 690 shutil.move(tmpfile, etpConst['serverconf']) 691 if status: 692 self.close_server_databases() 693 self.SystemSettings.clear() 694 self.setup_services() 695 self.show_interface_status() 696 return status

697

698 - def backup_entropy_settings(self):

699 for setting in self.__settings_to_backup: 700 if isinstance(setting, basestring): 701 self.ClientService.backup_constant(setting) 702 elif isinstance(setting, dict): 703 self.SystemSettings.set_persistent_setting(setting)

704

705 - def is_repository_initialized(self, repo):

706 707 def do_validate(dbc): 708 try: 709 dbc.validateDatabase() 710 return True 711 except SystemDatabaseError: 712 return False

713 714 dbc = self.open_server_repository(just_reading = True, repo = repo) 715 valid = do_validate(dbc) 716 self.close_server_database(dbc) 717 if not valid: # check online? 718 dbc = self.open_server_repository(read_only = False, 719 no_upload = True, repo = repo, is_new = True) 720 valid = do_validate(dbc) 721 self.close_server_database(dbc) 722 723 return valid

724

725 - def do_server_repository_sync_lock(self, repo, no_upload):

726 727 if repo == None: 728 repo = self.default_repository 729 730 # check if the database is locked locally 731 lock_file = self.MirrorsService.get_database_lockfile(repo) 732 if os.path.isfile(lock_file): 733 self.updateProgress( 734 red(_("Entropy database is already locked by you :-)")), 735 importance = 1, 736 type = "info", 737 header = red(" * ") 738 ) 739 else: 740 # check if the database is locked REMOTELY 741 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),) 742 self.updateProgress( 743 red(mytxt), 744 importance = 1, 745 type = "info", 746 header = red(" * "), 747 back = True 748 ) 749 for uri in self.get_remote_mirrors(repo): 750 given_up = self.MirrorsService.mirror_lock_check(uri, 751 repo = repo) 752 if given_up: 753 crippled_uri = \ 754 self.entropyTools.extract_ftp_host_from_uri(uri) 755 mytxt = "%s:" % (_("Mirrors status table"),) 756 self.updateProgress( 757 darkgreen(mytxt), 758 importance = 1, 759 type = "info", 760 header = brown(" * ") 761 ) 762 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo) 763 for db_uri, db_st1, db_st2 in dbstatus: 764 db_st1_info = darkgreen(_("Unlocked")) 765 if db_st1: 766 db_st1_info = red(_("Locked")) 767 db_st2_info = darkgreen(_("Unlocked")) 768 if db_st2: 769 db_st2_info = red(_("Locked")) 770 771 crippled_uri = \ 772 self.entropyTools.extract_ftp_host_from_uri(db_uri) 773 self.updateProgress( 774 "%s: [%s: %s] [%s: %s]" % ( 775 bold(crippled_uri), 776 brown(_("database")), 777 db_st1_info, 778 brown(_("download")), 779 db_st2_info, 780 ), 781 importance = 1, 782 type = "info", 783 header = "\t" 784 ) 785 786 raise OnlineMirrorError("OnlineMirrorError: %s %s" % ( 787 _("cannot lock mirror"), 788 crippled_uri, 789 ) 790 ) 791 792 # if we arrive here, it is because all the mirrors are unlocked 793 self.MirrorsService.lock_mirrors(True, repo = repo) 794 self.MirrorsService.sync_databases(no_upload, repo = repo)

795 796

797 - def init_generic_memory_server_repository(self, repoid, description, 798 mirrors = None, community_repo = False, service_url = None):

799 800 if mirrors is None: 801 mirrors = [] 802 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid) 803 self._memory_db_instances[repoid] = dbc 804 805 eapi3_port = int(etpConst['socket_service']['port']) 806 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port']) 807 # add to settings 808 repodata = { 809 'repoid': repoid, 810 'description': description, 811 'mirrors': mirrors, 812 'community': community_repo, 813 'service_port': eapi3_port, 814 'ssl_service_port': eapi3_ssl_port, 815 'service_url': service_url, 816 'handler': '', # not supported 817 'in_memory': True, 818 } 819 820 etpConst['server_repositories'][repoid] = repodata 821 self.SystemSettings.clear() 822 823 return dbc

824

825 - def open_memory_database(self, dbname = None):

826 if dbname == None: 827 dbname = etpConst['genericdbid'] 828 dbc = EntropyRepository( 829 readOnly = False, 830 dbFile = ':memory:', 831 clientDatabase = True, 832 dbname = dbname, 833 xcache = False, 834 indexing = False, 835 OutputInterface = self, 836 skipChecks = True 837 ) 838 dbc.initializeDatabase() 839 return dbc

840

841 - def open_server_repository( 842 self, 843 read_only = True, 844 no_upload = True, 845 just_reading = False, 846 repo = None, 847 indexing = True, 848 warnings = True, 849 do_cache = True, 850 use_branch = None, 851 lock_remote = True, 852 is_new = False, 853 do_treeupdates = True 854 ):

855 856 if repo == None: 857 repo = self.default_repository 858 859 if repo == etpConst['clientserverrepoid'] and self.community_repo: 860 return self.ClientService.clientDbconn 861 862 # in-memory server repos 863 if repo in self._memory_db_instances: 864 return self._memory_db_instances.get(repo) 865 866 if just_reading: 867 read_only = True 868 no_upload = True 869 870 local_dbfile = self.get_local_database_file(repo, use_branch) 871 if do_cache: 872 cached = self.__server_dbcache.get( 873 (repo, etpConst['systemroot'], local_dbfile, read_only, 874 no_upload, just_reading, use_branch, lock_remote,) 875 ) 876 if cached != None: 877 return cached 878 879 local_dbfile_dir = os.path.dirname(local_dbfile) 880 if not os.path.isdir(local_dbfile_dir): 881 os.makedirs(local_dbfile_dir) 882 883 if (not read_only) and (lock_remote) and \ 884 (repo not in self.__sync_lock_cache): 885 self.do_server_repository_sync_lock(repo, no_upload) 886 self.__sync_lock_cache.add(repo) 887 888 local_dbfile_exists = os.path.lexists(local_dbfile) 889 conn = EntropyRepository( 890 readOnly = read_only, 891 dbFile = local_dbfile, 892 noUpload = no_upload, 893 OutputInterface = self, 894 dbname = etpConst['serverdbid']+repo, 895 useBranch = use_branch, 896 lockRemote = lock_remote 897 ) 898 if not local_dbfile_exists: 899 # better than having a completely broken db 900 conn.readOnly = False 901 conn.initializeDatabase() 902 conn.commitChanges() 903 904 valid = True 905 try: 906 conn.validateDatabase() 907 except SystemDatabaseError: 908 valid = False 909 910 # verify if we need to update the database to sync 911 # with portage updates, we just ignore being readonly in the case 912 if (repo not in etpConst['server_treeupdatescalled']) and \ 913 (not just_reading): 914 # sometimes, when filling a new server db 915 # we need to avoid tree updates 916 if valid: 917 if do_treeupdates: 918 self.repository_packages_spm_sync(conn, 919 branch = use_branch, repo = repo) 920 elif warnings and not is_new: 921 mytxt = _("Entropy database is corrupted!") 922 self.updateProgress( 923 darkred(mytxt), 924 importance = 1, 925 type = "warning", 926 header = bold(" !!! ") 927 ) 928 929 if not read_only and valid and indexing: 930 931 self.updateProgress( 932 "[repo:%s|%s] %s" % ( 933 blue(repo), 934 red(_("database")), 935 blue(_("indexing database")), 936 ), 937 importance = 1, 938 type = "info", 939 header = brown(" @@ "), 940 back = True 941 ) 942 conn.createAllIndexes() 943 944 if do_cache: 945 # !!! also cache just_reading otherwise there will be 946 # real issues if the connection is opened several times 947 self.__server_dbcache[ 948 (repo, etpConst['systemroot'], local_dbfile, read_only, 949 no_upload, just_reading, use_branch, lock_remote,)] = conn 950 951 # auto-update package sets 952 if (not read_only) and (not is_new): 953 cur_sets = conn.retrievePackageSets() 954 sys_sets = self.get_configured_package_sets(repo) 955 if cur_sets != sys_sets: 956 self.update_database_package_sets(repo, dbconn = conn) 957 conn.commitChanges() 958 959 return conn

960

961 - def repository_packages_spm_sync(self, repo_db, branch = None, repo = None):

962 """ 963 Service method used to sync package names with Source Package Manager. 964 Source Package Manager can change package names, categories or slot 965 and Entropy repositories must be kept in sync. 966 967 In other words, it checks for /usr/portage/profiles/updates changes. 968 """ 969 970 if branch == None: 971 branch = self.SystemSettings['repositories']['branch'] 972 if repo == None: 973 repo = self.default_repository 974 975 etpConst['server_treeupdatescalled'].add(repo) 976 977 repo_updates_file = self.get_local_database_treeupdates_file(repo) 978 doRescan = False 979 980 stored_digest = repo_db.retrieveRepositoryUpdatesDigest(repo) 981 if stored_digest == -1: 982 doRescan = True 983 984 # check portage files for changes if doRescan is still false 985 portage_dirs_digest = "0" 986 if not doRescan: 987 988 if self.repository_treeupdate_digests.has_key(repo): 989 portage_dirs_digest = self.repository_treeupdate_digests.get( 990 repo) 991 else: 992 993 spm = self.Spm() 994 # grab portdir 995 updates_dir = etpConst['systemroot'] + \ 996 spm.get_setting("PORTDIR") + "/profiles/updates" 997 if os.path.isdir(updates_dir): 998 # get checksum 999 mdigest = self.entropyTools.md5obj_directory(updates_dir) 1000 # also checksum etpConst['etpdatabaseupdatefile'] 1001 if os.path.isfile(repo_updates_file): 1002 f = open(repo_updates_file) 1003 block = f.read(1024) 1004 while block: 1005 mdigest.update(block) 1006 block = f.read(1024) 1007 f.close() 1008 portage_dirs_digest = mdigest.hexdigest() 1009 self.repository_treeupdate_digests[repo] = \ 1010 portage_dirs_digest 1011 1012 if doRescan or (str(stored_digest) != str(portage_dirs_digest)): 1013 1014 # force parameters 1015 repo_db.readOnly = False 1016 repo_db.noUpload = True 1017 1018 # reset database tables 1019 repo_db.clearTreeupdatesEntries(repo) 1020 1021 updates_dir = etpConst['systemroot'] + \ 1022 self.Spm().get_setting("PORTDIR") + "/profiles/updates" 1023 update_files = self.entropyTools.sort_update_files( 1024 os.listdir(updates_dir)) 1025 update_files = [os.path.join(updates_dir, x) for x in update_files] 1026 # now load actions from files 1027 update_actions = [] 1028 for update_file in update_files: 1029 f = open(update_file, "r") 1030 mycontent = f.readlines() 1031 f.close() 1032 lines = [x.strip() for x in mycontent if x.strip()] 1033 update_actions.extend(lines) 1034 1035 # add entropy packages.db.repo_updates content 1036 if os.path.isfile(repo_updates_file): 1037 f = open(repo_updates_file, "r") 1038 mycontent = f.readlines() 1039 f.close() 1040 lines = [x.strip() for x in mycontent if x.strip() and \ 1041 not x.strip().startswith("#")] 1042 update_actions.extend(lines) 1043 # now filter the required actions 1044 update_actions = repo_db.filterTreeUpdatesActions(update_actions) 1045 if update_actions: 1046 1047 mytxt = "%s: %s. %s %s" % ( 1048 bold(_("ATTENTION")), 1049 red(_("forcing package updates")), 1050 red(_("Syncing with")), 1051 blue(updates_dir), 1052 ) 1053 self.updateProgress( 1054 mytxt, 1055 importance = 1, 1056 type = "info", 1057 header = brown(" * ") 1058 ) 1059 # lock database 1060 if repo_db.lockRemote: 1061 self.do_server_repository_sync_lock( 1062 repo, repo_db.noUpload) 1063 # now run queue 1064 try: 1065 quickpkg_list = repo_db.runTreeUpdatesActions( 1066 update_actions) 1067 except: 1068 # destroy digest 1069 repo_db.setRepositoryUpdatesDigest(repo, "-1") 1070 raise 1071 1072 if quickpkg_list: 1073 # quickpkg package and packages owning it as a dependency 1074 try: 1075 self._run_packages_spm_sync_quickpkg( 1076 quickpkg_list, repo_db, repo) 1077 except: 1078 self.entropyTools.print_traceback() 1079 mytxt = "%s: %s: %s, %s." % ( 1080 bold(_("WARNING")), 1081 red(_("Cannot complete quickpkg for atoms")), 1082 blue(str(sorted(quickpkg_list))), 1083 _("do it manually"), 1084 ) 1085 self.updateProgress( 1086 mytxt, 1087 importance = 1, 1088 type = "warning", 1089 header = darkred(" * ") 1090 ) 1091 repo_db.commitChanges() 1092 1093 # store new actions 1094 repo_db.addRepositoryUpdatesActions( 1095 repo, update_actions, branch) 1096 1097 # store new digest into database 1098 repo_db.setRepositoryUpdatesDigest( 1099 repo, portage_dirs_digest) 1100 repo_db.commitChanges()

1101

1102 - def _run_packages_spm_sync_quickpkg(self, atoms, repo_db, repo):

1103 """ 1104 Executes packages regeneration for given atoms. 1105 """ 1106 package_paths = set() 1107 runatoms = set() 1108 for myatom in atoms: 1109 mymatch = repo_db.atomMatch(myatom) 1110 if mymatch[0] == -1: 1111 continue 1112 myatom = repo_db.retrieveAtom(mymatch[0]) 1113 myatom = self.entropyTools.remove_tag(myatom) 1114 runatoms.add(myatom) 1115 1116 for myatom in runatoms: 1117 1118 self.updateProgress( 1119 red("%s: " % (_("repackaging"),) )+blue(myatom), 1120 importance = 1, 1121 type = "warning", 1122 header = blue(" # ") 1123 ) 1124 mydest = self.get_local_store_directory(repo = repo) 1125 try: 1126 mypath = self.quickpkg(myatom, mydest) 1127 except: 1128 # remove broken bin before raising 1129 mypath = os.path.join(mydest, 1130 os.path.basename(myatom) + etpConst['packagesext']) 1131 if os.path.isfile(mypath): 1132 os.remove(mypath) 1133 self.entropyTools.print_traceback() 1134 mytxt = "%s: %s: %s, %s." % ( 1135 bold(_("WARNING")), 1136 red(_("Cannot complete quickpkg for atom")), 1137 blue(myatom), 1138 _("do it manually"), 1139 ) 1140 self.updateProgress( 1141 mytxt, 1142 importance = 1, 1143 type = "warning", 1144 header = darkred(" * ") 1145 ) 1146 continue 1147 package_paths.add(mypath) 1148 packages_data = [(x, False,) for x in package_paths] 1149 idpackages = self.add_packages_to_repository( 1150 packages_data, repo = repo) 1151 1152 if not idpackages: 1153 1154 mytxt = "%s: %s. %s." % ( 1155 bold(_("ATTENTION")), 1156 red(_("package files rebuild did not run properly")), 1157 red(_("Please update packages manually")), 1158 ) 1159 self.updateProgress( 1160 mytxt, 1161 importance = 1, 1162 type = "warning", 1163 header = darkred(" * ") 1164 )

1165

1166 - def deps_tester(self, default_repo = None):

1167 1168 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1169 server_repos = sys_set['repositories'].keys() 1170 installed_packages = set() 1171 # if a default repository is passed, we will just test against it 1172 if default_repo: 1173 server_repos = [default_repo] 1174 1175 for repo in server_repos: 1176 dbconn = self.open_server_repository(read_only = True, 1177 no_upload = True, repo = repo, do_treeupdates = False) 1178 installed_packages |= set([(x, repo) for x in \ 1179 dbconn.listAllIdpackages()]) 1180 1181 1182 deps_not_satisfied = set() 1183 length = str((len(installed_packages))) 1184 count = 0 1185 mytxt = _("Checking") 1186 1187 for idpackage, repo in installed_packages: 1188 count += 1 1189 dbconn = self.open_server_repository(read_only = True, 1190 no_upload = True, repo = repo, do_treeupdates = False) 1191 1192 if (count%150 == 0) or (count == length) or (count == 1): 1193 atom = dbconn.retrieveAtom(idpackage) 1194 self.updateProgress( 1195 darkgreen(mytxt)+" "+bold(atom), 1196 importance = 0, 1197 type = "info", 1198 back = True, 1199 count = (count, length), 1200 header = darkred(" @@ ") 1201 ) 1202 1203 xdeps = dbconn.retrieveDependencies(idpackage) 1204 for xdep in xdeps: 1205 xid, xuseless = self.atom_match(xdep) 1206 if xid == -1: 1207 deps_not_satisfied.add(xdep) 1208 1209 return deps_not_satisfied

1210

1211 - def dependencies_test(self, repo = None):

1212 1213 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("...")) 1214 self.updateProgress( 1215 mytxt, 1216 importance = 2, 1217 type = "info", 1218 header = red(" @@ ") 1219 ) 1220 1221 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1222 server_repos = srv_set['repositories'].keys() 1223 deps_not_matched = self.deps_tester(repo) 1224 1225 if deps_not_matched: 1226 1227 crying_atoms = {} 1228 for atom in deps_not_matched: 1229 for repo in server_repos: 1230 dbconn = self.open_server_repository(just_reading = True, 1231 repo = repo, do_treeupdates = False) 1232 riddep = dbconn.searchDependency(atom) 1233 if riddep == -1: 1234 continue 1235 ridpackages = dbconn.searchIdpackageFromIddependency(riddep) 1236 for i in ridpackages: 1237 iatom = dbconn.retrieveAtom(i) 1238 if not crying_atoms.has_key(atom): 1239 crying_atoms[atom] = set() 1240 crying_atoms[atom].add((iatom, repo)) 1241 1242 mytxt = blue("%s:") % (_("These are the dependencies not found"),) 1243 self.updateProgress( 1244 mytxt, 1245 importance = 1, 1246 type = "info", 1247 header = red(" @@ ") 1248 ) 1249 mytxt = "%s:" % (_("Needed by"),) 1250 for atom in deps_not_matched: 1251 self.updateProgress( 1252 red(atom), 1253 importance = 1, 1254 type = "info", 1255 header = blue(" # ") 1256 ) 1257 if crying_atoms.has_key(atom): 1258 self.updateProgress( 1259 red(mytxt), 1260 importance = 0, 1261 type = "info", 1262 header = blue(" # ") 1263 ) 1264 for my_dep, myrepo in crying_atoms[atom]: 1265 self.updateProgress( 1266 "[%s:%s] %s" % ( 1267 blue(_("by repo")), 1268 darkred(myrepo), 1269 darkgreen(my_dep), 1270 ), 1271 importance = 0, 1272 type = "info", 1273 header = blue(" # ") 1274 ) 1275 else: 1276 1277 mytxt = blue(_("Every dependency is satisfied. It's all fine.")) 1278 self.updateProgress( 1279 mytxt, 1280 importance = 2, 1281 type = "info", 1282 header = red(" @@ ") 1283 ) 1284 1285 return deps_not_matched

1286

1287 - def test_shared_objects(self, get_files = False, repo = None):

1288 1289 # load db 1290 dbconn = self.open_server_repository(read_only = True, 1291 no_upload = True, repo = repo) 1292 QA = self.QA() 1293 packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn, 1294 broken_symbols = True) 1295 if status != 0: 1296 return 1, None 1297 1298 if get_files: 1299 return 0, brokenexecs 1300 1301 if (not brokenexecs) and (not packages_matched): 1302 mytxt = "%s." % (_("System is healthy"),) 1303 self.updateProgress( 1304 blue(mytxt), 1305 importance = 2, 1306 type = "info", 1307 header = red(" @@ ") 1308 ) 1309 return 0, None 1310 1311 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),) 1312 self.updateProgress( 1313 blue(mytxt), 1314 importance = 1, 1315 type = "info", 1316 header = red(" @@ ") 1317 ) 1318 1319 packages = self.Spm().search_paths_owners(brokenexecs) 1320 1321 if packages: 1322 mytxt = "%s:" % (_("These are the matched packages"),) 1323 self.updateProgress( 1324 red(mytxt), 1325 importance = 1, 1326 type = "info", 1327 header = red(" @@ ") 1328 ) 1329 for my_atom, my_elf_id in packages: 1330 package_slot = my_atom, my_elf_id 1331 self.updateProgress( 1332 "%s [elf:%s]" % ( 1333 purple(my_atom), 1334 darkgreen(str(my_elf_id)), 1335 ), 1336 importance = 0, 1337 type = "info", 1338 header = red(" # ") 1339 ) 1340 for filename in sorted(list(packages[package_slot])): 1341 self.updateProgress( 1342 darkgreen(filename), 1343 importance = 0, 1344 type = "info", 1345 header = brown(" => ") 1346 ) 1347 # print string 1348 pkgstring = ' '.join(["%s:%s" % ( 1349 self.entropyTools.dep_getkey(x[0]), x[1],) for x \ 1350 in sorted(packages)]) 1351 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,) 1352 self.updateProgress( 1353 mytxt, 1354 importance = 1, 1355 type = "info", 1356 header = red(" @@ ") 1357 ) 1358 else: 1359 self.updateProgress( 1360 red(_("No matched packages")), 1361 importance = 1, 1362 type = "info", 1363 header = red(" @@ ") 1364 ) 1365 1366 return 0, packages

1367

1368 - def orphaned_spm_packages_test(self):

1369 1370 mytxt = "%s %s" % ( 1371 blue(_("Running orphaned SPM packages test")), red("..."),) 1372 self.updateProgress( 1373 mytxt, 1374 importance = 2, 1375 type = "info", 1376 header = red(" @@ ") 1377 ) 1378 installed_packages = self.Spm().get_installed_packages() 1379 length = len(installed_packages) 1380 not_found = {} 1381 count = 0 1382 for installed_package in installed_packages: 1383 count += 1 1384 self.updateProgress( 1385 "%s: %s" % ( 1386 darkgreen(_("Scanning package")), 1387 brown(installed_package),), 1388 importance = 0, 1389 type = "info", 1390 back = True, 1391 count = (count, length), 1392 header = darkred(" @@ ") 1393 ) 1394 key, slot = (self.entropyTools.dep_getkey(installed_package), 1395 self.Spm().get_installed_package_metadata(installed_package, 1396 "SLOT"),) 1397 pkg_atom = "%s:%s" % (key, slot,) 1398 tree_atom = self.Spm().match_package(pkg_atom) 1399 if not tree_atom: 1400 not_found[installed_package] = pkg_atom 1401 self.updateProgress( 1402 "%s: %s" % ( 1403 blue(pkg_atom), 1404 darkred(_("not found anymore")), 1405 ), 1406 importance = 0, 1407 type = "warning", 1408 count = (count, length), 1409 header = darkred(" @@ ") 1410 ) 1411 1412 if not_found: 1413 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)]) 1414 self.updateProgress( 1415 "%s: %s" % ( 1416 blue(_("Packages string")), 1417 not_found_list, 1418 ), 1419 importance = 0, 1420 type = "warning", 1421 count = (count, length), 1422 header = darkred(" @@ ") 1423 ) 1424 1425 return not_found

1426

1427 - def depends_table_initialize(self, repo = None):

1428 dbconn = self.open_server_repository(read_only = False, 1429 no_upload = True, repo = repo) 1430 dbconn.regenerateReverseDependenciesMetadata() 1431 dbconn.taintDatabase() 1432 dbconn.commitChanges()

1433

1434 - def library_paths_table_initialize(self, repo = None):

1435 dbconn = self.open_server_repository(read_only = False, 1436 no_upload = True, repo = repo) 1437 dbconn.regenerateLibrarypathsidpackageTable() 1438 dbconn.taintDatabase() 1439 dbconn.commitChanges()

1440

1441 - def create_empty_database(self, dbpath = None, repo = None):

1442 if dbpath == None: 1443 dbpath = self.get_local_database_file(repo) 1444 1445 dbdir = os.path.dirname(dbpath) 1446 if not os.path.isdir(dbdir): 1447 os.makedirs(dbdir) 1448 1449 mytxt = red("%s ...") % (_("Initializing an empty database"),) 1450 self.updateProgress( 1451 mytxt, 1452 importance = 1, 1453 type = "info", 1454 header = darkgreen(" * "), 1455 back = True 1456 ) 1457 dbconn = self.ClientService.open_generic_database(dbpath) 1458 dbconn.initializeDatabase() 1459 dbconn.commitChanges() 1460 dbconn.closeDB() 1461 mytxt = "%s %s %s." % ( 1462 red(_("Entropy database file")), 1463 bold(dbpath), 1464 red(_("successfully initialized")), 1465 ) 1466 self.updateProgress( 1467 mytxt, 1468 importance = 1, 1469 type = "info", 1470 header = darkgreen(" * ") 1471 )

1472

1473 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

1474 1475 # check package_tag "no spaces" 1476 1477 try: 1478 package_tag = str(package_tag) 1479 if " " in package_tag: 1480 raise ValueError 1481 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,): 1482 self.updateProgress( 1483 "%s: %s" % ( 1484 blue(_("Invalid tag specified")), 1485 package_tag, 1486 ), 1487 importance = 1, type = "error", header = darkred(" !! ") 1488 ) 1489 return 1, package_tag 1490 1491 if repo == None: 1492 repo = self.default_repository 1493 1494 # sanity check 1495 invalid_atoms = [] 1496 dbconn = self.open_server_repository(read_only = True, 1497 no_upload = True, repo = repo) 1498 for idpackage in idpackages: 1499 ver_tag = dbconn.retrieveVersionTag(idpackage) 1500 if ver_tag: 1501 invalid_atoms.append(dbconn.retrieveAtom(idpackage)) 1502 1503 if invalid_atoms: 1504 self.updateProgress( 1505 "%s: %s" % ( 1506 blue(_("Packages already tagged, action aborted")), 1507 ', '.join([darkred(unicode(x)) for x in invalid_atoms]), 1508 ), 1509 importance = 1, type = "error", header = darkred(" !! ") 1510 ) 1511 return 2, invalid_atoms 1512 1513 matches = [(x, repo) for x in idpackages] 1514 status = 0 1515 data = self.move_packages( 1516 matches, to_repo = repo, from_repo = repo, ask = ask, 1517 do_copy = True, new_tag = package_tag 1518 ) 1519 return status, data

1520

1521 - def flushback_packages(self, from_branches, repo = None, ask = True):

1522 """ 1523 When creating a new branch, for space reasons, packages are not 1524 moved to a new location. This works fine until old branch is removed. 1525 To avoid inconsistences, before deciding to do that, all the packages 1526 in the old branch should be flushed back to the the currently configured 1527 branch. 1528 1529 @param from_branches -- list of branches to move packages from 1530 @type from_branches -- list 1531 @param repo -- repository to work on 1532 @type repo -- str 1533 @param ask -- user interactivity 1534 @type ask -- bool 1535 1536 @return status 1537 """ 1538 1539 status = True 1540 if repo == None: 1541 repo = self.default_repository 1542 branch = self.SystemSettings['repositories']['branch'] 1543 1544 if branch in from_branches: 1545 from_branches = [x for x in from_branches if x != branch] 1546 1547 self.updateProgress( 1548 "[%s=>%s|%s] %s" % ( 1549 darkgreen(', '.join(from_branches)), 1550 darkred(branch), 1551 brown(repo), 1552 blue(_("flushing back selected packages from branches")), 1553 ), 1554 importance = 2, 1555 type = "info", 1556 header = red(" @@ ") 1557 ) 1558 1559 dbconn = self.open_server_repository(read_only = True, 1560 no_upload = True, repo = repo) 1561 1562 idpackage_map = dict(((x, [],) for x in from_branches)) 1563 idpackages = dbconn.listAllIdpackages(order_by = 'atom') 1564 for idpackage in idpackages: 1565 download_url = dbconn.retrieveDownloadURL(idpackage) 1566 url_br = self.ClientService.get_branch_from_download_relative_uri( 1567 download_url) 1568 if url_br in from_branches: 1569 idpackage_map[url_br].append(idpackage) 1570 1571 mapped_branches = [x for x in idpackage_map if idpackage_map[x]] 1572 if not mapped_branches: 1573 self.updateProgress( 1574 "[%s=>%s|%s] %s !" % ( 1575 darkgreen(', '.join(from_branches)), 1576 darkred(branch), 1577 brown(repo), 1578 blue(_("nothing to do")), 1579 ), 1580 importance = 0, 1581 type = "warning", 1582 header = blue(" @@ ") 1583 ) 1584 return status 1585 1586 1587 all_fine = True 1588 tmp_down_dir = self.entropyTools.get_random_temp_file() 1589 os.makedirs(tmp_down_dir) 1590 1591 download_queue = {} 1592 local_up_dir = self.get_local_upload_directory(repo) 1593 local_basedir = os.path.join(local_up_dir, branch) 1594 dbconn = self.open_server_repository(read_only = False, 1595 no_upload = True, repo = repo) 1596 1597 def generate_queue(branch, repo, from_branch, down_q, idpackage_map): 1598 1599 self.updateProgress( 1600 "[%s=>%s|%s] %s" % ( 1601 darkgreen(from_branch), 1602 darkred(branch), 1603 brown(repo), 1604 brown(_("these are the packages that will be flushed")), 1605 ), 1606 importance = 1, 1607 type = "info", 1608 header = brown(" @@ ") 1609 ) 1610 1611 1612 for idpackage in idpackage_map[from_branch]: 1613 atom = dbconn.retrieveAtom(idpackage) 1614 self.updateProgress( 1615 "[%s=>%s|%s] %s" % ( 1616 darkgreen(from_branch), 1617 darkred(branch), 1618 brown(repo), 1619 purple(atom), 1620 ), 1621 importance = 0, 1622 type = "info", 1623 header = blue(" # ") 1624 ) 1625 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) 1626 pkg_fp = os.path.join(tmp_down_dir, pkg_fp) 1627 down_q.append((pkg_fp, idpackage,))

1628 1629 1630 for from_branch in sorted(mapped_branches): 1631 1632 download_queue[from_branch] = [] 1633 all_fine = False 1634 generate_queue(branch, repo, from_branch, 1635 download_queue[from_branch], idpackage_map) 1636 1637 if ask: 1638 rc_question = self.askQuestion( 1639 _("Would you like to continue ?")) 1640 if rc_question == "No": 1641 continue 1642 1643 remote_relative_path = self.get_remote_packages_relative_path(repo) 1644 1645 for uri in self.get_remote_mirrors(repo): 1646 1647 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1648 ftp_basedir = os.path.join(remote_relative_path, from_branch) 1649 1650 downloader_queue = [x[0] for x in download_queue[from_branch]] 1651 downloader = self.MirrorsService.FtpServerHandler( 1652 FtpInterface, 1653 self, 1654 [uri], 1655 downloader_queue, 1656 critical_files = downloader_queue, 1657 use_handlers = True, 1658 ftp_basedir = ftp_basedir, 1659 local_basedir = tmp_down_dir, 1660 download = True, 1661 repo = repo 1662 ) 1663 1664 errors, m_fine_uris, m_broken_uris = downloader.go() 1665 1666 if not errors: 1667 for downloaded_path, idpackage in \ 1668 download_queue[from_branch]: 1669 1670 self.updateProgress( 1671 "[%s=>%s|%s|%s] %s: %s" % ( 1672 darkgreen(from_branch), 1673 darkred(branch), 1674 brown(repo), 1675 dbconn.retrieveAtom(idpackage), 1676 blue(_("checking package hash")), 1677 darkgreen(os.path.basename(downloaded_path)), 1678 ), 1679 importance = 0, 1680 type = "info", 1681 header = brown(" "), 1682 back = True 1683 ) 1684 1685 md5hash = self.entropyTools.md5sum(downloaded_path) 1686 db_md5hash = dbconn.retrieveDigest(idpackage) 1687 if md5hash != db_md5hash: 1688 errors = True 1689 self.updateProgress( 1690 "[%s=>%s|%s|%s] %s: %s" % ( 1691 darkgreen(from_branch), 1692 darkred(branch), 1693 brown(repo), 1694 dbconn.retrieveAtom(idpackage), 1695 blue(_("hash does not match for")), 1696 darkgreen(os.path.basename(downloaded_path)), 1697 ), 1698 importance = 0, 1699 type = "error", 1700 header = brown(" ") 1701 ) 1702 continue 1703 1704 1705 1706 if errors: 1707 reason = _("wrong md5") 1708 if m_broken_uris: 1709 my_broken_uris = [ 1710 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 1711 for x, y in m_broken_uris] 1712 reason = my_broken_uris[0][1] 1713 1714 self.updateProgress( 1715 "[%s=>%s|%s] %s, %s: %s" % ( 1716 darkgreen(from_branch), 1717 darkred(branch), 1718 brown(repo), 1719 blue(_("download errors")), 1720 blue(_("reason")), 1721 reason, 1722 ), 1723 importance = 1, 1724 type = "error", 1725 header = darkred(" !!! ") 1726 ) 1727 # continuing if possible 1728 continue 1729 1730 all_fine = True 1731 1732 self.updateProgress( 1733 "[%s=>%s|%s] %s: %s" % ( 1734 darkgreen(from_branch), 1735 darkred(branch), 1736 brown(repo), 1737 blue(_("download completed successfully")), 1738 darkgreen(crippled_uri), 1739 ), 1740 importance = 1, 1741 type = "info", 1742 header = darkgreen(" * ") 1743 ) 1744 1745 if not all_fine: 1746 self.updateProgress( 1747 "[%s=>%s|%s] %s" % ( 1748 darkgreen(', '.join(from_branches)), 1749 darkred(branch), 1750 brown(repo), 1751 blue(_("error downloading packages from mirrors")), 1752 ), 1753 importance = 2, 1754 type = "error", 1755 header = darkred(" !!! ") 1756 ) 1757 return False 1758 1759 for from_branch in sorted(mapped_branches): 1760 1761 self.updateProgress( 1762 "[%s=>%s|%s] %s: %s" % ( 1763 darkgreen(from_branch), 1764 darkred(branch), 1765 brown(repo), 1766 blue(_("working on branch")), 1767 darkgreen(from_branch), 1768 ), 1769 importance = 1, 1770 type = "info", 1771 header = brown(" @@ ") 1772 ) 1773 1774 down_queue = download_queue[from_branch] 1775 for package_path, idpackage in down_queue: 1776 1777 self.updateProgress( 1778 "[%s=>%s|%s] %s: %s" % ( 1779 darkgreen(from_branch), 1780 darkred(branch), 1781 brown(repo), 1782 blue(_("updating package")), 1783 darkgreen(os.path.basename(package_path)), 1784 ), 1785 importance = 1, 1786 type = "info", 1787 header = brown(" "), 1788 back = True 1789 ) 1790 1791 # move files to upload 1792 package_name = os.path.basename(package_path) 1793 new_package_path = os.path.join(local_basedir, package_name) 1794 try: 1795 os.rename(package_path, new_package_path) 1796 except OSError: 1797 shutil.move(package_path, new_package_path) 1798 1799 # create md5 checksum 1800 self.entropyTools.create_md5_file(new_package_path) 1801 1802 # update database 1803 download_url = dbconn.retrieveDownloadURL(idpackage) 1804 download_url = \ 1805 self.ClientService.swap_branch_in_download_relative_uri( 1806 branch, download_url) 1807 dbconn.setDownloadURL(idpackage, download_url) 1808 dbconn.switchBranch(idpackage, branch) 1809 dbconn.commitChanges() 1810 1811 self.updateProgress( 1812 "[%s=>%s|%s] %s: %s" % ( 1813 darkgreen(from_branch), 1814 darkred(branch), 1815 brown(repo), 1816 blue(_("package flushed")), 1817 darkgreen(os.path.basename(package_path)), 1818 ), 1819 importance = 1, 1820 type = "info", 1821 header = brown(" ") 1822 ) 1823 1824 try: 1825 os.rmdir(tmp_down_dir) 1826 except OSError: 1827 pass 1828 1829 return True 1830

1831 - def move_packages(self, matches, to_repo, from_repo = None, ask = True, 1832 do_copy = False, new_tag = None, pull_deps = False):

1833 1834 if from_repo == None: 1835 from_repo = self.default_repository 1836 switched = set() 1837 1838 my_matches = list(matches) 1839 1840 # avoid setting __default__ as default server repo 1841 if etpConst['clientserverrepoid'] in (to_repo, from_repo): 1842 self.updateProgress( 1843 "%s: %s" % ( 1844 blue(_("Cannot touch system database")), 1845 red(etpConst['clientserverrepoid']), 1846 ), 1847 importance = 2, type = "warning", header = darkred(" @@ ") 1848 ) 1849 return switched 1850 1851 if not my_matches and from_repo: 1852 dbconn = self.open_server_repository(read_only = True, 1853 no_upload = True, repo = from_repo) 1854 my_matches = set( \ 1855 [(x, from_repo) for x in \ 1856 dbconn.listAllIdpackages()] 1857 ) 1858 1859 mytxt = _("Preparing to move selected packages to") 1860 if do_copy: 1861 mytxt = _("Preparing to copy selected packages to") 1862 self.updateProgress( 1863 "%s %s:" % ( 1864 blue(mytxt), 1865 red(to_repo), 1866 ), 1867 importance = 2, 1868 type = "info", 1869 header = red(" @@ ") 1870 ) 1871 self.updateProgress( 1872 "%s: %s" % ( 1873 bold(_("Note")), 1874 red(_("all old packages with conflicting scope will be " \ 1875 "removed from destination repo unless injected")), 1876 ), 1877 importance = 1, 1878 type = "info", 1879 header = red(" @@ ") 1880 ) 1881 1882 new_tag_string = '' 1883 if new_tag != None: 1884 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")), 1885 brown(new_tag),) 1886 1887 my_qa = self.QA() 1888 branch = self.SystemSettings['repositories']['branch'] 1889 pull_deps_matches = [] 1890 for idpackage, repo in my_matches: 1891 dbconn = self.open_server_repository(read_only = True, 1892 no_upload = True, repo = repo) 1893 self.updateProgress( 1894 "[%s=>%s|%s] %s " % ( 1895 darkgreen(repo), 1896 darkred(to_repo), 1897 brown(branch), 1898 blue(dbconn.retrieveAtom(idpackage)), 1899 ) + new_tag_string, 1900 importance = 0, 1901 type = "info", 1902 header = brown(" # ") 1903 ) 1904 # do we want to pull in also package dependencies? 1905 if pull_deps: 1906 dep_idpackages = my_qa.get_deep_dependency_list(dbconn, 1907 idpackage) 1908 for dep_idpackage in dep_idpackages: 1909 1910 my_dep_match = (dep_idpackage, repo,) 1911 if my_dep_match in pull_deps_matches: 1912 continue 1913 if my_dep_match in my_matches: 1914 continue 1915 1916 pull_deps_matches.append(my_dep_match) 1917 self.updateProgress( 1918 "[%s|%s] %s" % ( 1919 brown(branch), 1920 blue(_("dependency")), 1921 purple(dbconn.retrieveAtom(dep_idpackage)), 1922 ), 1923 importance = 0, 1924 type = "info", 1925 header = purple(" >> ") 1926 ) 1927 1928 if pull_deps: 1929 # put deps first! 1930 my_matches = pull_deps_matches + [x for x in my_matches if x not \ 1931 in pull_deps_matches] 1932 1933 if ask: 1934 rc_question = self.askQuestion(_("Would you like to continue ?")) 1935 if rc_question == "No": 1936 return switched 1937 1938 for idpackage, repo in my_matches: 1939 dbconn = self.open_server_repository(read_only = False, 1940 no_upload = True, repo = repo) 1941 match_branch = dbconn.retrieveBranch(idpackage) 1942 match_atom = dbconn.retrieveAtom(idpackage) 1943 package_filename = os.path.basename( 1944 dbconn.retrieveDownloadURL(idpackage)) 1945 self.updateProgress( 1946 "[%s=>%s|%s] %s: %s" % ( 1947 darkgreen(repo), 1948 darkred(to_repo), 1949 brown(branch), 1950 blue(_("switching")), 1951 darkgreen(match_atom), 1952 ), 1953 importance = 0, 1954 type = "info", 1955 header = red(" @@ "), 1956 back = True 1957 ) 1958 # move binary file 1959 from_file = os.path.join(self.get_local_packages_directory(repo), 1960 match_branch, package_filename) 1961 if not os.path.isfile(from_file): 1962 from_file = os.path.join(self.get_local_upload_directory(repo), 1963 match_branch, package_filename) 1964 if not os.path.isfile(from_file): 1965 self.updateProgress( 1966 "[%s=>%s|%s] %s: %s -> %s" % ( 1967 darkgreen(repo), 1968 darkred(to_repo), 1969 brown(branch), 1970 bold(_("cannot switch, package not found, skipping")), 1971 darkgreen(match_atom), 1972 red(from_file), 1973 ), 1974 importance = 1, 1975 type = "warning", 1976 header = darkred(" !!! ") 1977 ) 1978 continue 1979 1980 if new_tag != None: 1981 match_category = dbconn.retrieveCategory(idpackage) 1982 match_name = dbconn.retrieveName(idpackage) 1983 match_version = dbconn.retrieveVersion(idpackage) 1984 tagged_package_filename = \ 1985 self.entropyTools.create_package_filename( 1986 match_category, match_name, match_version, new_tag) 1987 to_file = os.path.join(self.get_local_upload_directory(to_repo), 1988 match_branch, tagged_package_filename) 1989 else: 1990 to_file = os.path.join(self.get_local_upload_directory(to_repo), 1991 match_branch, package_filename) 1992 if not os.path.isdir(os.path.dirname(to_file)): 1993 os.makedirs(os.path.dirname(to_file)) 1994 1995 copy_data = [ 1996 (from_file, to_file,), 1997 (from_file + etpConst['packagesmd5fileext'], 1998 to_file + etpConst['packagesmd5fileext'],), 1999 (from_file + etpConst['packagesexpirationfileext'], 2000 to_file + etpConst['packagesexpirationfileext'],) 2001 ] 2002 2003 for from_item, to_item in copy_data: 2004 self.updateProgress( 2005 "[%s=>%s|%s] %s: %s" % ( 2006 darkgreen(repo), 2007 darkred(to_repo), 2008 brown(branch), 2009 blue(_("moving file")), 2010 darkgreen(os.path.basename(from_item)), 2011 ), 2012 importance = 0, 2013 type = "info", 2014 header = red(" @@ "), 2015 back = True 2016 ) 2017 if os.path.isfile(from_item): 2018 shutil.copy2(from_item, to_item) 2019 2020 self.updateProgress( 2021 "[%s=>%s|%s] %s: %s" % ( 2022 darkgreen(repo), 2023 darkred(to_repo), 2024 brown(branch), 2025 blue(_("loading data from source database")), 2026 darkgreen(repo), 2027 ), 2028 importance = 0, 2029 type = "info", 2030 header = red(" @@ "), 2031 back = True 2032 ) 2033 # install package into destination db 2034 data = dbconn.getPackageData(idpackage) 2035 if new_tag != None: 2036 data['versiontag'] = new_tag 2037 2038 todbconn = self.open_server_repository(read_only = False, 2039 no_upload = True, repo = to_repo) 2040 2041 self.updateProgress( 2042 "[%s=>%s|%s] %s: %s" % ( 2043 darkgreen(repo), 2044 darkred(to_repo), 2045 brown(branch), 2046 blue(_("injecting data to destination database")), 2047 darkgreen(to_repo), 2048 ), 2049 importance = 0, 2050 type = "info", 2051 header = red(" @@ "), 2052 back = True 2053 ) 2054 new_idpackage, new_revision, new_data = todbconn.handlePackage(data) 2055 del data 2056 todbconn.commitChanges() 2057 2058 if not do_copy: 2059 self.updateProgress( 2060 "[%s=>%s|%s] %s: %s" % ( 2061 darkgreen(repo), 2062 darkred(to_repo), 2063 brown(branch), 2064 blue(_("removing entry from source database")), 2065 darkgreen(repo), 2066 ), 2067 importance = 0, 2068 type = "info", 2069 header = red(" @@ "), 2070 back = True 2071 ) 2072 2073 # remove package from old db 2074 dbconn.removePackage(idpackage) 2075 dbconn.commitChanges() 2076 2077 self.updateProgress( 2078 "[%s=>%s|%s] %s: %s" % ( 2079 darkgreen(repo), 2080 darkred(to_repo), 2081 brown(branch), 2082 blue(_("successfully handled atom")), 2083 darkgreen(match_atom), 2084 ), 2085 importance = 0, 2086 type = "info", 2087 header = blue(" @@ ") 2088 ) 2089 switched.add((idpackage, repo,)) 2090 2091 # just run this to make dev aware 2092 self.dependencies_test(to_repo) 2093 2094 return switched

2095 2096

2097 - def package_injector(self, package_file, inject = False, repo = None):

2098 2099 if repo == None: 2100 repo = self.default_repository 2101 2102 upload_dir = os.path.join(self.get_local_upload_directory(repo), 2103 self.SystemSettings['repositories']['branch']) 2104 if not os.path.isdir(upload_dir): 2105 os.makedirs(upload_dir) 2106 2107 dbconn = self.open_server_repository(read_only = False, 2108 no_upload = True, repo = repo) 2109 self.updateProgress( 2110 red("[repo: %s] %s: %s" % ( 2111 darkgreen(repo), 2112 _("adding package"), 2113 bold(os.path.basename(package_file)), 2114 ) 2115 ), 2116 importance = 1, 2117 type = "info", 2118 header = brown(" * "), 2119 back = True 2120 ) 2121 mydata = self.Spm().extract_package_metadata(package_file) 2122 mydata['injected'] = inject 2123 idpackage, revision, mydata = dbconn.handlePackage(mydata) 2124 2125 # set trashed counters 2126 trashing_counters = set() 2127 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2128 myserver_repos = srv_set['repositories'].keys() 2129 for myrepo in myserver_repos: 2130 mydbconn = self.open_server_repository(read_only = True, 2131 no_upload = True, repo = myrepo) 2132 mylist = mydbconn.retrieve_packages_to_remove( 2133 mydata['name'], 2134 mydata['category'], 2135 mydata['slot'], 2136 mydata['injected'] 2137 ) 2138 for myitem in mylist: 2139 trashing_counters.add(mydbconn.retrieveSpmUid(myitem)) 2140 2141 for mycounter in trashing_counters: 2142 dbconn.setTrashedUid(mycounter) 2143 2144 # add package info to our current server repository 2145 dbconn.dropInstalledPackageFromStore(idpackage) 2146 dbconn.storeInstalledPackage(idpackage, repo) 2147 atom = dbconn.retrieveAtom(idpackage) 2148 2149 self.updateProgress( 2150 "[repo:%s] %s: %s %s: %s" % ( 2151 darkgreen(repo), 2152 blue(_("added package")), 2153 darkgreen(atom), 2154 blue(_("rev")), # as in revision 2155 bold(str(revision)), 2156 ), 2157 importance = 1, 2158 type = "info", 2159 header = red(" @@ ") 2160 ) 2161 2162 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage)) 2163 if manual_deps: 2164 self.updateProgress( 2165 "[repo:%s] %s: %s" % ( 2166 darkgreen(repo), 2167 blue(_("manual dependencies for")), 2168 darkgreen(atom), 2169 ), 2170 importance = 1, 2171 type = "warning", 2172 header = darkgreen(" ## ") 2173 ) 2174 for m_dep in manual_deps: 2175 self.updateProgress( 2176 brown(m_dep), 2177 importance = 1, 2178 type = "warning", 2179 header = darkred(" # ") 2180 ) 2181 2182 download_url = self._setup_repository_package_filename(idpackage, 2183 repo = repo) 2184 downloadfile = os.path.basename(download_url) 2185 destination_path = os.path.join(upload_dir, downloadfile) 2186 try: 2187 os.rename(package_file, destination_path) 2188 except OSError: 2189 shutil.move(package_file, destination_path) 2190 2191 dbconn.commitChanges() 2192 return idpackage, destination_path

2193 2194 # this function changes the final repository package filename

2195 - def _setup_repository_package_filename(self, idpackage, repo = None):

2196 2197 dbconn = self.open_server_repository(read_only = False, 2198 no_upload = True, repo = repo) 2199 2200 downloadurl = dbconn.retrieveDownloadURL(idpackage) 2201 packagerev = dbconn.retrieveRevision(idpackage) 2202 downloaddir = os.path.dirname(downloadurl) 2203 downloadfile = os.path.basename(downloadurl) 2204 # add revision 2205 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev, 2206 etpConst['packagesext'],) 2207 downloadurl = os.path.join(downloaddir, downloadfile) 2208 2209 # update url 2210 dbconn.setDownloadURL(idpackage, downloadurl) 2211 2212 return downloadurl

2213

2214 - def add_packages_to_repository(self, packages_data, ask = True, 2215 repo = None):

2216 2217 if repo == None: 2218 repo = self.default_repository 2219 2220 mycount = 0 2221 maxcount = len(packages_data) 2222 idpackages_added = set() 2223 to_be_injected = set() 2224 my_qa = self.QA() 2225 missing_deps_taint = False 2226 for package_filepath, inject in packages_data: 2227 2228 mycount += 1 2229 self.updateProgress( 2230 "[repo:%s] %s: %s" % ( 2231 darkgreen(repo), 2232 blue(_("adding package")), 2233 darkgreen(os.path.basename(package_filepath)), 2234 ), 2235 importance = 1, 2236 type = "info", 2237 header = blue(" @@ "), 2238 count = (mycount, maxcount,) 2239 ) 2240 2241 try: 2242 # add to database 2243 idpackage, destination_path = self.package_injector( 2244 package_filepath, 2245 inject = inject, 2246 repo = repo 2247 ) 2248 idpackages_added.add(idpackage) 2249 to_be_injected.add((idpackage, destination_path)) 2250 except Exception, err: 2251 self.entropyTools.print_traceback() 2252 self.updateProgress( 2253 "[repo:%s] %s: %s" % ( 2254 darkgreen(repo), 2255 darkred(_("Exception caught, closing tasks")), 2256 darkgreen(unicode(err)), 2257 ), 2258 importance = 1, 2259 type = "error", 2260 header = bold(" !!! "), 2261 count = (mycount, maxcount,) 2262 ) 2263 # reinit depends table 2264 self.depends_table_initialize(repo) 2265 # reinit librarypathsidpackage table 2266 self.library_paths_table_initialize(repo) 2267 if idpackages_added: 2268 dbconn = self.open_server_repository(read_only = False, 2269 no_upload = True, repo = repo) 2270 missing_deps_taint = my_qa.test_missing_dependencies( 2271 idpackages_added, 2272 dbconn, 2273 ask = ask, 2274 repo = repo, 2275 self_check = True, 2276 black_list = \ 2277 self.get_missing_dependencies_blacklist( 2278 repo = repo), 2279 black_list_adder = \ 2280 self.add_missing_dependencies_blacklist_items 2281 ) 2282 my_qa.test_depends_linking(idpackages_added, dbconn, 2283 repo = repo) 2284 if to_be_injected: 2285 self.inject_database_into_packages(to_be_injected, 2286 repo = repo) 2287 # reinit depends table 2288 if missing_deps_taint: 2289 self.depends_table_initialize(repo) 2290 self.close_server_databases() 2291 raise 2292 2293 # reinit depends table 2294 self.depends_table_initialize(repo) 2295 # reinit librarypathsidpackage table 2296 self.library_paths_table_initialize(repo) 2297 2298 if idpackages_added: 2299 dbconn = self.open_server_repository(read_only = False, 2300 no_upload = True, repo = repo) 2301 missing_deps_taint = my_qa.test_missing_dependencies( 2302 idpackages_added, 2303 dbconn, 2304 ask = ask, 2305 repo = repo, 2306 self_check = True, 2307 black_list = \ 2308 self.get_missing_dependencies_blacklist(repo = repo), 2309 black_list_adder = \ 2310 self.add_missing_dependencies_blacklist_items 2311 ) 2312 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo) 2313 2314 # reinit depends table 2315 if missing_deps_taint: 2316 self.depends_table_initialize(repo) 2317 2318 # inject database into packages 2319 self.inject_database_into_packages(to_be_injected, repo = repo) 2320 2321 return idpackages_added

2322 2323

2324 - def inject_database_into_packages(self, injection_data, repo = None):

2325 2326 if repo == None: 2327 repo = self.default_repository 2328 2329 # now inject metadata into tbz2 packages 2330 self.updateProgress( 2331 "[repo:%s] %s:" % ( 2332 darkgreen(repo), 2333 blue(_("Injecting entropy metadata into built packages")), 2334 ), 2335 importance = 1, 2336 type = "info", 2337 header = red(" @@ ") 2338 ) 2339 2340 dbconn = self.open_server_repository(read_only = False, 2341 no_upload = True, repo = repo) 2342 for idpackage, package_path in injection_data: 2343 self.updateProgress( 2344 "[repo:%s|%s] %s: %s" % ( 2345 darkgreen(repo), 2346 brown(str(idpackage)), 2347 blue(_("injecting entropy metadata")), 2348 darkgreen(os.path.basename(package_path)), 2349 ), 2350 importance = 1, 2351 type = "info", 2352 header = blue(" @@ "), 2353 back = True 2354 ) 2355 data = dbconn.getPackageData(idpackage) 2356 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2357 dbpath = self.ClientService.inject_entropy_database_into_package( 2358 package_path, data, treeupdates_actions) 2359 digest = self.entropyTools.md5sum(package_path) 2360 # update digest 2361 dbconn.setDigest(idpackage, digest) 2362 # update signatures 2363 signatures = data['signatures'].copy() 2364 for hash_key in sorted(signatures): 2365 hash_func = getattr(self.entropyTools, hash_key) 2366 signatures[hash_key] = hash_func(package_path) 2367 dbconn.setSignatures(idpackage, signatures['sha1'], 2368 signatures['sha256'], signatures['sha512']) 2369 self.entropyTools.create_md5_file(package_path) 2370 # remove garbage 2371 os.remove(dbpath) 2372 self.updateProgress( 2373 "[repo:%s|%s] %s: %s" % ( 2374 darkgreen(repo), 2375 brown(str(idpackage)), 2376 blue(_("injection complete")), 2377 darkgreen(os.path.basename(package_path)), 2378 ), 2379 importance = 1, 2380 type = "info", 2381 header = red(" @@ ") 2382 ) 2383 dbconn.commitChanges()

2384

2385 - def check_config_file_updates(self):

2386 self.updateProgress( 2387 "[%s] %s" % ( 2388 red(_("config files")), # something short please 2389 blue(_("checking system")), 2390 ), 2391 importance = 1, 2392 type = "info", 2393 header = blue(" @@ "), 2394 back = True 2395 ) 2396 # scanning for config files not updated 2397 scandata = self.ClientService.FileUpdates.scanfs(dcache = False) 2398 if scandata: 2399 self.updateProgress( 2400 "[%s] %s" % ( 2401 red(_("config files")), # something short please 2402 blue(_("there are configuration files not updated yet")), 2403 ), 2404 importance = 1, 2405 type = "error", 2406 header = darkred(" @@ ") 2407 ) 2408 for key in scandata: 2409 self.updateProgress( 2410 "%s" % (brown(etpConst['systemroot'] + \ 2411 scandata[key]['destination'])), 2412 importance = 1, 2413 type = "info", 2414 header = "\t" 2415 ) 2416 return True 2417 return False

2418

2419 - def quickpkg(self, atom, storedir):

2420 return self.Spm().generate_package(atom, storedir)

2421

2422 - def remove_packages(self, idpackages, repo = None):

2423 2424 if repo == None: 2425 repo = self.default_repository 2426 2427 dbconn = self.open_server_repository(read_only = False, 2428 no_upload = True, repo = repo) 2429 for idpackage in idpackages: 2430 atom = dbconn.retrieveAtom(idpackage) 2431 self.updateProgress( 2432 "[repo:%s] %s: %s" % ( 2433 darkgreen(repo), 2434 blue(_("removing package")), 2435 darkgreen(atom), 2436 ), 2437 importance = 1, 2438 type = "info", 2439 header = brown(" @@ ") 2440 ) 2441 dbconn.removePackage(idpackage) 2442 self.close_server_database(dbconn) 2443 self.updateProgress( 2444 "[repo:%s] %s" % ( 2445 darkgreen(repo), 2446 blue(_("removal complete")), 2447 ), 2448 importance = 1, 2449 type = "info", 2450 header = brown(" @@ ") 2451 )

2452 2453

2454 - def bump_database(self, repo = None):

2455 dbconn = self.open_server_repository(read_only = False, 2456 no_upload = True, repo = repo) 2457 dbconn.taintDatabase() 2458 self.close_server_database(dbconn)

2459

2460 - def get_remote_mirrors(self, repo = None):

2461 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2462 if repo == None: 2463 repo = self.default_repository 2464 return srv_set['repositories'][repo]['mirrors'][:]

2465

2466 - def get_remote_packages_relative_path(self, repo = None):

2467 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2468 if repo == None: 2469 repo = self.default_repository 2470 return srv_set['repositories'][repo]['packages_relative_path']

2471

2472 - def get_remote_database_relative_path(self, repo = None):

2473 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2474 if repo == None: 2475 repo = self.default_repository 2476 return srv_set['repositories'][repo]['database_relative_path']

2477

2478 - def get_local_database_file(self, repo = None, branch = None):

2479 if repo == None: 2480 repo = self.default_repository 2481 return os.path.join(self.get_local_database_dir(repo, branch), 2482 etpConst['etpdatabasefile'])

2483

2484 - def get_local_store_directory(self, repo = None):

2485 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2486 if repo == None: 2487 repo = self.default_repository 2488 return srv_set['repositories'][repo]['store_dir']

2489

2490 - def get_local_upload_directory(self, repo = None):

2491 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2492 if repo == None: 2493 repo = self.default_repository 2494 return srv_set['repositories'][repo]['upload_dir']

2495

2496 - def get_local_packages_directory(self, repo = None):

2497 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2498 if repo == None: 2499 repo = self.default_repository 2500 return srv_set['repositories'][repo]['packages_dir']

2501

2502 - def get_local_database_taint_file(self, repo = None, branch = None):

2503 if repo == None: 2504 repo = self.default_repository 2505 return os.path.join(self.get_local_database_dir(repo, branch), 2506 etpConst['etpdatabasetaintfile'])

2507

2508 - def get_local_database_revision_file(self, repo = None, branch = None):

2509 if repo == None: 2510 repo = self.default_repository 2511 return os.path.join(self.get_local_database_dir(repo, branch), 2512 etpConst['etpdatabaserevisionfile'])

2513

2514 - def get_local_database_timestamp_file(self, repo = None, branch = None):

2515 if repo == None: 2516 repo = self.default_repository 2517 return os.path.join(self.get_local_database_dir(repo, branch), 2518 etpConst['etpdatabasetimestampfile'])

2519

2520 - def get_local_database_ca_cert_file(self, repo = None, branch = None):

2521 if repo == None: 2522 repo = self.default_repository 2523 return os.path.join(self.get_local_database_dir(repo, branch), 2524 etpConst['etpdatabasecacertfile'])

2525

2526 - def get_local_database_server_cert_file(self, repo = None, branch = None):

2527 if repo == None: 2528 repo = self.default_repository 2529 return os.path.join(self.get_local_database_dir(repo, branch), 2530 etpConst['etpdatabaseservercertfile'])

2531

2532 - def get_local_database_mask_file(self, repo = None, branch = None):

2533 if repo == None: 2534 repo = self.default_repository 2535 return os.path.join(self.get_local_database_dir(repo, branch), 2536 etpConst['etpdatabasemaskfile'])

2537

2538 - def get_local_database_system_mask_file(self, repo = None, branch = None):

2539 if repo == None: 2540 repo = self.default_repository 2541 return os.path.join(self.get_local_database_dir(repo, branch), 2542 etpConst['etpdatabasesytemmaskfile'])

2543

2544 - def get_local_database_confl_tagged_file(self, repo = None, branch = None):

2545 if repo == None: 2546 repo = self.default_repository 2547 return os.path.join(self.get_local_database_dir(repo, branch), 2548 etpConst['etpdatabaseconflictingtaggedfile'])

2549

2550 - def get_local_database_licensewhitelist_file(self, repo = None, 2551 branch = None):

2552 2553 if repo == None: 2554 repo = self.default_repository 2555 return os.path.join(self.get_local_database_dir(repo, branch), 2556 etpConst['etpdatabaselicwhitelistfile'])

2557

2558 - def get_local_database_rss_file(self, repo = None, branch = None):

2559 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2560 if repo == None: 2561 repo = self.default_repository 2562 return os.path.join(self.get_local_database_dir(repo, branch), 2563 srv_set['rss']['name'])

2564

2565 - def get_local_database_rsslight_file(self, repo = None, branch = None):

2566 if repo == None: 2567 repo = self.default_repository 2568 return os.path.join(self.get_local_database_dir(repo, branch), 2569 etpConst['rss-light-name'])

2570

2571 - def get_local_database_notice_board_file(self, repo = None, branch = None):

2572 if repo == None: 2573 repo = self.default_repository 2574 return os.path.join(self.get_local_database_dir(repo, branch), 2575 etpConst['rss-notice-board'])

2576

2577 - def get_local_database_treeupdates_file(self, repo = None, branch = None):

2578 if repo == None: 2579 repo = self.default_repository 2580 return os.path.join(self.get_local_database_dir(repo, branch), 2581 etpConst['etpdatabaseupdatefile'])

2582

2583 - def get_local_database_compressed_metafiles_file(self, repo = None, 2584 branch = None):

2585 2586 if repo == None: 2587 repo = self.default_repository 2588 return os.path.join(self.get_local_database_dir(repo, branch), 2589 etpConst['etpdatabasemetafilesfile'])

2590

2591 - def get_local_database_metafiles_not_found_file(self, repo = None, 2592 branch = None):

2593 2594 if repo == None: 2595 repo = self.default_repository 2596 return os.path.join(self.get_local_database_dir(repo, branch), 2597 etpConst['etpdatabasemetafilesnotfound'])

2598

2599 - def get_local_exp_based_pkgs_rm_whitelist_file(self, repo = None, 2600 branch = None):

2601 if repo == None: 2602 repo = self.default_repository 2603 return os.path.join(self.get_local_database_dir(repo, branch), 2604 etpConst['etpdatabaseexpbasedpkgsrm'])

2605

2606 - def get_local_pkglist_file(self, repo = None, branch = None):

2607 if repo == None: 2608 repo = self.default_repository 2609 return os.path.join(self.get_local_database_dir(repo, branch), 2610 etpConst['etpdatabasepkglist'])

2611

2612 - def get_local_database_sets_dir(self, repo = None, branch = None):

2613 if repo == None: 2614 repo = self.default_repository 2615 return os.path.join(self.get_local_database_dir(repo, branch), 2616 etpConst['confsetsdirname'])

2617

2618 - def get_local_post_branch_mig_script(self, repo = None, branch = None):

2619 if repo == None: 2620 repo = self.default_repository 2621 return os.path.join(self.get_local_database_dir(repo, branch), 2622 etpConst['etp_post_branch_hop_script'])

2623

2624 - def get_local_post_branch_upg_script(self, repo = None, branch = None):

2625 if repo == None: 2626 repo = self.default_repository 2627 return os.path.join(self.get_local_database_dir(repo, branch), 2628 etpConst['etp_post_branch_upgrade_script'])

2629

2630 - def get_local_critical_updates_file(self, repo = None, branch = None):

2631 if repo == None: 2632 repo = self.default_repository 2633 return os.path.join(self.get_local_database_dir(repo, branch), 2634 etpConst['etpdatabasecriticalfile'])

2635

2636 - def get_local_database_keywords_file(self, repo = None, branch = None):

2637 if repo == None: 2638 repo = self.default_repository 2639 return os.path.join(self.get_local_database_dir(repo, branch), 2640 etpConst['etpdatabasekeywordsfile'])

2641

2642 - def get_local_database_dir(self, repo = None, branch = None):

2643 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2644 if repo == None: 2645 repo = self.default_repository 2646 if branch == None: 2647 branch = self.SystemSettings['repositories']['branch'] 2648 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2649 branch)

2650

2651 - def get_missing_dependencies_blacklist_file(self, repo = None, 2652 branch = None):

2653 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2654 if repo == None: 2655 repo = self.default_repository 2656 if branch == None: 2657 branch = self.SystemSettings['repositories']['branch'] 2658 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2659 branch, etpConst['etpdatabasemissingdepsblfile'])

2660

2661 - def get_missing_dependencies_blacklist(self, repo = None, branch = None):

2662 if repo == None: 2663 repo = self.default_repository 2664 if branch == None: 2665 branch = self.SystemSettings['repositories']['branch'] 2666 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2667 wl_data = [] 2668 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK): 2669 f_wl = open(wl_file,"r") 2670 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \ 2671 not x.strip().startswith("#")] 2672 f_wl.close() 2673 return set(wl_data)

2674

2675 - def add_missing_dependencies_blacklist_items(self, items, repo = None, 2676 branch = None):

2677 2678 if repo == None: 2679 repo = self.default_repository 2680 if branch == None: 2681 branch = self.SystemSettings['repositories']['branch'] 2682 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2683 wl_dir = os.path.dirname(wl_file) 2684 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)): 2685 return 2686 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK): 2687 return 2688 f_wl = open(wl_file,"a+") 2689 f_wl.write('\n'.join(items)+'\n') 2690 f_wl.flush() 2691 f_wl.close()

2692

2693 - def get_local_database_revision(self, repo = None):

2694 2695 if repo == None: 2696 repo = self.default_repository 2697 2698 dbrev_file = self.get_local_database_revision_file(repo) 2699 if os.path.isfile(dbrev_file): 2700 f_rev = open(dbrev_file) 2701 rev = f_rev.readline().strip() 2702 f_rev.close() 2703 try: 2704 rev = int(rev) 2705 except ValueError: 2706 self.updateProgress( 2707 "[repo:%s] %s: %s - %s" % ( 2708 darkgreen(repo), 2709 blue(_("invalid database revision")), 2710 bold(rev), 2711 blue(_("defaulting to 0")), 2712 ), 2713 importance = 2, 2714 type = "error", 2715 header = darkred(" !!! ") 2716 ) 2717 rev = 0 2718 return rev 2719 else: 2720 return 0

2721

2722 - def get_remote_database_revision(self, repo = None):

2723 2724 if repo == None: 2725 repo = self.default_repository 2726 2727 remote_status = self.MirrorsService.get_remote_databases_status(repo) 2728 if not [x for x in remote_status if x[1]]: 2729 remote_revision = 0 2730 else: 2731 remote_revision = max([x[1] for x in remote_status]) 2732 2733 return remote_revision

2734

2735 - def get_branch_from_download_relative_uri(self, mypath):

2736 return self.ClientService.get_branch_from_download_relative_uri(mypath)

2737

2738 - def get_current_timestamp(self):

2739 from datetime import datetime 2740 import time 2741 return "%s" % (datetime.fromtimestamp(time.time()),)

2742

2743 - def create_repository_pkglist(self, repo = None, branch = None):

2744 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch) 2745 2746 tmp_pkglist_file = pkglist_file + ".tmp" 2747 dbconn = self.open_server_repository(repo = repo, just_reading = True, 2748 do_treeupdates = False) 2749 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True) 2750 2751 with open(tmp_pkglist_file, "w") as pkg_f: 2752 for pkg in pkglist: 2753 pkg_f.write(pkg + "\n") 2754 pkg_f.flush() 2755 2756 os.rename(tmp_pkglist_file, pkglist_file)

2757

2758 - def package_set_list(self, *args, **kwargs):

2759 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2760 repos = srv_set['repositories'].keys() 2761 kwargs['server_repos'] = repos 2762 kwargs['serverInstance'] = self 2763 return self.ClientService.package_set_list(*args, **kwargs)

2764

2765 - def package_set_search(self, *args, **kwargs):

2766 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2767 repos = srv_set['repositories'].keys() 2768 kwargs['server_repos'] = repos 2769 kwargs['serverInstance'] = self 2770 return self.ClientService.package_set_search(*args, **kwargs)

2771

2772 - def package_set_match(self, *args, **kwargs):

2773 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2774 repos = srv_set['repositories'].keys() 2775 kwargs['server_repos'] = repos 2776 kwargs['serverInstance'] = self 2777 return self.ClientService.package_set_match(*args, **kwargs)

2778

2779 - def atom_match(self, *args, **kwargs):

2780 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2781 repos = srv_set['repositories'].keys() 2782 kwargs['server_repos'] = repos 2783 kwargs['serverInstance'] = self 2784 return self.ClientService.atom_match(*args, **kwargs)

2785

2786 - def scan_package_changes(self):

2787 2788 spm = self.Spm() 2789 2790 spm_packages = spm.get_installed_packages() 2791 installed_packages = [] 2792 for spm_package in spm_packages: 2793 pkg_counter = spm.get_installed_package_metadata(spm_package, 2794 "COUNTER") 2795 try: 2796 pkg_counter = int(pkg_counter) 2797 except ValueError: 2798 continue 2799 installed_packages.append((spm_package, pkg_counter,)) 2800 2801 installed_counters = set() 2802 to_be_added = set() 2803 to_be_removed = set() 2804 to_be_injected = set() 2805 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2806 exp_based_scope = my_settings['exp_based_scope'] 2807 2808 server_repos = my_settings['repositories'].keys() 2809 2810 # packages to be added 2811 for spm_atom, spm_counter in installed_packages: 2812 found = False 2813 for server_repo in server_repos: 2814 installed_counters.add(spm_counter) 2815 server_dbconn = self.open_server_repository(read_only = True, 2816 no_upload = True, repo = server_repo) 2817 counter = server_dbconn.isSpmUidAvailable(spm_counter) 2818 if counter: 2819 found = True 2820 break 2821 if not found: 2822 to_be_added.add((spm_atom, spm_counter,)) 2823 2824 # packages to be removed from the database 2825 database_counters = {} 2826 for server_repo in server_repos: 2827 server_dbconn = self.open_server_repository(read_only = True, 2828 no_upload = True, repo = server_repo) 2829 database_counters[server_repo] = server_dbconn.listAllSpmUids() 2830 2831 ordered_counters = set() 2832 for server_repo in database_counters: 2833 for data in database_counters[server_repo]: 2834 ordered_counters.add((data, server_repo)) 2835 database_counters = ordered_counters 2836 2837 for (counter, idpackage,), xrepo in database_counters: 2838 2839 if counter < 0: 2840 continue # skip packages without valid counter 2841 2842 if counter in installed_counters: 2843 continue 2844 2845 dbconn = self.open_server_repository(read_only = True, 2846 no_upload = True, repo = xrepo) 2847 2848 dorm = True 2849 # check if the package is in to_be_added 2850 if to_be_added: 2851 2852 dorm = False 2853 atom = dbconn.retrieveAtom(idpackage) 2854 atomkey = self.entropyTools.dep_getkey(atom) 2855 atomtag = self.entropyTools.dep_gettag(atom) 2856 atomslot = dbconn.retrieveSlot(idpackage) 2857 2858 add = True 2859 for spm_atom, spm_counter in to_be_added: 2860 addslot = self.Spm().get_installed_package_metadata( 2861 spm_atom, "SLOT") 2862 addkey = self.entropyTools.dep_getkey(spm_atom) 2863 # workaround for ebuilds not having slot 2864 if addslot == None: 2865 addslot = '0' 2866 # atomtag != None is for handling tagged pkgs correctly 2867 if (atomkey == addkey) and \ 2868 ((str(atomslot) == str(addslot)) or (atomtag != None)): 2869 # do not add to to_be_removed 2870 add = False 2871 break 2872 2873 if not add: 2874 continue 2875 dorm = True 2876 2877 # checking if we are allowed to remove stuff on this repo 2878 # it xrepo is not the default one, we MUST skip this to 2879 # avoid touching what developer doesn't expect 2880 if dorm and (xrepo == self.default_repository): 2881 trashed = self.is_counter_trashed(counter) 2882 if trashed: 2883 # search into portage then 2884 try: 2885 key, slot = dbconn.retrieveKeySlot(idpackage) 2886 trashed = self.Spm().match_installed_package( 2887 key+":"+slot) 2888 except TypeError: # referred to retrieveKeySlot 2889 trashed = True 2890 if not trashed: 2891 2892 dbtag = dbconn.retrieveVersionTag(idpackage) 2893 if dbtag: 2894 is_injected = dbconn.isInjected(idpackage) 2895 if not is_injected: 2896 to_be_injected.add((idpackage, xrepo)) 2897 2898 elif exp_based_scope: 2899 2900 # check if support for this is set 2901 plg_id = self.sys_settings_fatscope_plugin_id 2902 exp_data = self.SystemSettings[plg_id]['repos'].get( 2903 xrepo, set()) 2904 2905 # only some packages are set, check if our is 2906 # in the list 2907 if (idpackage not in exp_data) and (-1 not in exp_data): 2908 to_be_removed.add((idpackage, xrepo)) 2909 continue 2910 2911 idpackage_expired = self.is_match_expired((idpackage, 2912 xrepo,)) 2913 2914 if idpackage_expired: 2915 # expired !!! 2916 # add this and its depends (reverse deps) 2917 to_be_removed.add((idpackage, xrepo)) 2918 for my_id in dbconn.retrieveReverseDependencies( 2919 idpackage): 2920 to_be_removed.add((my_id, xrepo)) 2921 2922 else: 2923 to_be_removed.add((idpackage, xrepo)) 2924 2925 return to_be_added, to_be_removed, to_be_injected

2926

2927 - def is_match_expired(self, match):

2928 2929 idpackage, repoid = match 2930 dbconn = self.open_server_repository(repo = repoid, just_reading = True) 2931 # 3600 * 24 = 86400 2932 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2933 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400 2934 cur_unix_time = self.entropyTools.get_current_unix_time() 2935 # if packages removal is triggered by expiration 2936 # we will have to check if our package is really 2937 # expired and remove its reverse deps too 2938 mydate = dbconn.retrieveCreationDate(idpackage) 2939 # cross fingers hoping that time is set correctly 2940 mydelta = cur_unix_time - float(mydate) 2941 if mydelta > pkg_exp_secs: 2942 return True 2943 return False

2944

2945 - def is_counter_trashed(self, counter):

2946 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2947 server_repos = srv_set['repositories'].keys() 2948 for repo in server_repos: 2949 dbconn = self.open_server_repository(read_only = True, 2950 no_upload = True, repo = repo) 2951 if dbconn.isSpmUidTrashed(counter): 2952 return True 2953 return False

2954

2955 - def transform_package_into_injected(self, idpackage, repo = None):

2956 dbconn = self.open_server_repository(read_only = False, 2957 no_upload = True, repo = repo) 2958 counter = dbconn.getNewNegativeSpmUid() 2959 dbconn.setSpmUid(idpackage, counter) 2960 dbconn.setInjected(idpackage)

2961

2962 - def initialize_server_database(self, empty = True, repo = None, 2963 warnings = True):

2964 2965 if repo == None: 2966 repo = self.default_repository 2967 2968 self.close_server_databases() 2969 revisions_match = {} 2970 treeupdates_actions = [] 2971 injected_packages = set() 2972 idpackages = set() 2973 idpackages_added = set() 2974 2975 mytxt = red("%s ...") % (_("Initializing Entropy database"),) 2976 self.updateProgress( 2977 mytxt, importance = 1, 2978 type = "info", header = darkgreen(" * "), 2979 back = True 2980 ) 2981 2982 if os.path.isfile(self.get_local_database_file(repo)): 2983 2984 dbconn = self.open_server_repository(read_only = True, 2985 no_upload = True, repo = repo, warnings = warnings) 2986 2987 try: 2988 dbconn.validateDatabase() 2989 idpackages = dbconn.listAllIdpackages() 2990 except SystemDatabaseError: 2991 pass 2992 2993 try: 2994 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2995 except dbconn.dbapi2.Error: 2996 pass 2997 2998 # save list of injected packages 2999 try: 3000 injected_packages = dbconn.listAllInjectedPackages( 3001 just_files = True) 3002 injected_packages = set([os.path.basename(x) for x \ 3003 in injected_packages]) 3004 except dbconn.dbapi2.Error: 3005 pass 3006 3007 for idpackage in idpackages: 3008 url = dbconn.retrieveDownloadURL(idpackage) 3009 package = os.path.basename(url) 3010 branch = dbconn.retrieveBranch(idpackage) 3011 revision = dbconn.retrieveRevision(idpackage) 3012 revisions_match[package] = (branch, revision,) 3013 3014 self.close_server_database(dbconn) 3015 3016 mytxt = "%s: %s: %s" % ( 3017 bold(_("WARNING")), 3018 red(_("database already exists")), 3019 self.get_local_database_file(repo), 3020 ) 3021 self.updateProgress( 3022 mytxt, 3023 importance = 1, 3024 type = "warning", 3025 header = darkred(" !!! ") 3026 ) 3027 3028 rc_question = self.askQuestion(_("Do you want to continue ?")) 3029 if rc_question == "No": 3030 return 3031 try: 3032 os.remove(self.get_local_database_file(repo)) 3033 except OSError: 3034 pass 3035 3036 3037 # initialize 3038 dbconn = self.open_server_repository(read_only = False, 3039 no_upload = True, repo = repo, is_new = True) 3040 dbconn.initializeDatabase() 3041 3042 if not empty: 3043 3044 revisions_file = "/entropy-revisions-dump.txt" 3045 # dump revisions - as a backup 3046 if revisions_match: 3047 self.updateProgress( 3048 "%s: %s" % ( 3049 red(_("Dumping current revisions to file")), 3050 darkgreen(revisions_file), 3051 ), 3052 importance = 1, 3053 type = "info", 3054 header = darkgreen(" * ") 3055 ) 3056 f_rev = open(revisions_file,"w") 3057 f_rev.write(str(revisions_match)) 3058 f_rev.flush() 3059 f_rev.close() 3060 3061 # dump treeupdates - as a backup 3062 treeupdates_file = "/entropy-treeupdates-dump.txt" 3063 if treeupdates_actions: 3064 self.updateProgress( 3065 "%s: %s" % ( 3066 # do not translate treeupdates 3067 red(_("Dumping current 'treeupdates' actions to file")), 3068 bold(treeupdates_file), 3069 ), 3070 importance = 1, 3071 type = "info", 3072 header = darkgreen(" * ") 3073 ) 3074 f_tree = open(treeupdates_file,"w") 3075 f_tree.write(str(treeupdates_actions)) 3076 f_tree.flush() 3077 f_tree.close() 3078 3079 rc_question = self.askQuestion( 3080 _("Would you like to sync packages first (important!) ?")) 3081 if rc_question == "Yes": 3082 self.MirrorsService.sync_packages(repo = repo) 3083 3084 # fill tree updates actions 3085 if treeupdates_actions: 3086 dbconn.bumpTreeUpdatesActions(treeupdates_actions) 3087 3088 # now fill the database 3089 pkg_branch_dir = os.path.join( 3090 self.get_local_packages_directory(repo), 3091 self.SystemSettings['repositories']['branch']) 3092 pkglist = os.listdir(pkg_branch_dir) 3093 # filter .md5 and .expired packages 3094 pkg_ext_len = len(etpConst['packagesext']) 3095 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \ 3096 etpConst['packagesext']) and not \ 3097 os.path.isfile(os.path.join(pkg_branch_dir, 3098 x + etpConst['packagesexpirationfileext']))] 3099 3100 if pkglist: 3101 self.updateProgress( 3102 "%s '%s' %s %s" % ( 3103 red(_("Reinitializing Entropy database for branch")), 3104 bold(self.SystemSettings['repositories']['branch']), 3105 red(_("using Packages in the repository")), 3106 red("..."), 3107 ), 3108 importance = 1, 3109 type = "info", 3110 header = darkgreen(" * ") 3111 ) 3112 3113 counter = 0 3114 maxcount = len(pkglist) 3115 branch = self.SystemSettings['repositories']['branch'] 3116 for pkg in pkglist: 3117 counter += 1 3118 3119 self.updateProgress( 3120 "[repo:%s|%s] %s: %s" % ( 3121 darkgreen(repo), 3122 brown(branch), 3123 blue(_("analyzing")), 3124 bold(pkg), 3125 ), 3126 importance = 1, 3127 type = "info", 3128 header = " ", 3129 back = True, 3130 count = (counter, maxcount,) 3131 ) 3132 3133 doinject = False 3134 if pkg in injected_packages: 3135 doinject = True 3136 3137 pkg_path = os.path.join(self.get_local_packages_directory(repo), 3138 branch, pkg) 3139 mydata = self.Spm().extract_package_metadata(pkg_path) 3140 mydata['injected'] = doinject 3141 3142 # get previous revision 3143 revision_avail = revisions_match.get(pkg) 3144 add_revision = 0 3145 if (revision_avail != None): 3146 if branch == revision_avail[0]: 3147 add_revision = revision_avail[1] 3148 3149 idpackage, revision, mydata_upd = dbconn.addPackage(mydata, 3150 revision = add_revision) 3151 idpackages_added.add(idpackage) 3152 3153 self.updateProgress( 3154 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % ( 3155 repo, 3156 brown(branch), 3157 darkgreen(str(counter)), 3158 blue(str(maxcount)), 3159 red(_("added package")), 3160 darkgreen(pkg), 3161 red(_("revision")), 3162 brown(str(revision)), 3163 ), 3164 importance = 1, 3165 type = "info", 3166 header = " ", 3167 back = True 3168 ) 3169 3170 self.depends_table_initialize(repo) 3171 self.library_paths_table_initialize(repo) 3172 3173 my_qa = self.QA() 3174 3175 if idpackages_added: 3176 dbconn = self.open_server_repository(read_only = False, 3177 no_upload = True, repo = repo) 3178 my_qa.test_missing_dependencies( 3179 idpackages_added, dbconn, ask = True, 3180 repo = repo, self_check = True, 3181 black_list = \ 3182 self.get_missing_dependencies_blacklist(repo = repo), 3183 black_list_adder = \ 3184 self.add_missing_dependencies_blacklist_items 3185 ) 3186 3187 dbconn.commitChanges() 3188 self.close_server_databases() 3189 3190 return 0

3191

3192 - def match_packages(self, packages, repo = None):

3193 3194 dbconn = self.open_server_repository(read_only = True, 3195 no_upload = True, repo = repo) 3196 if ("world" in packages) or not packages: 3197 return dbconn.listAllIdpackages(), True 3198 else: 3199 idpackages = set() 3200 for package in packages: 3201 matches = dbconn.atomMatch(package, multiMatch = True) 3202 if matches[1] == 0: 3203 idpackages |= matches[0] 3204 else: 3205 mytxt = "%s: %s: %s" % ( 3206 red(_("Attention")), 3207 blue(_("cannot match")), 3208 bold(package), 3209 ) 3210 self.updateProgress( 3211 mytxt, 3212 importance = 1, 3213 type = "warning", 3214 header = darkred(" !!! ") 3215 ) 3216 return idpackages, False

3217

3218 - def get_remote_package_checksum(self, repo, filename, branch):

3219 3220 import urllib2 3221 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3222 if not srv_set['repositories'][repo].has_key('handler'): 3223 return None 3224 url = srv_set['repositories'][repo]['handler'] 3225 3226 # does the package has "#" (== tag) ? hackish thing that works 3227 filename = filename.replace("#","%23") 3228 # "+" 3229 filename = filename.replace("+","%2b") 3230 request = os.path.join(url, etpConst['handlers']['md5sum']) 3231 request += filename+"&branch="+branch 3232 3233 proxy_settings = self.SystemSettings['system']['proxy'] 3234 try: 3235 mydict = {} 3236 if proxy_settings['ftp']: 3237 mydict['ftp'] = proxy_settings['ftp'] 3238 if proxy_settings['http']: 3239 mydict['http'] = proxy_settings['http'] 3240 if mydict: 3241 mydict['username'] = proxy_settings['username'] 3242 mydict['password'] = proxy_settings['password'] 3243 self.entropyTools.add_proxy_opener(urllib2, mydict) 3244 else: 3245 # unset 3246 urllib2._opener = None 3247 item = urllib2.urlopen(request) 3248 result = item.readline().strip() 3249 item.close() 3250 del item 3251 return result 3252 except (urllib2.URLError, urllib2.HTTPError,): # no HTTP support? 3253 return None

3254

3255 - def verify_remote_packages(self, packages, ask = True, repo = None):

3256 3257 if repo == None: 3258 repo = self.default_repository 3259 3260 self.updateProgress( 3261 "[%s] %s:" % ( 3262 red("remote"), 3263 blue(_("Integrity verification of the selected packages")), 3264 ), 3265 importance = 1, 3266 type = "info", 3267 header = blue(" @@ ") 3268 ) 3269 3270 idpackages, world = self.match_packages(packages) 3271 dbconn = self.open_server_repository(read_only = True, no_upload = True, 3272 repo = repo) 3273 branch = self.SystemSettings['repositories']['branch'] 3274 3275 if world: 3276 self.updateProgress( 3277 blue( 3278 _("All the packages in repository will be checked.")), 3279 importance = 1, 3280 type = "info", 3281 header = " " 3282 ) 3283 else: 3284 mytxt = red("%s:") % ( 3285 _("This is the list of the packages that would be checked"),) 3286 self.updateProgress( 3287 mytxt, 3288 importance = 1, 3289 type = "info", 3290 header = " " 3291 ) 3292 for idpackage in idpackages: 3293 pkgatom = dbconn.retrieveAtom(idpackage) 3294 down_url = dbconn.retrieveDownloadURL(idpackage) 3295 pkgfile = os.path.basename(down_url) 3296 self.updateProgress( 3297 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)), 3298 importance = 1, 3299 type = "info", 3300 header = darkgreen(" - ") 3301 ) 3302 3303 if ask: 3304 rc_question = self.askQuestion( 3305 _("Would you like to continue ?")) 3306 if rc_question == "No": 3307 return set(), set(), {} 3308 3309 match = set() 3310 not_match = set() 3311 broken_packages = {} 3312 3313 for uri in self.get_remote_mirrors(repo): 3314 3315 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3316 self.updateProgress( 3317 "[repo:%s] %s: %s" % ( 3318 darkgreen(repo), 3319 blue(_("Working on mirror")), 3320 brown(crippled_uri), 3321 ), 3322 importance = 1, 3323 type = "info", 3324 header = red(" @@ ") 3325 ) 3326 3327 3328 totalcounter = len(idpackages) 3329 currentcounter = 0 3330 for idpackage in idpackages: 3331 3332 currentcounter += 1 3333 pkgfile = dbconn.retrieveDownloadURL(idpackage) 3334 orig_branch = self.get_branch_from_download_relative_uri( 3335 pkgfile) 3336 3337 self.updateProgress( 3338 "[%s] %s: %s" % ( 3339 brown(crippled_uri), 3340 blue(_("checking hash")), 3341 darkgreen(pkgfile), 3342 ), 3343 importance = 1, 3344 type = "info", 3345 header = blue(" @@ "), 3346 back = True, 3347 count = (currentcounter, totalcounter,) 3348 ) 3349 3350 checksum_ok = False 3351 ck_remote = self.get_remote_package_checksum(repo, 3352 os.path.basename(pkgfile), orig_branch) 3353 if ck_remote == None: 3354 self.updateProgress( 3355 "[%s] %s: %s %s" % ( 3356 brown(crippled_uri), 3357 blue(_("digest verification of")), 3358 bold(pkgfile), 3359 blue(_("not supported")), 3360 ), 3361 importance = 1, 3362 type = "info", 3363 header = blue(" @@ "), 3364 count = (currentcounter, totalcounter,) 3365 ) 3366 elif len(ck_remote) == 32: 3367 pkghash = dbconn.retrieveDigest(idpackage) 3368 if ck_remote == pkghash: 3369 checksum_ok = True 3370 else: 3371 self.updateProgress( 3372 "[%s] %s: %s %s" % ( 3373 brown(crippled_uri), 3374 blue(_("digest verification of")), 3375 bold(pkgfile), 3376 blue(_("failed for unknown reasons")), 3377 ), 3378 importance = 1, 3379 type = "info", 3380 header = blue(" @@ "), 3381 count = (currentcounter, totalcounter,) 3382 ) 3383 3384 if checksum_ok: 3385 match.add(idpackage) 3386 else: 3387 not_match.add(idpackage) 3388 self.updateProgress( 3389 "[%s] %s: %s %s" % ( 3390 brown(crippled_uri), 3391 blue(_("package")), 3392 bold(pkgfile), 3393 red(_("NOT healthy")), 3394 ), 3395 importance = 1, 3396 type = "warning", 3397 header = darkred(" !!! "), 3398 count = (currentcounter, totalcounter,) 3399 ) 3400 if not broken_packages.has_key(crippled_uri): 3401 broken_packages[crippled_uri] = [] 3402 broken_packages[crippled_uri].append(pkgfile) 3403 3404 if broken_packages: 3405 mytxt = blue("%s:") % ( 3406 _("This is the list of broken packages"),) 3407 self.updateProgress( 3408 mytxt, 3409 importance = 1, 3410 type = "info", 3411 header = red(" * ") 3412 ) 3413 for mirror in broken_packages.keys(): 3414 mytxt = "%s: %s" % ( 3415 brown(_("Mirror")), 3416 bold(mirror), 3417 ) 3418 self.updateProgress( 3419 mytxt, 3420 importance = 1, 3421 type = "info", 3422 header = red(" <> ") 3423 ) 3424 for broken_package in broken_packages[mirror]: 3425 self.updateProgress( 3426 blue(broken_package), 3427 importance = 1, 3428 type = "info", 3429 header = red(" - ") 3430 ) 3431 3432 self.updateProgress( 3433 "%s:" % ( 3434 blue(_("Statistics")), 3435 ), 3436 importance = 1, 3437 type = "info", 3438 header = red(" @@ ") 3439 ) 3440 self.updateProgress( 3441 "[%s] %s:\t%s" % ( 3442 red(crippled_uri), 3443 brown(_("Number of checked packages")), 3444 brown(str(len(match) + len(not_match))), 3445 ), 3446 importance = 1, 3447 type = "info", 3448 header = brown(" # ") 3449 ) 3450 self.updateProgress( 3451 "[%s] %s:\t%s" % ( 3452 red(crippled_uri), 3453 darkgreen(_("Number of healthy packages")), 3454 darkgreen(str(len(match))), 3455 ), 3456 importance = 1, 3457 type = "info", 3458 header = brown(" # ") 3459 ) 3460 self.updateProgress( 3461 "[%s] %s:\t%s" % ( 3462 red(crippled_uri), 3463 darkred(_("Number of broken packages")), 3464 darkred(str(len(not_match))), 3465 ), 3466 importance = 1, 3467 type = "info", 3468 header = brown(" # ") 3469 ) 3470 3471 return match, not_match, broken_packages

3472 3473

3474 - def verify_local_packages(self, packages, ask = True, repo = None):

3475 3476 if repo == None: 3477 repo = self.default_repository 3478 3479 self.updateProgress( 3480 "[%s] %s:" % ( 3481 red(_("local")), 3482 blue(_("Integrity verification of the selected packages")), 3483 ), 3484 importance = 1, 3485 type = "info", 3486 header = darkgreen(" * ") 3487 ) 3488 3489 idpackages, world = self.match_packages(packages) 3490 dbconn = self.open_server_repository(read_only = True, 3491 no_upload = True, repo = repo) 3492 3493 if world: 3494 self.updateProgress( 3495 blue(_("All the packages in repository will be checked.")), 3496 importance = 1, 3497 type = "info", 3498 header = " " 3499 ) 3500 3501 to_download = set() 3502 available = set() 3503 for idpackage in idpackages: 3504 3505 pkgatom = dbconn.retrieveAtom(idpackage) 3506 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3507 pkg_rel_path = '/'.join(pkg_path.split("/")[2:]) 3508 3509 bindir_path = os.path.join(self.get_local_packages_directory(repo), 3510 pkg_rel_path) 3511 uploaddir_path = os.path.join(self.get_local_upload_directory(repo), 3512 pkg_rel_path) 3513 3514 if os.path.isfile(bindir_path): 3515 if not world: 3516 self.updateProgress( 3517 "[%s] %s :: %s" % ( 3518 darkgreen(_("available")), 3519 blue(pkgatom), 3520 darkgreen(pkg_rel_path), 3521 ), 3522 importance = 0, 3523 type = "info", 3524 header = darkgreen(" # ") 3525 ) 3526 available.add(idpackage) 3527 elif os.path.isfile(uploaddir_path): 3528 if not world: 3529 self.updateProgress( 3530 "[%s] %s :: %s" % ( 3531 darkred(_("upload/ignored")), 3532 blue(pkgatom), 3533 darkgreen(pkg_rel_path), 3534 ), 3535 importance = 0, 3536 type = "info", 3537 header = darkgreen(" # ") 3538 ) 3539 else: 3540 self.updateProgress( 3541 "[%s] %s :: %s" % ( 3542 brown(_("download")), 3543 blue(pkgatom), 3544 darkgreen(pkg_rel_path), 3545 ), 3546 importance = 0, 3547 type = "info", 3548 header = darkgreen(" # ") 3549 ) 3550 to_download.add((idpackage, pkg_path,)) 3551 3552 if ask: 3553 rc_question = self.askQuestion(_("Would you like to continue ?")) 3554 if rc_question == "No": 3555 return set(), set(), set(), set() 3556 3557 3558 fine = set() 3559 failed = set() 3560 downloaded_fine = set() 3561 downloaded_errors = set() 3562 3563 if to_download: 3564 3565 not_downloaded = set() 3566 mytxt = blue("%s ...") % (_("Starting to download missing files"),) 3567 self.updateProgress( 3568 mytxt, 3569 importance = 1, 3570 type = "info", 3571 header = " " 3572 ) 3573 for uri in self.get_remote_mirrors(repo): 3574 3575 if not_downloaded: 3576 mytxt = blue("%s ...") % ( 3577 _("Searching missing/broken files on another mirror"),) 3578 self.updateProgress( 3579 mytxt, 3580 importance = 1, 3581 type = "info", 3582 header = " " 3583 ) 3584 to_download = not_downloaded.copy() 3585 not_downloaded = set() 3586 3587 for idpackage, pkg_path in to_download: 3588 rc_down = self.MirrorsService.download_package(uri, 3589 pkg_path, repo = repo) 3590 if rc_down: 3591 downloaded_fine.add(idpackage) 3592 available.add(idpackage) 3593 else: 3594 not_downloaded.add(pkg_path) 3595 3596 if not not_downloaded: 3597 self.updateProgress( 3598 red(_("Binary packages downloaded successfully.")), 3599 importance = 1, 3600 type = "info", 3601 header = " " 3602 ) 3603 break 3604 3605 if not_downloaded: 3606 mytxt = blue("%s:") % ( 3607 _("These are the packages that cannot be found online"),) 3608 self.updateProgress( 3609 mytxt, 3610 importance = 1, 3611 type = "info", 3612 header = " " 3613 ) 3614 for pkg_path in not_downloaded: 3615 downloaded_errors.add(pkg_path) 3616 self.updateProgress( 3617 brown(pkg_path), 3618 importance = 1, 3619 type = "warning", 3620 header = red(" * ") 3621 ) 3622 downloaded_errors |= not_downloaded 3623 mytxt = "%s." % (_("They won't be checked"),) 3624 self.updateProgress( 3625 mytxt, 3626 importance = 1, 3627 type = "warning", 3628 header = " " 3629 ) 3630 3631 my_qa = self.QA() 3632 3633 totalcounter = str(len(available)) 3634 currentcounter = 0 3635 for idpackage in available: 3636 currentcounter += 1 3637 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3638 orig_branch = self.get_branch_from_download_relative_uri(pkg_path) 3639 pkgfile = os.path.basename(pkg_path) 3640 3641 self.updateProgress( 3642 "[branch:%s] %s %s" % ( 3643 brown(orig_branch), 3644 blue(_("checking status of")), 3645 darkgreen(pkgfile), 3646 ), 3647 importance = 1, 3648 type = "info", 3649 header = " ", 3650 back = True, 3651 count = (currentcounter, totalcounter,) 3652 ) 3653 3654 storedmd5 = dbconn.retrieveDigest(idpackage) 3655 pkgpath = os.path.join(self.get_local_packages_directory(repo), 3656 orig_branch, pkgfile) 3657 result = self.entropyTools.compare_md5(pkgpath, storedmd5) 3658 qa_fine = my_qa.entropy_package_checks(pkgpath) 3659 if result and qa_fine: 3660 fine.add(idpackage) 3661 else: 3662 failed.add(idpackage) 3663 self.updateProgress( 3664 "[branch:%s] %s %s %s: %s" % ( 3665 brown(orig_branch), 3666 blue(_("package")), 3667 darkgreen(pkg_path), 3668 blue(_("is corrupted, stored checksum")), 3669 brown(storedmd5), 3670 ), 3671 importance = 1, 3672 type = "info", 3673 header = " ", 3674 count = (currentcounter, totalcounter,) 3675 ) 3676 3677 if failed: 3678 mytxt = blue("%s:") % (_("This is the list of broken packages"),) 3679 self.updateProgress( 3680 mytxt, 3681 importance = 1, 3682 type = "warning", 3683 header = darkred(" # ") 3684 ) 3685 for idpackage in failed: 3686 atom = dbconn.retrieveAtom(idpackage) 3687 down_p = dbconn.retrieveDownloadURL(idpackage) 3688 self.updateProgress( 3689 blue("[atom:%s] %s" % (atom, down_p,)), 3690 importance = 0, 3691 type = "warning", 3692 header = brown(" # ") 3693 ) 3694 3695 # print stats 3696 self.updateProgress( 3697 red("Statistics:"), 3698 importance = 1, 3699 type = "info", 3700 header = blue(" * ") 3701 ) 3702 self.updateProgress( 3703 brown("%s => %s" % ( 3704 len(fine) + len(failed), 3705 _("checked packages"), 3706 ) 3707 ), 3708 importance = 0, 3709 type = "info", 3710 header = brown(" # ") 3711 ) 3712 self.updateProgress( 3713 darkgreen("%s => %s" % ( 3714 len(fine), 3715 _("healthy packages"), 3716 ) 3717 ), 3718 importance = 0, 3719 type = "info", 3720 header = brown(" # ") 3721 ) 3722 self.updateProgress( 3723 darkred("%s => %s" % ( 3724 len(failed), 3725 _("broken packages"), 3726 ) 3727 ), 3728 importance = 0, 3729 type = "info", 3730 header = brown(" # ") 3731 ) 3732 self.updateProgress( 3733 blue("%s => %s" % ( 3734 len(downloaded_fine), 3735 _("downloaded packages"), 3736 ) 3737 ), 3738 importance = 0, 3739 type = "info", 3740 header = brown(" # ") 3741 ) 3742 self.updateProgress( 3743 bold("%s => %s" % ( 3744 len(downloaded_errors), 3745 _("failed downloads"), 3746 ) 3747 ), 3748 importance = 0, 3749 type = "info", 3750 header = brown(" # ") 3751 ) 3752 3753 self.close_server_database(dbconn) 3754 return fine, failed, downloaded_fine, downloaded_errors

3755 3756

3757 - def switch_packages_branch(self, from_branch, to_branch, repo = None):

3758 3759 if repo == None: 3760 repo = self.default_repository 3761 3762 if to_branch != self.SystemSettings['repositories']['branch']: 3763 mytxt = "%s: %s %s" % ( 3764 blue(_("Please setup your branch to")), 3765 bold(to_branch), 3766 blue(_("and retry")), 3767 ) 3768 self.updateProgress( 3769 mytxt, 3770 importance = 1, 3771 type = "error", 3772 header = darkred(" !! ") 3773 ) 3774 return None 3775 3776 mytxt = red("%s ...") % (_("Copying database (if not exists)"),) 3777 self.updateProgress( 3778 mytxt, 3779 importance = 1, 3780 type = "info", 3781 header = darkgreen(" @@ ") 3782 ) 3783 branch_dbdir = self.get_local_database_dir(repo) 3784 old_branch_dbdir = self.get_local_database_dir(repo, from_branch) 3785 3786 # close all our databases 3787 self.close_server_databases() 3788 3789 # if database file did not exist got created as an empty file 3790 # we can just rm -rf it 3791 branch_dbfile = self.get_local_database_file(repo) 3792 if os.path.isfile(branch_dbfile): 3793 if self.entropyTools.get_file_size(branch_dbfile) == 0: 3794 shutil.rmtree(branch_dbdir, True) 3795 3796 if os.path.isdir(branch_dbdir): 3797 3798 while 1: 3799 rnd_num = self.entropyTools.get_random_number() 3800 backup_dbdir = branch_dbdir + str(rnd_num) 3801 if not os.path.isdir(backup_dbdir): 3802 break 3803 os.rename(branch_dbdir, backup_dbdir) 3804 3805 if os.path.isdir(old_branch_dbdir): 3806 shutil.copytree(old_branch_dbdir, branch_dbdir) 3807 3808 mytxt = red("%s ...") % (_("Switching packages"),) 3809 self.updateProgress( 3810 mytxt, 3811 importance = 1, 3812 type = "info", 3813 header = darkgreen(" @@ ") 3814 ) 3815 3816 dbconn = self.open_server_repository(read_only = False, 3817 no_upload = True, repo = repo, lock_remote = False) 3818 try: 3819 dbconn.validateDatabase() 3820 except SystemDatabaseError: 3821 self.handle_uninitialized_repository(repo) 3822 dbconn = self.open_server_repository(read_only = False, 3823 no_upload = True, repo = repo, lock_remote = False) 3824 3825 idpackages = dbconn.listAllIdpackages() 3826 already_switched = set() 3827 not_found = set() 3828 switched = set() 3829 ignored = set() 3830 no_checksum = set() 3831 3832 maxcount = len(idpackages) 3833 count = 0 3834 for idpackage in idpackages: 3835 count += 1 3836 3837 cur_branch = dbconn.retrieveBranch(idpackage) 3838 atom = dbconn.retrieveAtom(idpackage) 3839 if cur_branch == to_branch: 3840 already_switched.add(idpackage) 3841 self.updateProgress( 3842 red("%s %s, %s %s" % ( 3843 _("Ignoring"), 3844 bold(atom), 3845 _("already in branch"), 3846 cur_branch, 3847 ) 3848 ), 3849 importance = 0, 3850 type = "info", 3851 header = darkgreen(" @@ "), 3852 count = (count, maxcount,) 3853 ) 3854 ignored.add(idpackage) 3855 continue 3856 3857 self.updateProgress( 3858 "[%s=>%s] %s" % ( 3859 brown(cur_branch), 3860 bold(to_branch), 3861 darkgreen(atom), 3862 ), 3863 importance = 0, 3864 type = "info", 3865 header = darkgreen(" @@ "), 3866 back = True, 3867 count = (count, maxcount,) 3868 ) 3869 dbconn.switchBranch(idpackage, to_branch) 3870 dbconn.commitChanges() 3871 switched.add(idpackage) 3872 3873 dbconn.commitChanges() 3874 3875 # now migrate counters 3876 dbconn.moveSpmUidsToBranch(to_branch, from_branch = from_branch) 3877 3878 self.close_server_database(dbconn) 3879 mytxt = blue("%s.") % (_("migration loop completed"),) 3880 self.updateProgress( 3881 "[%s=>%s] %s" % ( 3882 brown(from_branch), 3883 bold(to_branch), 3884 mytxt, 3885 ), 3886 importance = 1, 3887 type = "info", 3888 header = darkgreen(" * ") 3889 ) 3890 3891 return switched, already_switched, ignored, not_found, no_checksum

3892

3893 - def get_entropy_sets(self, repo = None, branch = None):

3894 3895 if branch == None: 3896 branch = self.SystemSettings['repositories']['branch'] 3897 if repo == None: 3898 repo = self.default_repository 3899 3900 sets_dir = self.get_local_database_sets_dir(repo, branch) 3901 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 3902 return {} 3903 3904 mydata = {} 3905 items = os.listdir(sets_dir) 3906 for item in items: 3907 3908 try: 3909 item_clean = str(item) 3910 except (UnicodeEncodeError, UnicodeDecodeError,): 3911 continue 3912 item_path = os.path.join(sets_dir, item) 3913 if not (os.path.isfile(item_path) and \ 3914 os.access(item_path, os.R_OK)): 3915 continue 3916 item_elements = self.entropyTools.extract_packages_from_set_file( 3917 item_path) 3918 if item_elements: 3919 mydata[item_clean] = item_elements.copy() 3920 3921 return mydata

3922

3923 - def get_configured_package_sets(self, repo = None, branch = None, 3924 validate = True):

3925 3926 if branch == None: 3927 branch = self.SystemSettings['repositories']['branch'] 3928 if repo == None: 3929 repo = self.default_repository 3930 3931 # portage sets 3932 sets_data = self.Spm().get_package_sets(False) 3933 sets_data.update(self.get_entropy_sets(repo, branch)) 3934 3935 if validate: 3936 invalid_sets = set() 3937 # validate 3938 for setname in sets_data: 3939 good = True 3940 for atom in sets_data[setname]: 3941 dbconn = self.open_server_repository(just_reading = True, 3942 repo = repo) 3943 match = dbconn.atomMatch(atom) 3944 if match[0] == -1: 3945 good = False 3946 break 3947 if not good: 3948 invalid_sets.add(setname) 3949 for invalid_set in invalid_sets: 3950 del sets_data[invalid_set] 3951 3952 return sets_data

3953

3954 - def update_database_package_sets(self, repo = None, dbconn = None):

3955 3956 if repo == None: 3957 repo = self.default_repository 3958 package_sets = self.get_configured_package_sets(repo) 3959 if dbconn == None: 3960 dbconn = self.open_server_repository( 3961 read_only = False, no_upload = True, repo = repo, 3962 do_treeupdates = False) 3963 dbconn.clearPackageSets() 3964 if package_sets: 3965 dbconn.insertPackageSets(package_sets) 3966 dbconn.commitChanges()

3967

Source Code for Module entropy.server.interfaces.main