entropy.server.interfaces.main

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Server Main Interfaces}. 10 11 """ 12 13 from __future__ import with_statement 14 import os 15 import shutil 16 from entropy.core import Singleton 17 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ 18 SystemDatabaseError 19 from entropy.const import etpConst, etpSys, const_setup_perms, \ 20 const_create_working_dirs, const_extract_srv_repo_params, etpUi 21 from entropy.output import TextInterface, purple, red, darkgreen, \ 22 bold, brown, blue, darkred 23 from entropy.server.interfaces.mirrors import Server as MirrorsServer 24 from entropy.i18n import _ 25 from entropy.core.settings.base import SystemSettings 26 from entropy.core.settings.plugins.skel import SystemSettingsPlugin 27 from entropy.transceivers import FtpInterface 28 from entropy.db import EntropyRepository 29 from entropy.spm.plugins.factory import get_default_instance as get_spm 30

31 -class ServerSystemSettingsPlugin(SystemSettingsPlugin):

32

33 - def server_parser(self, sys_set):

34 35 """ 36 Parses Entropy server system configuration file. 37 38 @return dict data 39 """ 40 41 data = { 42 'repositories': etpConst['server_repositories'].copy(), 43 'default_repository_id': etpConst['officialserverrepositoryid'], 44 'packages_expiration_days': etpConst['packagesexpirationdays'], 45 'database_file_format': etpConst['etpdatabasefileformat'], 46 'disabled_eapis': set(), 47 'exp_based_scope': etpConst['expiration_based_scope'], 48 'rss': { 49 'enabled': etpConst['rss-feed'], 50 'name': etpConst['rss-name'], 51 'base_url': etpConst['rss-base-url'], 52 'website_url': etpConst['rss-website-url'], 53 'editor': etpConst['rss-managing-editor'], 54 'max_entries': etpConst['rss-max-entries'], 55 'light_max_entries': etpConst['rss-light-max-entries'], 56 }, 57 } 58 59 fake_instance = self._helper.fake_default_repo 60 61 if not os.access(etpConst['serverconf'], os.R_OK): 62 return data 63 64 with open(etpConst['serverconf'],"r") as server_f: 65 serverconf = [x.strip() for x in server_f.readlines() if x.strip()] 66 67 for line in serverconf: 68 69 split_line = line.split("|") 70 split_line_len = len(split_line) 71 72 if (line.find("officialserverrepositoryid|") != -1) and \ 73 (not line.startswith("#")) and (split_line_len == 2): 74 75 if not fake_instance: 76 data['default_repository_id'] = split_line[1].strip() 77 78 elif line.startswith("expiration-days|") and (split_line_len == 2): 79 80 mydays = split_line[1].strip() 81 try: 82 mydays = int(mydays) 83 data['packages_expiration_days'] = mydays 84 except ValueError: 85 continue 86 87 elif line.startswith("expiration-based-scope|") and \ 88 (split_line_len == 2): 89 90 exp_opt = split_line[1].strip().lower() 91 if exp_opt in ("enable", "enabled", "true", "1", "yes"): 92 data['exp_based_scope'] = True 93 else: 94 data['exp_based_scope'] = False 95 96 elif line.startswith("disabled-eapis|") and (split_line_len == 2): 97 98 mydis = split_line[1].strip().split(",") 99 try: 100 mydis = [int(x) for x in mydis] 101 mydis = set([x for x in mydis if x in (1, 2, 3,)]) 102 except ValueError: 103 continue 104 if (len(mydis) < 3) and mydis: 105 data['disabled_eapis'] = mydis 106 107 108 elif line.startswith("repository|") and (split_line_len in [5, 6]) \ 109 and (not fake_instance): 110 111 repoid, repodata = const_extract_srv_repo_params(line, 112 product = sys_set['repositories']['product']) 113 if repoid in data['repositories']: 114 # just update mirrors 115 data['repositories'][repoid]['mirrors'].extend( 116 repodata['mirrors']) 117 else: 118 data['repositories'][repoid] = repodata.copy() 119 120 elif line.startswith("database-format|") and (split_line_len == 2): 121 122 fmt = split_line[1] 123 if fmt in etpConst['etpdatabasesupportedcformats']: 124 data['database_file_format'] = fmt 125 126 elif line.startswith("rss-feed|") and (split_line_len == 2): 127 128 feed = split_line[1] 129 if feed in ("enable", "enabled", "true", "1"): 130 data['rss']['enabled'] = True 131 elif feed in ("disable", "disabled", "false", "0", "no",): 132 data['rss']['enabled'] = False 133 134 elif line.startswith("rss-name|") and (split_line_len == 2): 135 136 feedname = line.split("rss-name|")[1].strip() 137 data['rss']['name'] = feedname 138 139 elif line.startswith("rss-base-url|") and (split_line_len == 2): 140 141 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip() 142 if not data['rss']['base_url'][-1] == "/": 143 data['rss']['base_url'] += "/" 144 145 elif line.startswith("rss-website-url|") and (split_line_len == 2): 146 147 data['rss']['website_url'] = split_line[1].strip() 148 149 elif line.startswith("managing-editor|") and (split_line_len == 2): 150 151 data['rss']['editor'] = split_line[1].strip() 152 153 elif line.startswith("max-rss-entries|") and (split_line_len == 2): 154 155 try: 156 entries = int(split_line[1].strip()) 157 data['rss']['max_entries'] = entries 158 except (ValueError, IndexError,): 159 continue 160 161 elif line.startswith("max-rss-light-entries|") and \ 162 (split_line_len == 2): 163 164 try: 165 entries = int(split_line[1].strip()) 166 data['rss']['light_max_entries'] = entries 167 except (ValueError, IndexError,): 168 continue 169 170 # add system database if community repository mode is enabled 171 if self._helper.community_repo: 172 data['repositories'][etpConst['clientserverrepoid']] = {} 173 mydata = {} 174 mydata['description'] = "Community Repositories System Database" 175 mydata['mirrors'] = [] 176 mydata['community'] = False 177 data['repositories'][etpConst['clientserverrepoid']].update(mydata) 178 179 # expand paths 180 for repoid in data['repositories']: 181 data['repositories'][repoid]['packages_dir'] = \ 182 os.path.join( etpConst['entropyworkdir'], 183 "server", 184 repoid, 185 "packages", 186 etpSys['arch'] 187 ) 188 data['repositories'][repoid]['store_dir'] = \ 189 os.path.join( etpConst['entropyworkdir'], 190 "server", 191 repoid, 192 "store", 193 etpSys['arch'] 194 ) 195 data['repositories'][repoid]['upload_dir'] = \ 196 os.path.join( etpConst['entropyworkdir'], 197 "server", 198 repoid, 199 "upload", 200 etpSys['arch'] 201 ) 202 data['repositories'][repoid]['database_dir'] = \ 203 os.path.join( etpConst['entropyworkdir'], 204 "server", 205 repoid, 206 "database", 207 etpSys['arch'] 208 ) 209 data['repositories'][repoid]['packages_relative_path'] = \ 210 os.path.join( sys_set['repositories']['product'], 211 repoid, 212 "packages", 213 etpSys['arch'] 214 )+"/" 215 data['repositories'][repoid]['database_relative_path'] = \ 216 os.path.join( sys_set['repositories']['product'], 217 repoid, 218 "database", 219 etpSys['arch'] 220 )+"/" 221 222 # Support for shell variables 223 shell_repoid = os.getenv('ETP_REPO') 224 if shell_repoid: 225 data['default_repository_id'] = shell_repoid 226 227 expiration_days = os.getenv('ETP_EXPIRATION_DAYS') 228 if expiration_days: 229 try: 230 expiration_days = int(expiration_days) 231 data['packages_expiration_days'] = expiration_days 232 except ValueError: 233 pass 234 235 return data

236

237 -class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

238 239 import entropy.tools as entropyTools 240

241 - def repos_parser(self, sys_set):

242 243 data = {} 244 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 245 # if support is not enabled, don't waste time scanning files 246 srv_parser_data = sys_set[srv_plug_id]['server'] 247 if not srv_parser_data['exp_based_scope']: 248 return data 249 250 # get expiration-based packages removal data from config files 251 for repoid in srv_parser_data['repositories']: 252 253 # filter out system repository if community repository 254 # mode is enabled 255 if repoid == etpConst['clientserverrepoid']: 256 continue 257 258 idpackages = set() 259 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file( 260 repo = repoid) 261 dbconn = self._helper.open_server_repository( 262 just_reading = True, repo = repoid) 263 264 if os.access(exp_fp, os.R_OK | os.F_OK): 265 pkgs = self.entropyTools.generic_file_content_parser(exp_fp) 266 if '*' in pkgs: # wildcard support 267 idpackages.add(-1) 268 else: 269 for pkg in pkgs: 270 idpackage, rc_match = dbconn.atomMatch(pkg) 271 if rc_match: 272 continue 273 idpackages.add(idpackage) 274 275 data[repoid] = idpackages 276 277 return data

278 279

280 -class Server(Singleton, TextInterface):

281

282 - def init_singleton(self, default_repository = None, save_repository = False, 283 community_repo = False, fake_default_repo = False, 284 fake_default_repo_id = '::fake::', 285 fake_default_repo_desc = 'this is a fake repository'):

286 287 self.__instance_destroyed = False 288 if etpConst['uid'] != 0: 289 mytxt = _("Entropy Server interface must be run as root") 290 raise PermissionDenied("PermissionDenied: %s" % (mytxt,)) 291 292 # settings 293 self.SystemSettings = SystemSettings() 294 self.community_repo = community_repo 295 from entropy.db import dbapi2 296 self.dbapi2 = dbapi2 # export for third parties 297 etpSys['serverside'] = True 298 self._memory_db_instances = {} 299 self.fake_default_repo = fake_default_repo 300 self.indexing = False 301 self.xcache = False 302 self.MirrorsService = None 303 self.__server_dbcache = {} 304 self.repository_treeupdate_digests = {} 305 self.__settings_to_backup = [] 306 self.__do_save_repository = save_repository 307 self.__sync_lock_cache = set() 308 self.rssMessages = { 309 'added': {}, 310 'removed': {}, 311 'commitmessage': "", 312 'light': {}, 313 } 314 315 if fake_default_repo: 316 default_repository = fake_default_repo_id 317 etpConst['officialserverrepositoryid'] = fake_default_repo_id 318 self.init_generic_memory_server_repository(fake_default_repo_id, 319 fake_default_repo_desc) 320 321 # create our SystemSettings plugin 322 self.sys_settings_plugin_id = \ 323 etpConst['system_settings_plugins_ids']['server_plugin'] 324 self.sys_settings_plugin = ServerSystemSettingsPlugin( 325 self.sys_settings_plugin_id, self) 326 self.SystemSettings.add_plugin(self.sys_settings_plugin) 327 328 # Fatscope support SystemSettings plugin 329 self.sys_settings_fatscope_plugin_id = \ 330 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 331 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin( 332 self.sys_settings_fatscope_plugin_id, self) 333 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin) 334 335 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 336 self.default_repository = default_repository 337 if self.default_repository == None: 338 self.default_repository = srv_set['default_repository_id'] 339 340 if self.default_repository in srv_set['repositories']: 341 self.ensure_paths(self.default_repository) 342 343 # NOW DISABLED, deprecated! 344 # self.migrate_repository_databases_to_new_branched_path() 345 346 if self.default_repository not in srv_set['repositories']: 347 raise PermissionDenied("PermissionDenied: %s %s" % ( 348 self.default_repository, 349 _("repository not configured"), 350 ) 351 ) 352 if etpConst['clientserverrepoid'] == self.default_repository: 353 raise PermissionDenied("PermissionDenied: %s %s" % ( 354 etpConst['clientserverrepoid'], 355 _("protected repository id, can't use this, sorry dude..."), 356 ) 357 ) 358 359 self.switch_default_repository(self.default_repository)

360

361 - def destroy(self):

362 self.__instance_destroyed = True 363 if hasattr(self,'ClientService'): 364 self.ClientService.destroy() 365 if hasattr(self,'sys_settings_server_plugin'): 366 try: 367 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id) 368 except KeyError: 369 pass 370 if hasattr(self,'sys_settings_fatscope_plugin'): 371 try: 372 self.SystemSettings.remove_plugin( 373 self.sys_settings_fatscope_plugin) 374 except KeyError: 375 pass 376 self.close_server_databases()

377

378 - def is_destroyed(self):

379 return self.__instance_destroyed

380

381 - def __del__(self):

382 self.destroy()

383

384 - def ensure_paths(self, repo):

385 upload_dir = os.path.join(self.get_local_upload_directory(repo), 386 self.SystemSettings['repositories']['branch']) 387 db_dir = self.get_local_database_dir(repo) 388 for mydir in [upload_dir, db_dir]: 389 if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)): 390 os.makedirs(mydir) 391 const_setup_perms(mydir, etpConst['entropygid'])

392 393

394 - def migrate_repository_databases_to_new_branched_path(self):

395 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 396 migrated_filename = '.branch_migrated' 397 for repoid in srv_set['repositories'].keys(): 398 399 if repoid == etpConst['clientserverrepoid']: continue 400 mydir = srv_set['repositories'][repoid]['database_dir'] 401 if not os.path.isdir(mydir): # empty ? 402 continue 403 404 migrated_filepath = os.path.join(mydir, migrated_filename) 405 if os.path.isfile(migrated_filepath): 406 continue 407 408 my_branched_dir = self.get_local_database_dir(repoid) 409 if os.path.isdir(my_branched_dir): # wtf? do not touch 410 continue 411 412 self.updateProgress( 413 "[%s:%s] %s: %s, %s: %s" % ( 414 brown("repo"), 415 purple(repoid), 416 _("migrating database path from"), 417 brown(mydir), 418 _('to'), 419 brown(my_branched_dir), 420 ), 421 importance = 1, 422 type = "info", 423 header = bold(" @@ ") 424 ) 425 426 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \ 427 (os.path.isfile(os.path.join(mydir, x)) and \ 428 os.access(os.path.join(mydir, x), os.W_OK)) 429 ] 430 os.makedirs(my_branched_dir) 431 const_setup_perms(my_branched_dir, etpConst['entropygid']) 432 433 for repo_file in repo_files: 434 repo_filename = os.path.basename(repo_file) 435 shutil.move(repo_file, 436 os.path.join(my_branched_dir, repo_filename)) 437 438 f_migrated = open(migrated_filepath,"w") 439 f_migrated.write("done\n") 440 f_migrated.flush() 441 f_migrated.close()

442

443 - def setup_services(self):

444 self.setup_entropy_settings() 445 cs_name = 'ClientService' 446 if hasattr(self, cs_name): 447 obj = getattr(self, cs_name) 448 obj.destroy() 449 from entropy.client.interfaces import Client 450 self.ClientService = Client( 451 indexing = self.indexing, 452 xcache = self.xcache, 453 repo_validation = False, 454 noclientdb = 1 455 ) 456 from entropy.cache import EntropyCacher 457 self.Cacher = EntropyCacher() 458 self.ClientService.updateProgress = self.updateProgress 459 self.validRepositories = self.ClientService.validRepositories 460 self.entropyTools = self.ClientService.entropyTools 461 self.dumpTools = self.ClientService.dumpTools 462 self.QA = self.ClientService.QA 463 self.backup_entropy_settings() 464 465 self.MirrorsService = MirrorsServer(self)

466

467 - def Spm(self):

468 """ 469 Get Source Package Manager interface instance. 470 471 @return: Source Package Manager interface instance 472 @rtype: entropy.spm.plugins.skel.SpmPlugin based instance 473 """ 474 return get_spm(self)

475

476 - def setup_entropy_settings(self, repo = None):

477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 478 backup_list = [ 479 'etpdatabaseclientfilepath', 480 'clientdbid', 481 {'server': srv_set.copy()}, 482 ] 483 for setting in backup_list: 484 if setting not in self.__settings_to_backup: 485 self.__settings_to_backup.append(setting) 486 # setup client database 487 if not self.community_repo: 488 etpConst['etpdatabaseclientfilepath'] = \ 489 self.get_local_database_file(repo) 490 etpConst['clientdbid'] = etpConst['serverdbid'] 491 const_create_working_dirs()

492

493 - def close_server_databases(self):

494 if hasattr(self,'serverDbCache'): 495 for item in self.__server_dbcache: 496 try: 497 self.__server_dbcache[item].closeDB() 498 except self.dbapi2.ProgrammingError: # already closed? 499 pass 500 self.__server_dbcache.clear()

501

502 - def close_server_database(self, dbinstance):

503 found = None 504 for item in self.__server_dbcache: 505 if dbinstance == self.__server_dbcache[item]: 506 found = item 507 break 508 if found: 509 instance = self.__server_dbcache.pop(found) 510 instance.closeDB()

511

512 - def get_available_repositories(self):

513 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 514 return srv_set['repositories'].copy()

515

516 - def switch_default_repository(self, repoid, save = None, 517 handle_uninitialized = True):

518 519 # avoid setting __default__ as default server repo 520 if repoid == etpConst['clientserverrepoid']: 521 return 522 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 523 524 if save == None: 525 save = self.__do_save_repository 526 if repoid not in srv_set['repositories']: 527 raise PermissionDenied("PermissionDenied: %s %s" % ( 528 repoid, 529 _("repository not configured"), 530 ) 531 ) 532 self.close_server_databases() 533 srv_set['default_repository_id'] = repoid 534 self.default_repository = repoid 535 self.setup_services() 536 if save: 537 self.save_default_repository(repoid) 538 539 self.setup_community_repositories_settings() 540 self.show_interface_status() 541 if handle_uninitialized and etpUi['warn']: 542 self.handle_uninitialized_repository(repoid)

543

544 - def setup_community_repositories_settings(self):

545 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 546 if self.community_repo: 547 for repoid in srv_set['repositories']: 548 srv_set['repositories'][repoid]['community'] = True

549 550

551 - def handle_uninitialized_repository(self, repoid):

552 553 if not self.is_repository_initialized(repoid): 554 mytxt = blue("%s.") % ( 555 _("Your default repository is not initialized"),) 556 self.updateProgress( 557 "[%s:%s] %s" % ( 558 brown("repo"), 559 purple(repoid), 560 mytxt, 561 ), 562 importance = 1, 563 type = "warning", 564 header = darkred(" !!! ") 565 ) 566 answer = self.askQuestion( 567 _("Do you want to initialize your default repository ?")) 568 if answer == "No": 569 mytxt = red("%s.") % ( 570 _("Continuing with an uninitialized repository"),) 571 self.updateProgress( 572 "[%s:%s] %s" % ( 573 brown("repo"), 574 purple(repoid), 575 mytxt, 576 ), 577 importance = 1, 578 type = "warning", 579 header = darkred(" !!! ") 580 ) 581 else: 582 # move empty database for security sake 583 dbfile = self.get_local_database_file(repoid) 584 if os.path.isfile(dbfile): 585 shutil.move(dbfile, dbfile+".backup") 586 self.initialize_server_database(empty = True, 587 repo = repoid, warnings = False)

588 589

590 - def show_interface_status(self):

591 type_txt = _("server-side repository") 592 if self.community_repo: 593 type_txt = _("community repository") 594 # ..on repository: <repository_name> 595 mytxt = _("Entropy Server Interface Instance on repository") 596 self.updateProgress( 597 blue("%s: %s, %s: %s (%s: %s)" % ( 598 mytxt, 599 red(self.default_repository), 600 _("current branch"), 601 darkgreen(self.SystemSettings['repositories']['branch']), 602 purple(_("type")), 603 bold(type_txt), 604 ) 605 ), 606 importance = 2, 607 type = "info", 608 header = red(" @@ ") 609 ) 610 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 611 repos = srv_set['repositories'].keys() 612 mytxt = blue("%s:") % (_("Currently configured repositories"),) 613 self.updateProgress( 614 mytxt, 615 importance = 1, 616 type = "info", 617 header = red(" @@ ") 618 ) 619 for repo in repos: 620 self.updateProgress( 621 darkgreen(repo), 622 importance = 0, 623 type = "info", 624 header = brown(" # ") 625 )

626 627

628 - def save_default_repository(self, repoid):

629 630 # avoid setting __default__ as default server repo 631 if repoid == etpConst['clientserverrepoid']: 632 return 633 634 if os.path.isfile(etpConst['serverconf']): 635 f_srv = open(etpConst['serverconf'],"r") 636 content = f_srv.readlines() 637 f_srv.close() 638 content = [x.strip() for x in content] 639 found = False 640 new_content = [] 641 for line in content: 642 if line.strip().startswith("officialserverrepositoryid|"): 643 line = "officialserverrepositoryid|%s" % (repoid,) 644 found = True 645 new_content.append(line) 646 if not found: 647 new_content.append("officialserverrepositoryid|%s" % (repoid,)) 648 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w") 649 for line in new_content: 650 f_srv_t.write(line+"\n") 651 f_srv_t.flush() 652 f_srv_t.close() 653 os.rename(etpConst['serverconf']+".save_default_repo_tmp", 654 etpConst['serverconf']) 655 else: 656 f_srv = open(etpConst['serverconf'],"w") 657 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,)) 658 f_srv.flush() 659 f_srv.close()

660

661 - def toggle_repository(self, repoid, enable = True):

662 663 # avoid setting __default__ as default server repo 664 if repoid == etpConst['clientserverrepoid']: 665 return False 666 667 if not os.path.isfile(etpConst['serverconf']): 668 return None 669 f_srv = open(etpConst['serverconf']) 670 tmpfile = etpConst['serverconf']+".switch" 671 mycontent = [x.strip() for x in f_srv.readlines()] 672 f_srv.close() 673 f_tmp = open(tmpfile,"w") 674 st = "repository|%s" % (repoid,) 675 status = False 676 for line in mycontent: 677 if enable: 678 if (line.find(st) != -1) and line.startswith("#") and \ 679 (len(line.split("|")) == 5): 680 line = line[1:] 681 status = True 682 else: 683 if (line.find(st) != -1) and not line.startswith("#") and \ 684 (len(line.split("|")) == 5): 685 line = "#"+line 686 status = True 687 f_tmp.write(line+"\n") 688 f_tmp.flush() 689 f_tmp.close() 690 shutil.move(tmpfile, etpConst['serverconf']) 691 if status: 692 self.close_server_databases() 693 self.SystemSettings.clear() 694 self.setup_services() 695 self.show_interface_status() 696 return status

697

698 - def backup_entropy_settings(self):

699 for setting in self.__settings_to_backup: 700 if isinstance(setting, basestring): 701 self.ClientService.backup_constant(setting) 702 elif isinstance(setting, dict): 703 self.SystemSettings.set_persistent_setting(setting)

704

705 - def is_repository_initialized(self, repo):

706 707 def do_validate(dbc): 708 try: 709 dbc.validateDatabase() 710 return True 711 except SystemDatabaseError: 712 return False

713 714 dbc = self.open_server_repository(just_reading = True, repo = repo) 715 valid = do_validate(dbc) 716 self.close_server_database(dbc) 717 if not valid: # check online? 718 dbc = self.open_server_repository(read_only = False, 719 no_upload = True, repo = repo, is_new = True) 720 valid = do_validate(dbc) 721 self.close_server_database(dbc) 722 723 return valid

724

725 - def do_server_repository_sync_lock(self, repo, no_upload):

726 727 if repo == None: 728 repo = self.default_repository 729 730 # check if the database is locked locally 731 lock_file = self.MirrorsService.get_database_lockfile(repo) 732 if os.path.isfile(lock_file): 733 self.updateProgress( 734 red(_("Entropy database is already locked by you :-)")), 735 importance = 1, 736 type = "info", 737 header = red(" * ") 738 ) 739 else: 740 # check if the database is locked REMOTELY 741 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),) 742 self.updateProgress( 743 red(mytxt), 744 importance = 1, 745 type = "info", 746 header = red(" * "), 747 back = True 748 ) 749 for uri in self.get_remote_mirrors(repo): 750 given_up = self.MirrorsService.mirror_lock_check(uri, 751 repo = repo) 752 if given_up: 753 crippled_uri = \ 754 self.entropyTools.extract_ftp_host_from_uri(uri) 755 mytxt = "%s:" % (_("Mirrors status table"),) 756 self.updateProgress( 757 darkgreen(mytxt), 758 importance = 1, 759 type = "info", 760 header = brown(" * ") 761 ) 762 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo) 763 for db_uri, db_st1, db_st2 in dbstatus: 764 db_st1_info = darkgreen(_("Unlocked")) 765 if db_st1: 766 db_st1_info = red(_("Locked")) 767 db_st2_info = darkgreen(_("Unlocked")) 768 if db_st2: 769 db_st2_info = red(_("Locked")) 770 771 crippled_uri = \ 772 self.entropyTools.extract_ftp_host_from_uri(db_uri) 773 self.updateProgress( 774 "%s: [%s: %s] [%s: %s]" % ( 775 bold(crippled_uri), 776 brown(_("database")), 777 db_st1_info, 778 brown(_("download")), 779 db_st2_info, 780 ), 781 importance = 1, 782 type = "info", 783 header = "\t" 784 ) 785 786 raise OnlineMirrorError("OnlineMirrorError: %s %s" % ( 787 _("cannot lock mirror"), 788 crippled_uri, 789 ) 790 ) 791 792 # if we arrive here, it is because all the mirrors are unlocked 793 self.MirrorsService.lock_mirrors(True, repo = repo) 794 self.MirrorsService.sync_databases(no_upload, repo = repo)

795 796

797 - def init_generic_memory_server_repository(self, repoid, description, 798 mirrors = None, community_repo = False, service_url = None):

799 800 if mirrors is None: 801 mirrors = [] 802 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid) 803 self._memory_db_instances[repoid] = dbc 804 805 eapi3_port = int(etpConst['socket_service']['port']) 806 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port']) 807 # add to settings 808 repodata = { 809 'repoid': repoid, 810 'description': description, 811 'mirrors': mirrors, 812 'community': community_repo, 813 'service_port': eapi3_port, 814 'ssl_service_port': eapi3_ssl_port, 815 'service_url': service_url, 816 'handler': '', # not supported 817 'in_memory': True, 818 } 819 820 etpConst['server_repositories'][repoid] = repodata 821 self.SystemSettings.clear() 822 823 return dbc

824

825 - def open_memory_database(self, dbname = None):

826 if dbname == None: 827 dbname = etpConst['genericdbid'] 828 dbc = EntropyRepository( 829 readOnly = False, 830 dbFile = ':memory:', 831 clientDatabase = True, 832 dbname = dbname, 833 xcache = False, 834 indexing = False, 835 OutputInterface = self, 836 skipChecks = True 837 ) 838 dbc.initializeDatabase() 839 return dbc

840

841 - def open_server_repository( 842 self, 843 read_only = True, 844 no_upload = True, 845 just_reading = False, 846 repo = None, 847 indexing = True, 848 warnings = True, 849 do_cache = True, 850 use_branch = None, 851 lock_remote = True, 852 is_new = False, 853 do_treeupdates = True 854 ):

855 856 if repo == None: 857 repo = self.default_repository 858 859 if repo == etpConst['clientserverrepoid'] and self.community_repo: 860 return self.ClientService.clientDbconn 861 862 # in-memory server repos 863 if repo in self._memory_db_instances: 864 return self._memory_db_instances.get(repo) 865 866 if just_reading: 867 read_only = True 868 no_upload = True 869 870 local_dbfile = self.get_local_database_file(repo, use_branch) 871 if do_cache: 872 cached = self.__server_dbcache.get( 873 (repo, etpConst['systemroot'], local_dbfile, read_only, 874 no_upload, just_reading, use_branch, lock_remote,) 875 ) 876 if cached != None: 877 return cached 878 879 local_dbfile_dir = os.path.dirname(local_dbfile) 880 if not os.path.isdir(local_dbfile_dir): 881 os.makedirs(local_dbfile_dir) 882 883 if (not read_only) and (lock_remote) and \ 884 (repo not in self.__sync_lock_cache): 885 self.do_server_repository_sync_lock(repo, no_upload) 886 self.__sync_lock_cache.add(repo) 887 888 local_dbfile_exists = os.path.lexists(local_dbfile) 889 conn = EntropyRepository( 890 readOnly = read_only, 891 dbFile = local_dbfile, 892 noUpload = no_upload, 893 OutputInterface = self, 894 dbname = etpConst['serverdbid']+repo, 895 useBranch = use_branch, 896 lockRemote = lock_remote 897 ) 898 if not local_dbfile_exists: 899 # better than having a completely broken db 900 conn.readOnly = False 901 conn.initializeDatabase() 902 conn.commitChanges() 903 904 valid = True 905 try: 906 conn.validateDatabase() 907 except SystemDatabaseError: 908 valid = False 909 910 # verify if we need to update the database to sync 911 # with portage updates, we just ignore being readonly in the case 912 if (repo not in etpConst['server_treeupdatescalled']) and \ 913 (not just_reading): 914 # sometimes, when filling a new server db 915 # we need to avoid tree updates 916 if valid: 917 if do_treeupdates: 918 self.repository_packages_spm_sync(conn, 919 branch = use_branch, repo = repo) 920 elif warnings and not is_new: 921 mytxt = _("Entropy database is corrupted!") 922 self.updateProgress( 923 darkred(mytxt), 924 importance = 1, 925 type = "warning", 926 header = bold(" !!! ") 927 ) 928 929 if not read_only and valid and indexing: 930 931 self.updateProgress( 932 "[repo:%s|%s] %s" % ( 933 blue(repo), 934 red(_("database")), 935 blue(_("indexing database")), 936 ), 937 importance = 1, 938 type = "info", 939 header = brown(" @@ "), 940 back = True 941 ) 942 conn.createAllIndexes() 943 944 if do_cache: 945 # !!! also cache just_reading otherwise there will be 946 # real issues if the connection is opened several times 947 self.__server_dbcache[ 948 (repo, etpConst['systemroot'], local_dbfile, read_only, 949 no_upload, just_reading, use_branch, lock_remote,)] = conn 950 951 # auto-update package sets 952 if (not read_only) and (not is_new): 953 cur_sets = conn.retrievePackageSets() 954 sys_sets = self.get_configured_package_sets(repo) 955 if cur_sets != sys_sets: 956 self.update_database_package_sets(repo, dbconn = conn) 957 conn.commitChanges() 958 959 return conn

960

961 - def repository_packages_spm_sync(self, repo_db, branch = None, repo = None):

962 """ 963 Service method used to sync package names with Source Package Manager. 964 Source Package Manager can change package names, categories or slot 965 and Entropy repositories must be kept in sync. 966 967 In other words, it checks for /usr/portage/profiles/updates changes. 968 """ 969 970 if branch == None: 971 branch = self.SystemSettings['repositories']['branch'] 972 if repo == None: 973 repo = self.default_repository 974 975 etpConst['server_treeupdatescalled'].add(repo) 976 977 repo_updates_file = self.get_local_database_treeupdates_file(repo) 978 doRescan = False 979 980 stored_digest = repo_db.retrieveRepositoryUpdatesDigest(repo) 981 if stored_digest == -1: 982 doRescan = True 983 984 # check portage files for changes if doRescan is still false 985 portage_dirs_digest = "0" 986 if not doRescan: 987 988 if self.repository_treeupdate_digests.has_key(repo): 989 portage_dirs_digest = self.repository_treeupdate_digests.get( 990 repo) 991 else: 992 993 spm = self.Spm() 994 # grab portdir 995 updates_dir = etpConst['systemroot'] + \ 996 spm.get_setting("PORTDIR") + "/profiles/updates" 997 if os.path.isdir(updates_dir): 998 # get checksum 999 mdigest = self.entropyTools.md5obj_directory(updates_dir) 1000 # also checksum etpConst['etpdatabaseupdatefile'] 1001 if os.path.isfile(repo_updates_file): 1002 f = open(repo_updates_file) 1003 block = f.read(1024) 1004 while block: 1005 mdigest.update(block) 1006 block = f.read(1024) 1007 f.close() 1008 portage_dirs_digest = mdigest.hexdigest() 1009 self.repository_treeupdate_digests[repo] = \ 1010 portage_dirs_digest 1011 1012 if doRescan or (str(stored_digest) != str(portage_dirs_digest)): 1013 1014 # force parameters 1015 repo_db.readOnly = False 1016 repo_db.noUpload = True 1017 1018 # reset database tables 1019 repo_db.clearTreeupdatesEntries(repo) 1020 1021 updates_dir = etpConst['systemroot'] + \ 1022 self.Spm().get_setting("PORTDIR") + "/profiles/updates" 1023 update_files = self.entropyTools.sort_update_files( 1024 os.listdir(updates_dir)) 1025 update_files = [os.path.join(updates_dir, x) for x in update_files] 1026 # now load actions from files 1027 update_actions = [] 1028 for update_file in update_files: 1029 f = open(update_file, "r") 1030 mycontent = f.readlines() 1031 f.close() 1032 lines = [x.strip() for x in mycontent if x.strip()] 1033 update_actions.extend(lines) 1034 1035 # add entropy packages.db.repo_updates content 1036 if os.path.isfile(repo_updates_file): 1037 f = open(repo_updates_file, "r") 1038 mycontent = f.readlines() 1039 f.close() 1040 lines = [x.strip() for x in mycontent if x.strip() and \ 1041 not x.strip().startswith("#")] 1042 update_actions.extend(lines) 1043 # now filter the required actions 1044 update_actions = repo_db.filterTreeUpdatesActions(update_actions) 1045 if update_actions: 1046 1047 mytxt = "%s: %s. %s %s" % ( 1048 bold(_("ATTENTION")), 1049 red(_("forcing package updates")), 1050 red(_("Syncing with")), 1051 blue(updates_dir), 1052 ) 1053 self.updateProgress( 1054 mytxt, 1055 importance = 1, 1056 type = "info", 1057 header = brown(" * ") 1058 ) 1059 # lock database 1060 if repo_db.lockRemote: 1061 self.do_server_repository_sync_lock( 1062 repo, repo_db.noUpload) 1063 # now run queue 1064 try: 1065 quickpkg_list = repo_db.runTreeUpdatesActions( 1066 update_actions) 1067 except: 1068 # destroy digest 1069 repo_db.setRepositoryUpdatesDigest(repo, "-1") 1070 raise 1071 1072 if quickpkg_list: 1073 # quickpkg package and packages owning it as a dependency 1074 try: 1075 self._run_packages_spm_sync_quickpkg( 1076 quickpkg_list, repo_db, repo) 1077 except: 1078 self.entropyTools.print_traceback() 1079 mytxt = "%s: %s: %s, %s." % ( 1080 bold(_("WARNING")), 1081 red(_("Cannot complete quickpkg for atoms")), 1082 blue(str(sorted(quickpkg_list))), 1083 _("do it manually"), 1084 ) 1085 self.updateProgress( 1086 mytxt, 1087 importance = 1, 1088 type = "warning", 1089 header = darkred(" * ") 1090 ) 1091 repo_db.commitChanges() 1092 1093 # store new actions 1094 repo_db.addRepositoryUpdatesActions( 1095 repo, update_actions, branch) 1096 1097 # store new digest into database 1098 repo_db.setRepositoryUpdatesDigest( 1099 repo, portage_dirs_digest) 1100 repo_db.commitChanges()

1101

1102 - def _run_packages_spm_sync_quickpkg(self, atoms, repo_db, repo):

1103 """ 1104 Executes packages regeneration for given atoms. 1105 """ 1106 package_paths = set() 1107 runatoms = set() 1108 for myatom in atoms: 1109 mymatch = repo_db.atomMatch(myatom) 1110 if mymatch[0] == -1: 1111 continue 1112 myatom = repo_db.retrieveAtom(mymatch[0]) 1113 myatom = self.entropyTools.remove_tag(myatom) 1114 runatoms.add(myatom) 1115 1116 for myatom in runatoms: 1117 1118 self.updateProgress( 1119 red("%s: " % (_("repackaging"),) )+blue(myatom), 1120 importance = 1, 1121 type = "warning", 1122 header = blue(" # ") 1123 ) 1124 mydest = self.get_local_store_directory(repo = repo) 1125 try: 1126 mypath = self.quickpkg(myatom, mydest) 1127 except: 1128 # remove broken bin before raising 1129 mypath = os.path.join(mydest, 1130 os.path.basename(myatom) + etpConst['packagesext']) 1131 if os.path.isfile(mypath): 1132 os.remove(mypath) 1133 self.entropyTools.print_traceback() 1134 mytxt = "%s: %s: %s, %s." % ( 1135 bold(_("WARNING")), 1136 red(_("Cannot complete quickpkg for atom")), 1137 blue(myatom), 1138 _("do it manually"), 1139 ) 1140 self.updateProgress( 1141 mytxt, 1142 importance = 1, 1143 type = "warning", 1144 header = darkred(" * ") 1145 ) 1146 continue 1147 package_paths.add(mypath) 1148 packages_data = [(x, False,) for x in package_paths] 1149 idpackages = self.add_packages_to_repository( 1150 packages_data, repo = repo) 1151 1152 if not idpackages: 1153 1154 mytxt = "%s: %s. %s." % ( 1155 bold(_("ATTENTION")), 1156 red(_("package files rebuild did not run properly")), 1157 red(_("Please update packages manually")), 1158 ) 1159 self.updateProgress( 1160 mytxt, 1161 importance = 1, 1162 type = "warning", 1163 header = darkred(" * ") 1164 )

1165

1166 - def deps_tester(self, default_repo = None):

1167 1168 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1169 server_repos = sys_set['repositories'].keys() 1170 installed_packages = set() 1171 # if a default repository is passed, we will just test against it 1172 if default_repo: 1173 server_repos = [default_repo] 1174 1175 for repo in server_repos: 1176 dbconn = self.open_server_repository(read_only = True, 1177 no_upload = True, repo = repo, do_treeupdates = False) 1178 installed_packages |= set([(x, repo) for x in \ 1179 dbconn.listAllIdpackages()]) 1180 1181 1182 deps_not_satisfied = set() 1183 length = str((len(installed_packages))) 1184 count = 0 1185 mytxt = _("Checking") 1186 1187 for idpackage, repo in installed_packages: 1188 count += 1 1189 dbconn = self.open_server_repository(read_only = True, 1190 no_upload = True, repo = repo, do_treeupdates = False) 1191 1192 if (count%150 == 0) or (count == length) or (count == 1): 1193 atom = dbconn.retrieveAtom(idpackage) 1194 self.updateProgress( 1195 darkgreen(mytxt)+" "+bold(atom), 1196 importance = 0, 1197 type = "info", 1198 back = True, 1199 count = (count, length), 1200 header = darkred(" @@ ") 1201 ) 1202 1203 xdeps = dbconn.retrieveDependencies(idpackage) 1204 for xdep in xdeps: 1205 xid, xuseless = self.atom_match(xdep) 1206 if xid == -1: 1207 deps_not_satisfied.add(xdep) 1208 1209 return deps_not_satisfied

1210

1211 - def dependencies_test(self, repo = None):

1212 1213 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("...")) 1214 self.updateProgress( 1215 mytxt, 1216 importance = 2, 1217 type = "info", 1218 header = red(" @@ ") 1219 ) 1220 1221 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1222 server_repos = srv_set['repositories'].keys() 1223 deps_not_matched = self.deps_tester(repo) 1224 1225 if deps_not_matched: 1226 1227 crying_atoms = {} 1228 for atom in deps_not_matched: 1229 for repo in server_repos: 1230 dbconn = self.open_server_repository(just_reading = True, 1231 repo = repo, do_treeupdates = False) 1232 riddep = dbconn.searchDependency(atom) 1233 if riddep == -1: 1234 continue 1235 ridpackages = dbconn.searchIdpackageFromIddependency(riddep) 1236 for i in ridpackages: 1237 iatom = dbconn.retrieveAtom(i) 1238 if not crying_atoms.has_key(atom): 1239 crying_atoms[atom] = set() 1240 crying_atoms[atom].add((iatom, repo)) 1241 1242 mytxt = blue("%s:") % (_("These are the dependencies not found"),) 1243 self.updateProgress( 1244 mytxt, 1245 importance = 1, 1246 type = "info", 1247 header = red(" @@ ") 1248 ) 1249 mytxt = "%s:" % (_("Needed by"),) 1250 for atom in deps_not_matched: 1251 self.updateProgress( 1252 red(atom), 1253 importance = 1, 1254 type = "info", 1255 header = blue(" # ") 1256 ) 1257 if crying_atoms.has_key(atom): 1258 self.updateProgress( 1259 red(mytxt), 1260 importance = 0, 1261 type = "info", 1262 header = blue(" # ") 1263 ) 1264 for my_dep, myrepo in crying_atoms[atom]: 1265 self.updateProgress( 1266 "[%s:%s] %s" % ( 1267 blue(_("by repo")), 1268 darkred(myrepo), 1269 darkgreen(my_dep), 1270 ), 1271 importance = 0, 1272 type = "info", 1273 header = blue(" # ") 1274 ) 1275 else: 1276 1277 mytxt = blue(_("Every dependency is satisfied. It's all fine.")) 1278 self.updateProgress( 1279 mytxt, 1280 importance = 2, 1281 type = "info", 1282 header = red(" @@ ") 1283 ) 1284 1285 return deps_not_matched

1286

1287 - def test_shared_objects(self, get_files = False, repo = None, 1288 dump_results_to_file = False):

1289 1290 pkg_list_path = None 1291 if dump_results_to_file: 1292 tmp_dir = os.path.dirname(self.entropyTools.get_random_temp_file()) 1293 pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt") 1294 dmp_data = [ 1295 (_("Broken and matched packages list"), pkg_list_path,), 1296 ] 1297 mytxt = "%s:" % (purple(_("Dumping results into these files")),) 1298 self.updateProgress( 1299 mytxt, 1300 importance = 1, 1301 type = "info", 1302 header = blue(" @@ ") 1303 ) 1304 for txt, path in dmp_data: 1305 mytxt = "%s: %s" % (blue(txt), path,) 1306 self.updateProgress( 1307 mytxt, 1308 importance = 0, 1309 type = "info", 1310 header = darkgreen(" ## ") 1311 ) 1312 1313 1314 # load db 1315 dbconn = self.open_server_repository(read_only = True, 1316 no_upload = True, repo = repo) 1317 QA = self.QA() 1318 packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn, 1319 broken_symbols = True, dump_results_to_file = dump_results_to_file) 1320 if status != 0: 1321 return 1, None 1322 1323 if get_files: 1324 return 0, brokenexecs 1325 1326 if (not brokenexecs) and (not packages_matched): 1327 mytxt = "%s." % (_("System is healthy"),) 1328 self.updateProgress( 1329 blue(mytxt), 1330 importance = 2, 1331 type = "info", 1332 header = red(" @@ ") 1333 ) 1334 return 0, None 1335 1336 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),) 1337 self.updateProgress( 1338 blue(mytxt), 1339 importance = 1, 1340 type = "info", 1341 header = red(" @@ ") 1342 ) 1343 1344 packages = self.Spm().search_paths_owners(brokenexecs) 1345 1346 if packages: 1347 mytxt = "%s:" % (_("These are the matched packages"),) 1348 self.updateProgress( 1349 red(mytxt), 1350 importance = 1, 1351 type = "info", 1352 header = red(" @@ ") 1353 ) 1354 for my_atom, my_elf_id in packages: 1355 package_slot = my_atom, my_elf_id 1356 self.updateProgress( 1357 "%s [elf:%s]" % ( 1358 purple(my_atom), 1359 darkgreen(str(my_elf_id)), 1360 ), 1361 importance = 0, 1362 type = "info", 1363 header = red(" # ") 1364 ) 1365 for filename in sorted(packages[package_slot]): 1366 self.updateProgress( 1367 darkgreen(filename), 1368 importance = 0, 1369 type = "info", 1370 header = brown(" => ") 1371 ) 1372 1373 pkgstring_list = sorted(["%s:%s" % ( 1374 self.entropyTools.dep_getkey(x[0]), x[1],) for x \ 1375 in sorted(packages)]) 1376 if pkg_list_path is not None: 1377 with open(pkg_list_path, "w") as pkg_f: 1378 for pkgstr in pkgstring_list: 1379 pkg_f.write(pkgstr + "\n") 1380 pkg_f.flush() 1381 pkgstring = ' '.join(pkgstring_list) 1382 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,) 1383 self.updateProgress( 1384 mytxt, 1385 importance = 1, 1386 type = "info", 1387 header = red(" @@ ") 1388 ) 1389 else: 1390 self.updateProgress( 1391 red(_("No matched packages")), 1392 importance = 1, 1393 type = "info", 1394 header = red(" @@ ") 1395 ) 1396 1397 return 0, packages

1398

1399 - def orphaned_spm_packages_test(self):

1400 1401 mytxt = "%s %s" % ( 1402 blue(_("Running orphaned SPM packages test")), red("..."),) 1403 self.updateProgress( 1404 mytxt, 1405 importance = 2, 1406 type = "info", 1407 header = red(" @@ ") 1408 ) 1409 installed_packages = self.Spm().get_installed_packages() 1410 length = len(installed_packages) 1411 not_found = {} 1412 count = 0 1413 for installed_package in installed_packages: 1414 count += 1 1415 self.updateProgress( 1416 "%s: %s" % ( 1417 darkgreen(_("Scanning package")), 1418 brown(installed_package),), 1419 importance = 0, 1420 type = "info", 1421 back = True, 1422 count = (count, length), 1423 header = darkred(" @@ ") 1424 ) 1425 key, slot = (self.entropyTools.dep_getkey(installed_package), 1426 self.Spm().get_installed_package_metadata(installed_package, 1427 "SLOT"),) 1428 pkg_atom = "%s:%s" % (key, slot,) 1429 tree_atom = self.Spm().match_package(pkg_atom) 1430 if not tree_atom: 1431 not_found[installed_package] = pkg_atom 1432 self.updateProgress( 1433 "%s: %s" % ( 1434 blue(pkg_atom), 1435 darkred(_("not found anymore")), 1436 ), 1437 importance = 0, 1438 type = "warning", 1439 count = (count, length), 1440 header = darkred(" @@ ") 1441 ) 1442 1443 if not_found: 1444 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)]) 1445 self.updateProgress( 1446 "%s: %s" % ( 1447 blue(_("Packages string")), 1448 not_found_list, 1449 ), 1450 importance = 0, 1451 type = "warning", 1452 count = (count, length), 1453 header = darkred(" @@ ") 1454 ) 1455 1456 return not_found

1457

1458 - def depends_table_initialize(self, repo = None):

1459 dbconn = self.open_server_repository(read_only = False, 1460 no_upload = True, repo = repo) 1461 dbconn.regenerateReverseDependenciesMetadata() 1462 dbconn.taintDatabase() 1463 dbconn.commitChanges()

1464

1465 - def library_paths_table_initialize(self, repo = None):

1466 dbconn = self.open_server_repository(read_only = False, 1467 no_upload = True, repo = repo) 1468 dbconn.regenerateLibrarypathsidpackageTable() 1469 dbconn.taintDatabase() 1470 dbconn.commitChanges()

1471

1472 - def create_empty_database(self, dbpath = None, repo = None):

1473 if dbpath == None: 1474 dbpath = self.get_local_database_file(repo) 1475 1476 dbdir = os.path.dirname(dbpath) 1477 if not os.path.isdir(dbdir): 1478 os.makedirs(dbdir) 1479 1480 mytxt = red("%s ...") % (_("Initializing an empty database"),) 1481 self.updateProgress( 1482 mytxt, 1483 importance = 1, 1484 type = "info", 1485 header = darkgreen(" * "), 1486 back = True 1487 ) 1488 dbconn = self.ClientService.open_generic_database(dbpath) 1489 dbconn.initializeDatabase() 1490 dbconn.commitChanges() 1491 dbconn.closeDB() 1492 mytxt = "%s %s %s." % ( 1493 red(_("Entropy database file")), 1494 bold(dbpath), 1495 red(_("successfully initialized")), 1496 ) 1497 self.updateProgress( 1498 mytxt, 1499 importance = 1, 1500 type = "info", 1501 header = darkgreen(" * ") 1502 )

1503

1504 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

1505 1506 # check package_tag "no spaces" 1507 1508 try: 1509 package_tag = str(package_tag) 1510 if " " in package_tag: 1511 raise ValueError 1512 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,): 1513 self.updateProgress( 1514 "%s: %s" % ( 1515 blue(_("Invalid tag specified")), 1516 package_tag, 1517 ), 1518 importance = 1, type = "error", header = darkred(" !! ") 1519 ) 1520 return 1, package_tag 1521 1522 if repo == None: 1523 repo = self.default_repository 1524 1525 # sanity check 1526 invalid_atoms = [] 1527 dbconn = self.open_server_repository(read_only = True, 1528 no_upload = True, repo = repo) 1529 for idpackage in idpackages: 1530 ver_tag = dbconn.retrieveVersionTag(idpackage) 1531 if ver_tag: 1532 invalid_atoms.append(dbconn.retrieveAtom(idpackage)) 1533 1534 if invalid_atoms: 1535 self.updateProgress( 1536 "%s: %s" % ( 1537 blue(_("Packages already tagged, action aborted")), 1538 ', '.join([darkred(unicode(x)) for x in invalid_atoms]), 1539 ), 1540 importance = 1, type = "error", header = darkred(" !! ") 1541 ) 1542 return 2, invalid_atoms 1543 1544 matches = [(x, repo) for x in idpackages] 1545 status = 0 1546 data = self.move_packages( 1547 matches, to_repo = repo, from_repo = repo, ask = ask, 1548 do_copy = True, new_tag = package_tag 1549 ) 1550 return status, data

1551

1552 - def flushback_packages(self, from_branches, repo = None, ask = True):

1553 """ 1554 When creating a new branch, for space reasons, packages are not 1555 moved to a new location. This works fine until old branch is removed. 1556 To avoid inconsistences, before deciding to do that, all the packages 1557 in the old branch should be flushed back to the the currently configured 1558 branch. 1559 1560 @param from_branches -- list of branches to move packages from 1561 @type from_branches -- list 1562 @param repo -- repository to work on 1563 @type repo -- str 1564 @param ask -- user interactivity 1565 @type ask -- bool 1566 1567 @return status 1568 """ 1569 1570 status = True 1571 if repo == None: 1572 repo = self.default_repository 1573 branch = self.SystemSettings['repositories']['branch'] 1574 1575 if branch in from_branches: 1576 from_branches = [x for x in from_branches if x != branch] 1577 1578 self.updateProgress( 1579 "[%s=>%s|%s] %s" % ( 1580 darkgreen(', '.join(from_branches)), 1581 darkred(branch), 1582 brown(repo), 1583 blue(_("flushing back selected packages from branches")), 1584 ), 1585 importance = 2, 1586 type = "info", 1587 header = red(" @@ ") 1588 ) 1589 1590 dbconn = self.open_server_repository(read_only = True, 1591 no_upload = True, repo = repo) 1592 1593 idpackage_map = dict(((x, [],) for x in from_branches)) 1594 idpackages = dbconn.listAllIdpackages(order_by = 'atom') 1595 for idpackage in idpackages: 1596 download_url = dbconn.retrieveDownloadURL(idpackage) 1597 url_br = self.ClientService.get_branch_from_download_relative_uri( 1598 download_url) 1599 if url_br in from_branches: 1600 idpackage_map[url_br].append(idpackage) 1601 1602 mapped_branches = [x for x in idpackage_map if idpackage_map[x]] 1603 if not mapped_branches: 1604 self.updateProgress( 1605 "[%s=>%s|%s] %s !" % ( 1606 darkgreen(', '.join(from_branches)), 1607 darkred(branch), 1608 brown(repo), 1609 blue(_("nothing to do")), 1610 ), 1611 importance = 0, 1612 type = "warning", 1613 header = blue(" @@ ") 1614 ) 1615 return status 1616 1617 1618 all_fine = True 1619 tmp_down_dir = self.entropyTools.get_random_temp_file() 1620 os.makedirs(tmp_down_dir) 1621 1622 download_queue = {} 1623 local_up_dir = self.get_local_upload_directory(repo) 1624 local_basedir = os.path.join(local_up_dir, branch) 1625 dbconn = self.open_server_repository(read_only = False, 1626 no_upload = True, repo = repo) 1627 1628 def generate_queue(branch, repo, from_branch, down_q, idpackage_map): 1629 1630 self.updateProgress( 1631 "[%s=>%s|%s] %s" % ( 1632 darkgreen(from_branch), 1633 darkred(branch), 1634 brown(repo), 1635 brown(_("these are the packages that will be flushed")), 1636 ), 1637 importance = 1, 1638 type = "info", 1639 header = brown(" @@ ") 1640 ) 1641 1642 1643 for idpackage in idpackage_map[from_branch]: 1644 atom = dbconn.retrieveAtom(idpackage) 1645 self.updateProgress( 1646 "[%s=>%s|%s] %s" % ( 1647 darkgreen(from_branch), 1648 darkred(branch), 1649 brown(repo), 1650 purple(atom), 1651 ), 1652 importance = 0, 1653 type = "info", 1654 header = blue(" # ") 1655 ) 1656 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) 1657 pkg_fp = os.path.join(tmp_down_dir, pkg_fp) 1658 down_q.append((pkg_fp, idpackage,))

1659 1660 1661 for from_branch in sorted(mapped_branches): 1662 1663 download_queue[from_branch] = [] 1664 all_fine = False 1665 generate_queue(branch, repo, from_branch, 1666 download_queue[from_branch], idpackage_map) 1667 1668 if ask: 1669 rc_question = self.askQuestion( 1670 _("Would you like to continue ?")) 1671 if rc_question == "No": 1672 continue 1673 1674 remote_relative_path = self.get_remote_packages_relative_path(repo) 1675 1676 for uri in self.get_remote_mirrors(repo): 1677 1678 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1679 ftp_basedir = os.path.join(remote_relative_path, from_branch) 1680 1681 downloader_queue = [x[0] for x in download_queue[from_branch]] 1682 downloader = self.MirrorsService.FtpServerHandler( 1683 FtpInterface, 1684 self, 1685 [uri], 1686 downloader_queue, 1687 critical_files = downloader_queue, 1688 use_handlers = True, 1689 ftp_basedir = ftp_basedir, 1690 local_basedir = tmp_down_dir, 1691 download = True, 1692 repo = repo 1693 ) 1694 1695 errors, m_fine_uris, m_broken_uris = downloader.go() 1696 1697 if not errors: 1698 for downloaded_path, idpackage in \ 1699 download_queue[from_branch]: 1700 1701 self.updateProgress( 1702 "[%s=>%s|%s|%s] %s: %s" % ( 1703 darkgreen(from_branch), 1704 darkred(branch), 1705 brown(repo), 1706 dbconn.retrieveAtom(idpackage), 1707 blue(_("checking package hash")), 1708 darkgreen(os.path.basename(downloaded_path)), 1709 ), 1710 importance = 0, 1711 type = "info", 1712 header = brown(" "), 1713 back = True 1714 ) 1715 1716 md5hash = self.entropyTools.md5sum(downloaded_path) 1717 db_md5hash = dbconn.retrieveDigest(idpackage) 1718 if md5hash != db_md5hash: 1719 errors = True 1720 self.updateProgress( 1721 "[%s=>%s|%s|%s] %s: %s" % ( 1722 darkgreen(from_branch), 1723 darkred(branch), 1724 brown(repo), 1725 dbconn.retrieveAtom(idpackage), 1726 blue(_("hash does not match for")), 1727 darkgreen(os.path.basename(downloaded_path)), 1728 ), 1729 importance = 0, 1730 type = "error", 1731 header = brown(" ") 1732 ) 1733 continue 1734 1735 1736 1737 if errors: 1738 reason = _("wrong md5") 1739 if m_broken_uris: 1740 my_broken_uris = [ 1741 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 1742 for x, y in m_broken_uris] 1743 reason = my_broken_uris[0][1] 1744 1745 self.updateProgress( 1746 "[%s=>%s|%s] %s, %s: %s" % ( 1747 darkgreen(from_branch), 1748 darkred(branch), 1749 brown(repo), 1750 blue(_("download errors")), 1751 blue(_("reason")), 1752 reason, 1753 ), 1754 importance = 1, 1755 type = "error", 1756 header = darkred(" !!! ") 1757 ) 1758 # continuing if possible 1759 continue 1760 1761 all_fine = True 1762 1763 self.updateProgress( 1764 "[%s=>%s|%s] %s: %s" % ( 1765 darkgreen(from_branch), 1766 darkred(branch), 1767 brown(repo), 1768 blue(_("download completed successfully")), 1769 darkgreen(crippled_uri), 1770 ), 1771 importance = 1, 1772 type = "info", 1773 header = darkgreen(" * ") 1774 ) 1775 1776 if not all_fine: 1777 self.updateProgress( 1778 "[%s=>%s|%s] %s" % ( 1779 darkgreen(', '.join(from_branches)), 1780 darkred(branch), 1781 brown(repo), 1782 blue(_("error downloading packages from mirrors")), 1783 ), 1784 importance = 2, 1785 type = "error", 1786 header = darkred(" !!! ") 1787 ) 1788 return False 1789 1790 for from_branch in sorted(mapped_branches): 1791 1792 self.updateProgress( 1793 "[%s=>%s|%s] %s: %s" % ( 1794 darkgreen(from_branch), 1795 darkred(branch), 1796 brown(repo), 1797 blue(_("working on branch")), 1798 darkgreen(from_branch), 1799 ), 1800 importance = 1, 1801 type = "info", 1802 header = brown(" @@ ") 1803 ) 1804 1805 down_queue = download_queue[from_branch] 1806 for package_path, idpackage in down_queue: 1807 1808 self.updateProgress( 1809 "[%s=>%s|%s] %s: %s" % ( 1810 darkgreen(from_branch), 1811 darkred(branch), 1812 brown(repo), 1813 blue(_("updating package")), 1814 darkgreen(os.path.basename(package_path)), 1815 ), 1816 importance = 1, 1817 type = "info", 1818 header = brown(" "), 1819 back = True 1820 ) 1821 1822 # move files to upload 1823 package_name = os.path.basename(package_path) 1824 new_package_path = os.path.join(local_basedir, package_name) 1825 try: 1826 os.rename(package_path, new_package_path) 1827 except OSError: 1828 shutil.move(package_path, new_package_path) 1829 1830 # create md5 checksum 1831 self.entropyTools.create_md5_file(new_package_path) 1832 1833 # update database 1834 download_url = dbconn.retrieveDownloadURL(idpackage) 1835 download_url = \ 1836 self.ClientService.swap_branch_in_download_relative_uri( 1837 branch, download_url) 1838 dbconn.setDownloadURL(idpackage, download_url) 1839 dbconn.switchBranch(idpackage, branch) 1840 dbconn.commitChanges() 1841 1842 self.updateProgress( 1843 "[%s=>%s|%s] %s: %s" % ( 1844 darkgreen(from_branch), 1845 darkred(branch), 1846 brown(repo), 1847 blue(_("package flushed")), 1848 darkgreen(os.path.basename(package_path)), 1849 ), 1850 importance = 1, 1851 type = "info", 1852 header = brown(" ") 1853 ) 1854 1855 try: 1856 os.rmdir(tmp_down_dir) 1857 except OSError: 1858 pass 1859 1860 return True 1861

1862 - def move_packages(self, matches, to_repo, from_repo = None, ask = True, 1863 do_copy = False, new_tag = None, pull_deps = False):

1864 1865 if from_repo == None: 1866 from_repo = self.default_repository 1867 switched = set() 1868 1869 my_matches = list(matches) 1870 1871 # avoid setting __default__ as default server repo 1872 if etpConst['clientserverrepoid'] in (to_repo, from_repo): 1873 self.updateProgress( 1874 "%s: %s" % ( 1875 blue(_("Cannot touch system database")), 1876 red(etpConst['clientserverrepoid']), 1877 ), 1878 importance = 2, type = "warning", header = darkred(" @@ ") 1879 ) 1880 return switched 1881 1882 if not my_matches and from_repo: 1883 dbconn = self.open_server_repository(read_only = True, 1884 no_upload = True, repo = from_repo) 1885 my_matches = set( \ 1886 [(x, from_repo) for x in \ 1887 dbconn.listAllIdpackages()] 1888 ) 1889 1890 mytxt = _("Preparing to move selected packages to") 1891 if do_copy: 1892 mytxt = _("Preparing to copy selected packages to") 1893 self.updateProgress( 1894 "%s %s:" % ( 1895 blue(mytxt), 1896 red(to_repo), 1897 ), 1898 importance = 2, 1899 type = "info", 1900 header = red(" @@ ") 1901 ) 1902 self.updateProgress( 1903 "%s: %s" % ( 1904 bold(_("Note")), 1905 red(_("all old packages with conflicting scope will be " \ 1906 "removed from destination repo unless injected")), 1907 ), 1908 importance = 1, 1909 type = "info", 1910 header = red(" @@ ") 1911 ) 1912 1913 new_tag_string = '' 1914 if new_tag != None: 1915 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")), 1916 brown(new_tag),) 1917 1918 my_qa = self.QA() 1919 branch = self.SystemSettings['repositories']['branch'] 1920 pull_deps_matches = [] 1921 for idpackage, repo in my_matches: 1922 dbconn = self.open_server_repository(read_only = True, 1923 no_upload = True, repo = repo) 1924 self.updateProgress( 1925 "[%s=>%s|%s] %s " % ( 1926 darkgreen(repo), 1927 darkred(to_repo), 1928 brown(branch), 1929 blue(dbconn.retrieveAtom(idpackage)), 1930 ) + new_tag_string, 1931 importance = 0, 1932 type = "info", 1933 header = brown(" # ") 1934 ) 1935 # do we want to pull in also package dependencies? 1936 if pull_deps: 1937 dep_idpackages = my_qa.get_deep_dependency_list(dbconn, 1938 idpackage) 1939 for dep_idpackage in dep_idpackages: 1940 1941 my_dep_match = (dep_idpackage, repo,) 1942 if my_dep_match in pull_deps_matches: 1943 continue 1944 if my_dep_match in my_matches: 1945 continue 1946 1947 pull_deps_matches.append(my_dep_match) 1948 self.updateProgress( 1949 "[%s|%s] %s" % ( 1950 brown(branch), 1951 blue(_("dependency")), 1952 purple(dbconn.retrieveAtom(dep_idpackage)), 1953 ), 1954 importance = 0, 1955 type = "info", 1956 header = purple(" >> ") 1957 ) 1958 1959 if pull_deps: 1960 # put deps first! 1961 my_matches = pull_deps_matches + [x for x in my_matches if x not \ 1962 in pull_deps_matches] 1963 1964 if ask: 1965 rc_question = self.askQuestion(_("Would you like to continue ?")) 1966 if rc_question == "No": 1967 return switched 1968 1969 for idpackage, repo in my_matches: 1970 dbconn = self.open_server_repository(read_only = False, 1971 no_upload = True, repo = repo) 1972 match_branch = dbconn.retrieveBranch(idpackage) 1973 match_atom = dbconn.retrieveAtom(idpackage) 1974 package_filename = os.path.basename( 1975 dbconn.retrieveDownloadURL(idpackage)) 1976 self.updateProgress( 1977 "[%s=>%s|%s] %s: %s" % ( 1978 darkgreen(repo), 1979 darkred(to_repo), 1980 brown(branch), 1981 blue(_("switching")), 1982 darkgreen(match_atom), 1983 ), 1984 importance = 0, 1985 type = "info", 1986 header = red(" @@ "), 1987 back = True 1988 ) 1989 # move binary file 1990 from_file = os.path.join(self.get_local_packages_directory(repo), 1991 match_branch, package_filename) 1992 if not os.path.isfile(from_file): 1993 from_file = os.path.join(self.get_local_upload_directory(repo), 1994 match_branch, package_filename) 1995 if not os.path.isfile(from_file): 1996 self.updateProgress( 1997 "[%s=>%s|%s] %s: %s -> %s" % ( 1998 darkgreen(repo), 1999 darkred(to_repo), 2000 brown(branch), 2001 bold(_("cannot switch, package not found, skipping")), 2002 darkgreen(match_atom), 2003 red(from_file), 2004 ), 2005 importance = 1, 2006 type = "warning", 2007 header = darkred(" !!! ") 2008 ) 2009 continue 2010 2011 if new_tag != None: 2012 match_category = dbconn.retrieveCategory(idpackage) 2013 match_name = dbconn.retrieveName(idpackage) 2014 match_version = dbconn.retrieveVersion(idpackage) 2015 tagged_package_filename = \ 2016 self.entropyTools.create_package_filename( 2017 match_category, match_name, match_version, new_tag) 2018 to_file = os.path.join(self.get_local_upload_directory(to_repo), 2019 match_branch, tagged_package_filename) 2020 else: 2021 to_file = os.path.join(self.get_local_upload_directory(to_repo), 2022 match_branch, package_filename) 2023 if not os.path.isdir(os.path.dirname(to_file)): 2024 os.makedirs(os.path.dirname(to_file)) 2025 2026 copy_data = [ 2027 (from_file, to_file,), 2028 (from_file + etpConst['packagesmd5fileext'], 2029 to_file + etpConst['packagesmd5fileext'],), 2030 (from_file + etpConst['packagesexpirationfileext'], 2031 to_file + etpConst['packagesexpirationfileext'],) 2032 ] 2033 2034 for from_item, to_item in copy_data: 2035 self.updateProgress( 2036 "[%s=>%s|%s] %s: %s" % ( 2037 darkgreen(repo), 2038 darkred(to_repo), 2039 brown(branch), 2040 blue(_("moving file")), 2041 darkgreen(os.path.basename(from_item)), 2042 ), 2043 importance = 0, 2044 type = "info", 2045 header = red(" @@ "), 2046 back = True 2047 ) 2048 if os.path.isfile(from_item): 2049 shutil.copy2(from_item, to_item) 2050 2051 self.updateProgress( 2052 "[%s=>%s|%s] %s: %s" % ( 2053 darkgreen(repo), 2054 darkred(to_repo), 2055 brown(branch), 2056 blue(_("loading data from source database")), 2057 darkgreen(repo), 2058 ), 2059 importance = 0, 2060 type = "info", 2061 header = red(" @@ "), 2062 back = True 2063 ) 2064 # install package into destination db 2065 data = dbconn.getPackageData(idpackage) 2066 if new_tag != None: 2067 data['versiontag'] = new_tag 2068 2069 todbconn = self.open_server_repository(read_only = False, 2070 no_upload = True, repo = to_repo) 2071 2072 self.updateProgress( 2073 "[%s=>%s|%s] %s: %s" % ( 2074 darkgreen(repo), 2075 darkred(to_repo), 2076 brown(branch), 2077 blue(_("injecting data to destination database")), 2078 darkgreen(to_repo), 2079 ), 2080 importance = 0, 2081 type = "info", 2082 header = red(" @@ "), 2083 back = True 2084 ) 2085 new_idpackage, new_revision, new_data = todbconn.handlePackage(data) 2086 del data 2087 todbconn.commitChanges() 2088 2089 if not do_copy: 2090 self.updateProgress( 2091 "[%s=>%s|%s] %s: %s" % ( 2092 darkgreen(repo), 2093 darkred(to_repo), 2094 brown(branch), 2095 blue(_("removing entry from source database")), 2096 darkgreen(repo), 2097 ), 2098 importance = 0, 2099 type = "info", 2100 header = red(" @@ "), 2101 back = True 2102 ) 2103 2104 # remove package from old db 2105 dbconn.removePackage(idpackage) 2106 dbconn.commitChanges() 2107 2108 self.updateProgress( 2109 "[%s=>%s|%s] %s: %s" % ( 2110 darkgreen(repo), 2111 darkred(to_repo), 2112 brown(branch), 2113 blue(_("successfully handled atom")), 2114 darkgreen(match_atom), 2115 ), 2116 importance = 0, 2117 type = "info", 2118 header = blue(" @@ ") 2119 ) 2120 switched.add((idpackage, repo,)) 2121 2122 # just run this to make dev aware 2123 self.dependencies_test(to_repo) 2124 2125 return switched

2126 2127

2128 - def package_injector(self, package_file, inject = False, repo = None):

2129 2130 if repo == None: 2131 repo = self.default_repository 2132 2133 upload_dir = os.path.join(self.get_local_upload_directory(repo), 2134 self.SystemSettings['repositories']['branch']) 2135 if not os.path.isdir(upload_dir): 2136 os.makedirs(upload_dir) 2137 2138 dbconn = self.open_server_repository(read_only = False, 2139 no_upload = True, repo = repo) 2140 self.updateProgress( 2141 red("[repo: %s] %s: %s" % ( 2142 darkgreen(repo), 2143 _("adding package"), 2144 bold(os.path.basename(package_file)), 2145 ) 2146 ), 2147 importance = 1, 2148 type = "info", 2149 header = brown(" * "), 2150 back = True 2151 ) 2152 mydata = self.Spm().extract_package_metadata(package_file) 2153 mydata['injected'] = inject 2154 idpackage, revision, mydata = dbconn.handlePackage(mydata) 2155 2156 # set trashed counters 2157 trashing_counters = set() 2158 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2159 myserver_repos = srv_set['repositories'].keys() 2160 for myrepo in myserver_repos: 2161 mydbconn = self.open_server_repository(read_only = True, 2162 no_upload = True, repo = myrepo) 2163 mylist = mydbconn.retrieve_packages_to_remove( 2164 mydata['name'], 2165 mydata['category'], 2166 mydata['slot'], 2167 mydata['injected'] 2168 ) 2169 for myitem in mylist: 2170 trashing_counters.add(mydbconn.retrieveSpmUid(myitem)) 2171 2172 for mycounter in trashing_counters: 2173 dbconn.setTrashedUid(mycounter) 2174 2175 # add package info to our current server repository 2176 dbconn.dropInstalledPackageFromStore(idpackage) 2177 dbconn.storeInstalledPackage(idpackage, repo) 2178 atom = dbconn.retrieveAtom(idpackage) 2179 2180 self.updateProgress( 2181 "[repo:%s] %s: %s %s: %s" % ( 2182 darkgreen(repo), 2183 blue(_("added package")), 2184 darkgreen(atom), 2185 blue(_("rev")), # as in revision 2186 bold(str(revision)), 2187 ), 2188 importance = 1, 2189 type = "info", 2190 header = red(" @@ ") 2191 ) 2192 2193 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage)) 2194 if manual_deps: 2195 self.updateProgress( 2196 "[repo:%s] %s: %s" % ( 2197 darkgreen(repo), 2198 blue(_("manual dependencies for")), 2199 darkgreen(atom), 2200 ), 2201 importance = 1, 2202 type = "warning", 2203 header = darkgreen(" ## ") 2204 ) 2205 for m_dep in manual_deps: 2206 self.updateProgress( 2207 brown(m_dep), 2208 importance = 1, 2209 type = "warning", 2210 header = darkred(" # ") 2211 ) 2212 2213 download_url = self._setup_repository_package_filename(idpackage, 2214 repo = repo) 2215 downloadfile = os.path.basename(download_url) 2216 destination_path = os.path.join(upload_dir, downloadfile) 2217 try: 2218 os.rename(package_file, destination_path) 2219 except OSError: 2220 shutil.move(package_file, destination_path) 2221 2222 dbconn.commitChanges() 2223 return idpackage, destination_path

2224 2225 # this function changes the final repository package filename

2226 - def _setup_repository_package_filename(self, idpackage, repo = None):

2227 2228 dbconn = self.open_server_repository(read_only = False, 2229 no_upload = True, repo = repo) 2230 2231 downloadurl = dbconn.retrieveDownloadURL(idpackage) 2232 packagerev = dbconn.retrieveRevision(idpackage) 2233 downloaddir = os.path.dirname(downloadurl) 2234 downloadfile = os.path.basename(downloadurl) 2235 # add revision 2236 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev, 2237 etpConst['packagesext'],) 2238 downloadurl = os.path.join(downloaddir, downloadfile) 2239 2240 # update url 2241 dbconn.setDownloadURL(idpackage, downloadurl) 2242 2243 return downloadurl

2244

2245 - def add_packages_to_repository(self, packages_data, ask = True, 2246 repo = None):

2247 2248 if repo == None: 2249 repo = self.default_repository 2250 2251 mycount = 0 2252 maxcount = len(packages_data) 2253 idpackages_added = set() 2254 to_be_injected = set() 2255 my_qa = self.QA() 2256 missing_deps_taint = False 2257 for package_filepath, inject in packages_data: 2258 2259 mycount += 1 2260 self.updateProgress( 2261 "[repo:%s] %s: %s" % ( 2262 darkgreen(repo), 2263 blue(_("adding package")), 2264 darkgreen(os.path.basename(package_filepath)), 2265 ), 2266 importance = 1, 2267 type = "info", 2268 header = blue(" @@ "), 2269 count = (mycount, maxcount,) 2270 ) 2271 2272 try: 2273 # add to database 2274 idpackage, destination_path = self.package_injector( 2275 package_filepath, 2276 inject = inject, 2277 repo = repo 2278 ) 2279 idpackages_added.add(idpackage) 2280 to_be_injected.add((idpackage, destination_path)) 2281 except Exception, err: 2282 self.entropyTools.print_traceback() 2283 self.updateProgress( 2284 "[repo:%s] %s: %s" % ( 2285 darkgreen(repo), 2286 darkred(_("Exception caught, closing tasks")), 2287 darkgreen(unicode(err)), 2288 ), 2289 importance = 1, 2290 type = "error", 2291 header = bold(" !!! "), 2292 count = (mycount, maxcount,) 2293 ) 2294 # reinit depends table 2295 self.depends_table_initialize(repo) 2296 # reinit librarypathsidpackage table 2297 self.library_paths_table_initialize(repo) 2298 if idpackages_added: 2299 dbconn = self.open_server_repository(read_only = False, 2300 no_upload = True, repo = repo) 2301 missing_deps_taint = my_qa.test_missing_dependencies( 2302 idpackages_added, 2303 dbconn, 2304 ask = ask, 2305 repo = repo, 2306 self_check = True, 2307 black_list = \ 2308 self.get_missing_dependencies_blacklist( 2309 repo = repo), 2310 black_list_adder = \ 2311 self.add_missing_dependencies_blacklist_items 2312 ) 2313 my_qa.test_depends_linking(idpackages_added, dbconn, 2314 repo = repo) 2315 if to_be_injected: 2316 self.inject_database_into_packages(to_be_injected, 2317 repo = repo) 2318 # reinit depends table 2319 if missing_deps_taint: 2320 self.depends_table_initialize(repo) 2321 self.close_server_databases() 2322 raise 2323 2324 # reinit depends table 2325 self.depends_table_initialize(repo) 2326 # reinit librarypathsidpackage table 2327 self.library_paths_table_initialize(repo) 2328 2329 if idpackages_added: 2330 dbconn = self.open_server_repository(read_only = False, 2331 no_upload = True, repo = repo) 2332 missing_deps_taint = my_qa.test_missing_dependencies( 2333 idpackages_added, 2334 dbconn, 2335 ask = ask, 2336 repo = repo, 2337 self_check = True, 2338 black_list = \ 2339 self.get_missing_dependencies_blacklist(repo = repo), 2340 black_list_adder = \ 2341 self.add_missing_dependencies_blacklist_items 2342 ) 2343 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo) 2344 2345 # reinit depends table 2346 if missing_deps_taint: 2347 self.depends_table_initialize(repo) 2348 2349 # inject database into packages 2350 self.inject_database_into_packages(to_be_injected, repo = repo) 2351 2352 return idpackages_added

2353 2354

2355 - def inject_database_into_packages(self, injection_data, repo = None):

2356 2357 if repo == None: 2358 repo = self.default_repository 2359 2360 # now inject metadata into tbz2 packages 2361 self.updateProgress( 2362 "[repo:%s] %s:" % ( 2363 darkgreen(repo), 2364 blue(_("Injecting entropy metadata into built packages")), 2365 ), 2366 importance = 1, 2367 type = "info", 2368 header = red(" @@ ") 2369 ) 2370 2371 dbconn = self.open_server_repository(read_only = False, 2372 no_upload = True, repo = repo) 2373 for idpackage, package_path in injection_data: 2374 self.updateProgress( 2375 "[repo:%s|%s] %s: %s" % ( 2376 darkgreen(repo), 2377 brown(str(idpackage)), 2378 blue(_("injecting entropy metadata")), 2379 darkgreen(os.path.basename(package_path)), 2380 ), 2381 importance = 1, 2382 type = "info", 2383 header = blue(" @@ "), 2384 back = True 2385 ) 2386 data = dbconn.getPackageData(idpackage) 2387 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2388 dbpath = self.ClientService.inject_entropy_database_into_package( 2389 package_path, data, treeupdates_actions) 2390 digest = self.entropyTools.md5sum(package_path) 2391 # update digest 2392 dbconn.setDigest(idpackage, digest) 2393 # update signatures 2394 signatures = data['signatures'].copy() 2395 for hash_key in sorted(signatures): 2396 hash_func = getattr(self.entropyTools, hash_key) 2397 signatures[hash_key] = hash_func(package_path) 2398 dbconn.setSignatures(idpackage, signatures['sha1'], 2399 signatures['sha256'], signatures['sha512']) 2400 self.entropyTools.create_md5_file(package_path) 2401 # remove garbage 2402 os.remove(dbpath) 2403 self.updateProgress( 2404 "[repo:%s|%s] %s: %s" % ( 2405 darkgreen(repo), 2406 brown(str(idpackage)), 2407 blue(_("injection complete")), 2408 darkgreen(os.path.basename(package_path)), 2409 ), 2410 importance = 1, 2411 type = "info", 2412 header = red(" @@ ") 2413 ) 2414 dbconn.commitChanges()

2415

2416 - def check_config_file_updates(self):

2417 self.updateProgress( 2418 "[%s] %s" % ( 2419 red(_("config files")), # something short please 2420 blue(_("checking system")), 2421 ), 2422 importance = 1, 2423 type = "info", 2424 header = blue(" @@ "), 2425 back = True 2426 ) 2427 # scanning for config files not updated 2428 scandata = self.ClientService.FileUpdates.scanfs(dcache = False) 2429 if scandata: 2430 self.updateProgress( 2431 "[%s] %s" % ( 2432 red(_("config files")), # something short please 2433 blue(_("there are configuration files not updated yet")), 2434 ), 2435 importance = 1, 2436 type = "error", 2437 header = darkred(" @@ ") 2438 ) 2439 for key in scandata: 2440 self.updateProgress( 2441 "%s" % (brown(etpConst['systemroot'] + \ 2442 scandata[key]['destination'])), 2443 importance = 1, 2444 type = "info", 2445 header = "\t" 2446 ) 2447 return True 2448 return False

2449

2450 - def quickpkg(self, atom, storedir):

2451 return self.Spm().generate_package(atom, storedir)

2452

2453 - def remove_packages(self, idpackages, repo = None):

2454 2455 if repo == None: 2456 repo = self.default_repository 2457 2458 dbconn = self.open_server_repository(read_only = False, 2459 no_upload = True, repo = repo) 2460 for idpackage in idpackages: 2461 atom = dbconn.retrieveAtom(idpackage) 2462 self.updateProgress( 2463 "[repo:%s] %s: %s" % ( 2464 darkgreen(repo), 2465 blue(_("removing package")), 2466 darkgreen(atom), 2467 ), 2468 importance = 1, 2469 type = "info", 2470 header = brown(" @@ ") 2471 ) 2472 dbconn.removePackage(idpackage) 2473 self.close_server_database(dbconn) 2474 self.updateProgress( 2475 "[repo:%s] %s" % ( 2476 darkgreen(repo), 2477 blue(_("removal complete")), 2478 ), 2479 importance = 1, 2480 type = "info", 2481 header = brown(" @@ ") 2482 )

2483 2484

2485 - def bump_database(self, repo = None):

2486 dbconn = self.open_server_repository(read_only = False, 2487 no_upload = True, repo = repo) 2488 dbconn.taintDatabase() 2489 self.close_server_database(dbconn)

2490

2491 - def get_remote_mirrors(self, repo = None):

2492 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2493 if repo == None: 2494 repo = self.default_repository 2495 return srv_set['repositories'][repo]['mirrors'][:]

2496

2497 - def get_remote_packages_relative_path(self, repo = None):

2498 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2499 if repo == None: 2500 repo = self.default_repository 2501 return srv_set['repositories'][repo]['packages_relative_path']

2502

2503 - def get_remote_database_relative_path(self, repo = None):

2504 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2505 if repo == None: 2506 repo = self.default_repository 2507 return srv_set['repositories'][repo]['database_relative_path']

2508

2509 - def get_local_database_file(self, repo = None, branch = None):

2510 if repo == None: 2511 repo = self.default_repository 2512 return os.path.join(self.get_local_database_dir(repo, branch), 2513 etpConst['etpdatabasefile'])

2514

2515 - def get_local_store_directory(self, repo = None):

2516 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2517 if repo == None: 2518 repo = self.default_repository 2519 return srv_set['repositories'][repo]['store_dir']

2520

2521 - def get_local_upload_directory(self, repo = None):

2522 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2523 if repo == None: 2524 repo = self.default_repository 2525 return srv_set['repositories'][repo]['upload_dir']

2526

2527 - def get_local_packages_directory(self, repo = None):

2528 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2529 if repo == None: 2530 repo = self.default_repository 2531 return srv_set['repositories'][repo]['packages_dir']

2532

2533 - def get_local_database_taint_file(self, repo = None, branch = None):

2534 if repo == None: 2535 repo = self.default_repository 2536 return os.path.join(self.get_local_database_dir(repo, branch), 2537 etpConst['etpdatabasetaintfile'])

2538

2539 - def get_local_database_revision_file(self, repo = None, branch = None):

2540 if repo == None: 2541 repo = self.default_repository 2542 return os.path.join(self.get_local_database_dir(repo, branch), 2543 etpConst['etpdatabaserevisionfile'])

2544

2545 - def get_local_database_timestamp_file(self, repo = None, branch = None):

2546 if repo == None: 2547 repo = self.default_repository 2548 return os.path.join(self.get_local_database_dir(repo, branch), 2549 etpConst['etpdatabasetimestampfile'])

2550

2551 - def get_local_database_ca_cert_file(self, repo = None, branch = None):

2552 if repo == None: 2553 repo = self.default_repository 2554 return os.path.join(self.get_local_database_dir(repo, branch), 2555 etpConst['etpdatabasecacertfile'])

2556

2557 - def get_local_database_server_cert_file(self, repo = None, branch = None):

2558 if repo == None: 2559 repo = self.default_repository 2560 return os.path.join(self.get_local_database_dir(repo, branch), 2561 etpConst['etpdatabaseservercertfile'])

2562

2563 - def get_local_database_mask_file(self, repo = None, branch = None):

2564 if repo == None: 2565 repo = self.default_repository 2566 return os.path.join(self.get_local_database_dir(repo, branch), 2567 etpConst['etpdatabasemaskfile'])

2568

2569 - def get_local_database_system_mask_file(self, repo = None, branch = None):

2570 if repo == None: 2571 repo = self.default_repository 2572 return os.path.join(self.get_local_database_dir(repo, branch), 2573 etpConst['etpdatabasesytemmaskfile'])

2574

2575 - def get_local_database_confl_tagged_file(self, repo = None, branch = None):

2576 if repo == None: 2577 repo = self.default_repository 2578 return os.path.join(self.get_local_database_dir(repo, branch), 2579 etpConst['etpdatabaseconflictingtaggedfile'])

2580

2581 - def get_local_database_licensewhitelist_file(self, repo = None, 2582 branch = None):

2583 2584 if repo == None: 2585 repo = self.default_repository 2586 return os.path.join(self.get_local_database_dir(repo, branch), 2587 etpConst['etpdatabaselicwhitelistfile'])

2588

2589 - def get_local_database_rss_file(self, repo = None, branch = None):

2590 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2591 if repo == None: 2592 repo = self.default_repository 2593 return os.path.join(self.get_local_database_dir(repo, branch), 2594 srv_set['rss']['name'])

2595

2596 - def get_local_database_rsslight_file(self, repo = None, branch = None):

2597 if repo == None: 2598 repo = self.default_repository 2599 return os.path.join(self.get_local_database_dir(repo, branch), 2600 etpConst['rss-light-name'])

2601

2602 - def get_local_database_notice_board_file(self, repo = None, branch = None):

2603 if repo == None: 2604 repo = self.default_repository 2605 return os.path.join(self.get_local_database_dir(repo, branch), 2606 etpConst['rss-notice-board'])

2607

2608 - def get_local_database_treeupdates_file(self, repo = None, branch = None):

2609 if repo == None: 2610 repo = self.default_repository 2611 return os.path.join(self.get_local_database_dir(repo, branch), 2612 etpConst['etpdatabaseupdatefile'])

2613

2614 - def get_local_database_compressed_metafiles_file(self, repo = None, 2615 branch = None):

2616 2617 if repo == None: 2618 repo = self.default_repository 2619 return os.path.join(self.get_local_database_dir(repo, branch), 2620 etpConst['etpdatabasemetafilesfile'])

2621

2622 - def get_local_database_metafiles_not_found_file(self, repo = None, 2623 branch = None):

2624 2625 if repo == None: 2626 repo = self.default_repository 2627 return os.path.join(self.get_local_database_dir(repo, branch), 2628 etpConst['etpdatabasemetafilesnotfound'])

2629

2630 - def get_local_exp_based_pkgs_rm_whitelist_file(self, repo = None, 2631 branch = None):

2632 if repo == None: 2633 repo = self.default_repository 2634 return os.path.join(self.get_local_database_dir(repo, branch), 2635 etpConst['etpdatabaseexpbasedpkgsrm'])

2636

2637 - def get_local_pkglist_file(self, repo = None, branch = None):

2638 if repo == None: 2639 repo = self.default_repository 2640 return os.path.join(self.get_local_database_dir(repo, branch), 2641 etpConst['etpdatabasepkglist'])

2642

2643 - def get_local_database_sets_dir(self, repo = None, branch = None):

2644 if repo == None: 2645 repo = self.default_repository 2646 return os.path.join(self.get_local_database_dir(repo, branch), 2647 etpConst['confsetsdirname'])

2648

2649 - def get_local_post_branch_mig_script(self, repo = None, branch = None):

2650 if repo == None: 2651 repo = self.default_repository 2652 return os.path.join(self.get_local_database_dir(repo, branch), 2653 etpConst['etp_post_branch_hop_script'])

2654

2655 - def get_local_post_branch_upg_script(self, repo = None, branch = None):

2656 if repo == None: 2657 repo = self.default_repository 2658 return os.path.join(self.get_local_database_dir(repo, branch), 2659 etpConst['etp_post_branch_upgrade_script'])

2660

2661 - def get_local_critical_updates_file(self, repo = None, branch = None):

2662 if repo == None: 2663 repo = self.default_repository 2664 return os.path.join(self.get_local_database_dir(repo, branch), 2665 etpConst['etpdatabasecriticalfile'])

2666

2667 - def get_local_database_keywords_file(self, repo = None, branch = None):

2668 if repo == None: 2669 repo = self.default_repository 2670 return os.path.join(self.get_local_database_dir(repo, branch), 2671 etpConst['etpdatabasekeywordsfile'])

2672

2673 - def get_local_database_dir(self, repo = None, branch = None):

2674 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2675 if repo == None: 2676 repo = self.default_repository 2677 if branch == None: 2678 branch = self.SystemSettings['repositories']['branch'] 2679 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2680 branch)

2681

2682 - def get_missing_dependencies_blacklist_file(self, repo = None, 2683 branch = None):

2684 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2685 if repo == None: 2686 repo = self.default_repository 2687 if branch == None: 2688 branch = self.SystemSettings['repositories']['branch'] 2689 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2690 branch, etpConst['etpdatabasemissingdepsblfile'])

2691

2692 - def get_missing_dependencies_blacklist(self, repo = None, branch = None):

2693 if repo == None: 2694 repo = self.default_repository 2695 if branch == None: 2696 branch = self.SystemSettings['repositories']['branch'] 2697 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2698 wl_data = [] 2699 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK): 2700 f_wl = open(wl_file,"r") 2701 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \ 2702 not x.strip().startswith("#")] 2703 f_wl.close() 2704 return set(wl_data)

2705

2706 - def add_missing_dependencies_blacklist_items(self, items, repo = None, 2707 branch = None):

2708 2709 if repo == None: 2710 repo = self.default_repository 2711 if branch == None: 2712 branch = self.SystemSettings['repositories']['branch'] 2713 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2714 wl_dir = os.path.dirname(wl_file) 2715 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)): 2716 return 2717 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK): 2718 return 2719 f_wl = open(wl_file,"a+") 2720 f_wl.write('\n'.join(items)+'\n') 2721 f_wl.flush() 2722 f_wl.close()

2723

2724 - def get_local_database_revision(self, repo = None):

2725 2726 if repo == None: 2727 repo = self.default_repository 2728 2729 dbrev_file = self.get_local_database_revision_file(repo) 2730 if os.path.isfile(dbrev_file): 2731 f_rev = open(dbrev_file) 2732 rev = f_rev.readline().strip() 2733 f_rev.close() 2734 try: 2735 rev = int(rev) 2736 except ValueError: 2737 self.updateProgress( 2738 "[repo:%s] %s: %s - %s" % ( 2739 darkgreen(repo), 2740 blue(_("invalid database revision")), 2741 bold(rev), 2742 blue(_("defaulting to 0")), 2743 ), 2744 importance = 2, 2745 type = "error", 2746 header = darkred(" !!! ") 2747 ) 2748 rev = 0 2749 return rev 2750 else: 2751 return 0

2752

2753 - def get_remote_database_revision(self, repo = None):

2754 2755 if repo == None: 2756 repo = self.default_repository 2757 2758 remote_status = self.MirrorsService.get_remote_databases_status(repo) 2759 if not [x for x in remote_status if x[1]]: 2760 remote_revision = 0 2761 else: 2762 remote_revision = max([x[1] for x in remote_status]) 2763 2764 return remote_revision

2765

2766 - def get_branch_from_download_relative_uri(self, mypath):

2767 return self.ClientService.get_branch_from_download_relative_uri(mypath)

2768

2769 - def get_current_timestamp(self):

2770 from datetime import datetime 2771 import time 2772 return "%s" % (datetime.fromtimestamp(time.time()),)

2773

2774 - def create_repository_pkglist(self, repo = None, branch = None):

2775 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch) 2776 2777 tmp_pkglist_file = pkglist_file + ".tmp" 2778 dbconn = self.open_server_repository(repo = repo, just_reading = True, 2779 do_treeupdates = False) 2780 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True) 2781 2782 with open(tmp_pkglist_file, "w") as pkg_f: 2783 for pkg in pkglist: 2784 pkg_f.write(pkg + "\n") 2785 pkg_f.flush() 2786 2787 os.rename(tmp_pkglist_file, pkglist_file)

2788

2789 - def package_set_list(self, *args, **kwargs):

2790 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2791 repos = srv_set['repositories'].keys() 2792 kwargs['server_repos'] = repos 2793 kwargs['serverInstance'] = self 2794 return self.ClientService.package_set_list(*args, **kwargs)

2795

2796 - def package_set_search(self, *args, **kwargs):

2797 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2798 repos = srv_set['repositories'].keys() 2799 kwargs['server_repos'] = repos 2800 kwargs['serverInstance'] = self 2801 return self.ClientService.package_set_search(*args, **kwargs)

2802

2803 - def package_set_match(self, *args, **kwargs):

2804 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2805 repos = srv_set['repositories'].keys() 2806 kwargs['server_repos'] = repos 2807 kwargs['serverInstance'] = self 2808 return self.ClientService.package_set_match(*args, **kwargs)

2809

2810 - def atom_match(self, *args, **kwargs):

2811 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2812 repos = srv_set['repositories'].keys() 2813 kwargs['server_repos'] = repos 2814 kwargs['serverInstance'] = self 2815 return self.ClientService.atom_match(*args, **kwargs)

2816

2817 - def scan_package_changes(self):

2818 2819 spm = self.Spm() 2820 2821 spm_packages = spm.get_installed_packages() 2822 installed_packages = [] 2823 for spm_package in spm_packages: 2824 pkg_counter = spm.get_installed_package_metadata(spm_package, 2825 "COUNTER") 2826 try: 2827 pkg_counter = int(pkg_counter) 2828 except ValueError: 2829 continue 2830 installed_packages.append((spm_package, pkg_counter,)) 2831 2832 installed_counters = set() 2833 to_be_added = set() 2834 to_be_removed = set() 2835 to_be_injected = set() 2836 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2837 exp_based_scope = my_settings['exp_based_scope'] 2838 2839 server_repos = my_settings['repositories'].keys() 2840 2841 # packages to be added 2842 for spm_atom, spm_counter in installed_packages: 2843 found = False 2844 for server_repo in server_repos: 2845 installed_counters.add(spm_counter) 2846 server_dbconn = self.open_server_repository(read_only = True, 2847 no_upload = True, repo = server_repo) 2848 counter = server_dbconn.isSpmUidAvailable(spm_counter) 2849 if counter: 2850 found = True 2851 break 2852 if not found: 2853 to_be_added.add((spm_atom, spm_counter,)) 2854 2855 # packages to be removed from the database 2856 database_counters = {} 2857 for server_repo in server_repos: 2858 server_dbconn = self.open_server_repository(read_only = True, 2859 no_upload = True, repo = server_repo) 2860 database_counters[server_repo] = server_dbconn.listAllSpmUids() 2861 2862 ordered_counters = set() 2863 for server_repo in database_counters: 2864 for data in database_counters[server_repo]: 2865 ordered_counters.add((data, server_repo)) 2866 database_counters = ordered_counters 2867 2868 for (counter, idpackage,), xrepo in database_counters: 2869 2870 if counter < 0: 2871 continue # skip packages without valid counter 2872 2873 if counter in installed_counters: 2874 continue 2875 2876 dbconn = self.open_server_repository(read_only = True, 2877 no_upload = True, repo = xrepo) 2878 2879 dorm = True 2880 # check if the package is in to_be_added 2881 if to_be_added: 2882 2883 dorm = False 2884 atom = dbconn.retrieveAtom(idpackage) 2885 atomkey = self.entropyTools.dep_getkey(atom) 2886 atomtag = self.entropyTools.dep_gettag(atom) 2887 atomslot = dbconn.retrieveSlot(idpackage) 2888 2889 add = True 2890 for spm_atom, spm_counter in to_be_added: 2891 addslot = self.Spm().get_installed_package_metadata( 2892 spm_atom, "SLOT") 2893 addkey = self.entropyTools.dep_getkey(spm_atom) 2894 # workaround for ebuilds not having slot 2895 if addslot == None: 2896 addslot = '0' 2897 # atomtag != None is for handling tagged pkgs correctly 2898 if (atomkey == addkey) and \ 2899 ((str(atomslot) == str(addslot)) or (atomtag != None)): 2900 # do not add to to_be_removed 2901 add = False 2902 break 2903 2904 if not add: 2905 continue 2906 dorm = True 2907 2908 # checking if we are allowed to remove stuff on this repo 2909 # it xrepo is not the default one, we MUST skip this to 2910 # avoid touching what developer doesn't expect 2911 if dorm and (xrepo == self.default_repository): 2912 trashed = self.is_counter_trashed(counter) 2913 if trashed: 2914 # search into portage then 2915 try: 2916 key, slot = dbconn.retrieveKeySlot(idpackage) 2917 trashed = self.Spm().match_installed_package( 2918 key+":"+slot) 2919 except TypeError: # referred to retrieveKeySlot 2920 trashed = True 2921 if not trashed: 2922 2923 dbtag = dbconn.retrieveVersionTag(idpackage) 2924 if dbtag: 2925 is_injected = dbconn.isInjected(idpackage) 2926 if not is_injected: 2927 to_be_injected.add((idpackage, xrepo)) 2928 2929 elif exp_based_scope: 2930 2931 # check if support for this is set 2932 plg_id = self.sys_settings_fatscope_plugin_id 2933 exp_data = self.SystemSettings[plg_id]['repos'].get( 2934 xrepo, set()) 2935 2936 # only some packages are set, check if our is 2937 # in the list 2938 if (idpackage not in exp_data) and (-1 not in exp_data): 2939 to_be_removed.add((idpackage, xrepo)) 2940 continue 2941 2942 idpackage_expired = self.is_match_expired((idpackage, 2943 xrepo,)) 2944 2945 if idpackage_expired: 2946 # expired !!! 2947 # add this and its depends (reverse deps) 2948 to_be_removed.add((idpackage, xrepo)) 2949 for my_id in dbconn.retrieveReverseDependencies( 2950 idpackage): 2951 to_be_removed.add((my_id, xrepo)) 2952 2953 else: 2954 to_be_removed.add((idpackage, xrepo)) 2955 2956 return to_be_added, to_be_removed, to_be_injected

2957

2958 - def is_match_expired(self, match):

2959 2960 idpackage, repoid = match 2961 dbconn = self.open_server_repository(repo = repoid, just_reading = True) 2962 # 3600 * 24 = 86400 2963 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2964 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400 2965 cur_unix_time = self.entropyTools.get_current_unix_time() 2966 # if packages removal is triggered by expiration 2967 # we will have to check if our package is really 2968 # expired and remove its reverse deps too 2969 mydate = dbconn.retrieveCreationDate(idpackage) 2970 # cross fingers hoping that time is set correctly 2971 mydelta = cur_unix_time - float(mydate) 2972 if mydelta > pkg_exp_secs: 2973 return True 2974 return False

2975

2976 - def is_counter_trashed(self, counter):

2977 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2978 server_repos = srv_set['repositories'].keys() 2979 for repo in server_repos: 2980 dbconn = self.open_server_repository(read_only = True, 2981 no_upload = True, repo = repo) 2982 if dbconn.isSpmUidTrashed(counter): 2983 return True 2984 return False

2985

2986 - def transform_package_into_injected(self, idpackage, repo = None):

2987 dbconn = self.open_server_repository(read_only = False, 2988 no_upload = True, repo = repo) 2989 counter = dbconn.getNewNegativeSpmUid() 2990 dbconn.setSpmUid(idpackage, counter) 2991 dbconn.setInjected(idpackage)

2992

2993 - def initialize_server_database(self, empty = True, repo = None, 2994 warnings = True):

2995 2996 if repo == None: 2997 repo = self.default_repository 2998 2999 self.close_server_databases() 3000 revisions_match = {} 3001 treeupdates_actions = [] 3002 injected_packages = set() 3003 idpackages = set() 3004 idpackages_added = set() 3005 3006 mytxt = red("%s ...") % (_("Initializing Entropy database"),) 3007 self.updateProgress( 3008 mytxt, importance = 1, 3009 type = "info", header = darkgreen(" * "), 3010 back = True 3011 ) 3012 3013 if os.path.isfile(self.get_local_database_file(repo)): 3014 3015 dbconn = self.open_server_repository(read_only = True, 3016 no_upload = True, repo = repo, warnings = warnings) 3017 3018 try: 3019 dbconn.validateDatabase() 3020 idpackages = dbconn.listAllIdpackages() 3021 except SystemDatabaseError: 3022 pass 3023 3024 try: 3025 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 3026 except dbconn.dbapi2.Error: 3027 pass 3028 3029 # save list of injected packages 3030 try: 3031 injected_packages = dbconn.listAllInjectedPackages( 3032 just_files = True) 3033 injected_packages = set([os.path.basename(x) for x \ 3034 in injected_packages]) 3035 except dbconn.dbapi2.Error: 3036 pass 3037 3038 for idpackage in idpackages: 3039 url = dbconn.retrieveDownloadURL(idpackage) 3040 package = os.path.basename(url) 3041 branch = dbconn.retrieveBranch(idpackage) 3042 revision = dbconn.retrieveRevision(idpackage) 3043 revisions_match[package] = (branch, revision,) 3044 3045 self.close_server_database(dbconn) 3046 3047 mytxt = "%s: %s: %s" % ( 3048 bold(_("WARNING")), 3049 red(_("database already exists")), 3050 self.get_local_database_file(repo), 3051 ) 3052 self.updateProgress( 3053 mytxt, 3054 importance = 1, 3055 type = "warning", 3056 header = darkred(" !!! ") 3057 ) 3058 3059 rc_question = self.askQuestion(_("Do you want to continue ?")) 3060 if rc_question == "No": 3061 return 3062 try: 3063 os.remove(self.get_local_database_file(repo)) 3064 except OSError: 3065 pass 3066 3067 3068 # initialize 3069 dbconn = self.open_server_repository(read_only = False, 3070 no_upload = True, repo = repo, is_new = True) 3071 dbconn.initializeDatabase() 3072 3073 if not empty: 3074 3075 revisions_file = "/entropy-revisions-dump.txt" 3076 # dump revisions - as a backup 3077 if revisions_match: 3078 self.updateProgress( 3079 "%s: %s" % ( 3080 red(_("Dumping current revisions to file")), 3081 darkgreen(revisions_file), 3082 ), 3083 importance = 1, 3084 type = "info", 3085 header = darkgreen(" * ") 3086 ) 3087 f_rev = open(revisions_file,"w") 3088 f_rev.write(str(revisions_match)) 3089 f_rev.flush() 3090 f_rev.close() 3091 3092 # dump treeupdates - as a backup 3093 treeupdates_file = "/entropy-treeupdates-dump.txt" 3094 if treeupdates_actions: 3095 self.updateProgress( 3096 "%s: %s" % ( 3097 # do not translate treeupdates 3098 red(_("Dumping current 'treeupdates' actions to file")), 3099 bold(treeupdates_file), 3100 ), 3101 importance = 1, 3102 type = "info", 3103 header = darkgreen(" * ") 3104 ) 3105 f_tree = open(treeupdates_file,"w") 3106 f_tree.write(str(treeupdates_actions)) 3107 f_tree.flush() 3108 f_tree.close() 3109 3110 rc_question = self.askQuestion( 3111 _("Would you like to sync packages first (important!) ?")) 3112 if rc_question == "Yes": 3113 self.MirrorsService.sync_packages(repo = repo) 3114 3115 # fill tree updates actions 3116 if treeupdates_actions: 3117 dbconn.bumpTreeUpdatesActions(treeupdates_actions) 3118 3119 # now fill the database 3120 pkg_branch_dir = os.path.join( 3121 self.get_local_packages_directory(repo), 3122 self.SystemSettings['repositories']['branch']) 3123 pkglist = os.listdir(pkg_branch_dir) 3124 # filter .md5 and .expired packages 3125 pkg_ext_len = len(etpConst['packagesext']) 3126 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \ 3127 etpConst['packagesext']) and not \ 3128 os.path.isfile(os.path.join(pkg_branch_dir, 3129 x + etpConst['packagesexpirationfileext']))] 3130 3131 if pkglist: 3132 self.updateProgress( 3133 "%s '%s' %s %s" % ( 3134 red(_("Reinitializing Entropy database for branch")), 3135 bold(self.SystemSettings['repositories']['branch']), 3136 red(_("using Packages in the repository")), 3137 red("..."), 3138 ), 3139 importance = 1, 3140 type = "info", 3141 header = darkgreen(" * ") 3142 ) 3143 3144 counter = 0 3145 maxcount = len(pkglist) 3146 branch = self.SystemSettings['repositories']['branch'] 3147 for pkg in pkglist: 3148 counter += 1 3149 3150 self.updateProgress( 3151 "[repo:%s|%s] %s: %s" % ( 3152 darkgreen(repo), 3153 brown(branch), 3154 blue(_("analyzing")), 3155 bold(pkg), 3156 ), 3157 importance = 1, 3158 type = "info", 3159 header = " ", 3160 back = True, 3161 count = (counter, maxcount,) 3162 ) 3163 3164 doinject = False 3165 if pkg in injected_packages: 3166 doinject = True 3167 3168 pkg_path = os.path.join(self.get_local_packages_directory(repo), 3169 branch, pkg) 3170 mydata = self.Spm().extract_package_metadata(pkg_path) 3171 mydata['injected'] = doinject 3172 3173 # get previous revision 3174 revision_avail = revisions_match.get(pkg) 3175 add_revision = 0 3176 if (revision_avail != None): 3177 if branch == revision_avail[0]: 3178 add_revision = revision_avail[1] 3179 3180 idpackage, revision, mydata_upd = dbconn.addPackage(mydata, 3181 revision = add_revision) 3182 idpackages_added.add(idpackage) 3183 3184 self.updateProgress( 3185 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % ( 3186 repo, 3187 brown(branch), 3188 darkgreen(str(counter)), 3189 blue(str(maxcount)), 3190 red(_("added package")), 3191 darkgreen(pkg), 3192 red(_("revision")), 3193 brown(str(revision)), 3194 ), 3195 importance = 1, 3196 type = "info", 3197 header = " ", 3198 back = True 3199 ) 3200 3201 self.depends_table_initialize(repo) 3202 self.library_paths_table_initialize(repo) 3203 3204 my_qa = self.QA() 3205 3206 if idpackages_added: 3207 dbconn = self.open_server_repository(read_only = False, 3208 no_upload = True, repo = repo) 3209 my_qa.test_missing_dependencies( 3210 idpackages_added, dbconn, ask = True, 3211 repo = repo, self_check = True, 3212 black_list = \ 3213 self.get_missing_dependencies_blacklist(repo = repo), 3214 black_list_adder = \ 3215 self.add_missing_dependencies_blacklist_items 3216 ) 3217 3218 dbconn.commitChanges() 3219 self.close_server_databases() 3220 3221 return 0

3222

3223 - def match_packages(self, packages, repo = None):

3224 3225 dbconn = self.open_server_repository(read_only = True, 3226 no_upload = True, repo = repo) 3227 if ("world" in packages) or not packages: 3228 return dbconn.listAllIdpackages(), True 3229 else: 3230 idpackages = set() 3231 for package in packages: 3232 matches = dbconn.atomMatch(package, multiMatch = True) 3233 if matches[1] == 0: 3234 idpackages |= matches[0] 3235 else: 3236 mytxt = "%s: %s: %s" % ( 3237 red(_("Attention")), 3238 blue(_("cannot match")), 3239 bold(package), 3240 ) 3241 self.updateProgress( 3242 mytxt, 3243 importance = 1, 3244 type = "warning", 3245 header = darkred(" !!! ") 3246 ) 3247 return idpackages, False

3248

3249 - def get_remote_package_checksum(self, repo, filename, branch):

3250 3251 import urllib2 3252 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3253 if not srv_set['repositories'][repo].has_key('handler'): 3254 return None 3255 url = srv_set['repositories'][repo]['handler'] 3256 3257 # does the package has "#" (== tag) ? hackish thing that works 3258 filename = filename.replace("#","%23") 3259 # "+" 3260 filename = filename.replace("+","%2b") 3261 request = os.path.join(url, etpConst['handlers']['md5sum']) 3262 request += filename+"&branch="+branch 3263 3264 proxy_settings = self.SystemSettings['system']['proxy'] 3265 try: 3266 mydict = {} 3267 if proxy_settings['ftp']: 3268 mydict['ftp'] = proxy_settings['ftp'] 3269 if proxy_settings['http']: 3270 mydict['http'] = proxy_settings['http'] 3271 if mydict: 3272 mydict['username'] = proxy_settings['username'] 3273 mydict['password'] = proxy_settings['password'] 3274 self.entropyTools.add_proxy_opener(urllib2, mydict) 3275 else: 3276 # unset 3277 urllib2._opener = None 3278 item = urllib2.urlopen(request) 3279 result = item.readline().strip() 3280 item.close() 3281 del item 3282 return result 3283 except (urllib2.URLError, urllib2.HTTPError,): # no HTTP support? 3284 return None

3285

3286 - def verify_remote_packages(self, packages, ask = True, repo = None):

3287 3288 if repo == None: 3289 repo = self.default_repository 3290 3291 self.updateProgress( 3292 "[%s] %s:" % ( 3293 red("remote"), 3294 blue(_("Integrity verification of the selected packages")), 3295 ), 3296 importance = 1, 3297 type = "info", 3298 header = blue(" @@ ") 3299 ) 3300 3301 idpackages, world = self.match_packages(packages) 3302 dbconn = self.open_server_repository(read_only = True, no_upload = True, 3303 repo = repo) 3304 branch = self.SystemSettings['repositories']['branch'] 3305 3306 if world: 3307 self.updateProgress( 3308 blue( 3309 _("All the packages in repository will be checked.")), 3310 importance = 1, 3311 type = "info", 3312 header = " " 3313 ) 3314 else: 3315 mytxt = red("%s:") % ( 3316 _("This is the list of the packages that would be checked"),) 3317 self.updateProgress( 3318 mytxt, 3319 importance = 1, 3320 type = "info", 3321 header = " " 3322 ) 3323 for idpackage in idpackages: 3324 pkgatom = dbconn.retrieveAtom(idpackage) 3325 down_url = dbconn.retrieveDownloadURL(idpackage) 3326 pkgfile = os.path.basename(down_url) 3327 self.updateProgress( 3328 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)), 3329 importance = 1, 3330 type = "info", 3331 header = darkgreen(" - ") 3332 ) 3333 3334 if ask: 3335 rc_question = self.askQuestion( 3336 _("Would you like to continue ?")) 3337 if rc_question == "No": 3338 return set(), set(), {} 3339 3340 match = set() 3341 not_match = set() 3342 broken_packages = {} 3343 3344 for uri in self.get_remote_mirrors(repo): 3345 3346 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3347 self.updateProgress( 3348 "[repo:%s] %s: %s" % ( 3349 darkgreen(repo), 3350 blue(_("Working on mirror")), 3351 brown(crippled_uri), 3352 ), 3353 importance = 1, 3354 type = "info", 3355 header = red(" @@ ") 3356 ) 3357 3358 3359 totalcounter = len(idpackages) 3360 currentcounter = 0 3361 for idpackage in idpackages: 3362 3363 currentcounter += 1 3364 pkgfile = dbconn.retrieveDownloadURL(idpackage) 3365 orig_branch = self.get_branch_from_download_relative_uri( 3366 pkgfile) 3367 3368 self.updateProgress( 3369 "[%s] %s: %s" % ( 3370 brown(crippled_uri), 3371 blue(_("checking hash")), 3372 darkgreen(pkgfile), 3373 ), 3374 importance = 1, 3375 type = "info", 3376 header = blue(" @@ "), 3377 back = True, 3378 count = (currentcounter, totalcounter,) 3379 ) 3380 3381 checksum_ok = False 3382 ck_remote = self.get_remote_package_checksum(repo, 3383 os.path.basename(pkgfile), orig_branch) 3384 if ck_remote == None: 3385 self.updateProgress( 3386 "[%s] %s: %s %s" % ( 3387 brown(crippled_uri), 3388 blue(_("digest verification of")), 3389 bold(pkgfile), 3390 blue(_("not supported")), 3391 ), 3392 importance = 1, 3393 type = "info", 3394 header = blue(" @@ "), 3395 count = (currentcounter, totalcounter,) 3396 ) 3397 elif len(ck_remote) == 32: 3398 pkghash = dbconn.retrieveDigest(idpackage) 3399 if ck_remote == pkghash: 3400 checksum_ok = True 3401 else: 3402 self.updateProgress( 3403 "[%s] %s: %s %s" % ( 3404 brown(crippled_uri), 3405 blue(_("digest verification of")), 3406 bold(pkgfile), 3407 blue(_("failed for unknown reasons")), 3408 ), 3409 importance = 1, 3410 type = "info", 3411 header = blue(" @@ "), 3412 count = (currentcounter, totalcounter,) 3413 ) 3414 3415 if checksum_ok: 3416 match.add(idpackage) 3417 else: 3418 not_match.add(idpackage) 3419 self.updateProgress( 3420 "[%s] %s: %s %s" % ( 3421 brown(crippled_uri), 3422 blue(_("package")), 3423 bold(pkgfile), 3424 red(_("NOT healthy")), 3425 ), 3426 importance = 1, 3427 type = "warning", 3428 header = darkred(" !!! "), 3429 count = (currentcounter, totalcounter,) 3430 ) 3431 if not broken_packages.has_key(crippled_uri): 3432 broken_packages[crippled_uri] = [] 3433 broken_packages[crippled_uri].append(pkgfile) 3434 3435 if broken_packages: 3436 mytxt = blue("%s:") % ( 3437 _("This is the list of broken packages"),) 3438 self.updateProgress( 3439 mytxt, 3440 importance = 1, 3441 type = "info", 3442 header = red(" * ") 3443 ) 3444 for mirror in broken_packages.keys(): 3445 mytxt = "%s: %s" % ( 3446 brown(_("Mirror")), 3447 bold(mirror), 3448 ) 3449 self.updateProgress( 3450 mytxt, 3451 importance = 1, 3452 type = "info", 3453 header = red(" <> ") 3454 ) 3455 for broken_package in broken_packages[mirror]: 3456 self.updateProgress( 3457 blue(broken_package), 3458 importance = 1, 3459 type = "info", 3460 header = red(" - ") 3461 ) 3462 3463 self.updateProgress( 3464 "%s:" % ( 3465 blue(_("Statistics")), 3466 ), 3467 importance = 1, 3468 type = "info", 3469 header = red(" @@ ") 3470 ) 3471 self.updateProgress( 3472 "[%s] %s:\t%s" % ( 3473 red(crippled_uri), 3474 brown(_("Number of checked packages")), 3475 brown(str(len(match) + len(not_match))), 3476 ), 3477 importance = 1, 3478 type = "info", 3479 header = brown(" # ") 3480 ) 3481 self.updateProgress( 3482 "[%s] %s:\t%s" % ( 3483 red(crippled_uri), 3484 darkgreen(_("Number of healthy packages")), 3485 darkgreen(str(len(match))), 3486 ), 3487 importance = 1, 3488 type = "info", 3489 header = brown(" # ") 3490 ) 3491 self.updateProgress( 3492 "[%s] %s:\t%s" % ( 3493 red(crippled_uri), 3494 darkred(_("Number of broken packages")), 3495 darkred(str(len(not_match))), 3496 ), 3497 importance = 1, 3498 type = "info", 3499 header = brown(" # ") 3500 ) 3501 3502 return match, not_match, broken_packages

3503 3504

3505 - def verify_local_packages(self, packages, ask = True, repo = None):

3506 3507 if repo == None: 3508 repo = self.default_repository 3509 3510 self.updateProgress( 3511 "[%s] %s:" % ( 3512 red(_("local")), 3513 blue(_("Integrity verification of the selected packages")), 3514 ), 3515 importance = 1, 3516 type = "info", 3517 header = darkgreen(" * ") 3518 ) 3519 3520 idpackages, world = self.match_packages(packages) 3521 dbconn = self.open_server_repository(read_only = True, 3522 no_upload = True, repo = repo) 3523 3524 if world: 3525 self.updateProgress( 3526 blue(_("All the packages in repository will be checked.")), 3527 importance = 1, 3528 type = "info", 3529 header = " " 3530 ) 3531 3532 to_download = set() 3533 available = set() 3534 for idpackage in idpackages: 3535 3536 pkgatom = dbconn.retrieveAtom(idpackage) 3537 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3538 pkg_rel_path = '/'.join(pkg_path.split("/")[2:]) 3539 3540 bindir_path = os.path.join(self.get_local_packages_directory(repo), 3541 pkg_rel_path) 3542 uploaddir_path = os.path.join(self.get_local_upload_directory(repo), 3543 pkg_rel_path) 3544 3545 if os.path.isfile(bindir_path): 3546 if not world: 3547 self.updateProgress( 3548 "[%s] %s :: %s" % ( 3549 darkgreen(_("available")), 3550 blue(pkgatom), 3551 darkgreen(pkg_rel_path), 3552 ), 3553 importance = 0, 3554 type = "info", 3555 header = darkgreen(" # ") 3556 ) 3557 available.add(idpackage) 3558 elif os.path.isfile(uploaddir_path): 3559 if not world: 3560 self.updateProgress( 3561 "[%s] %s :: %s" % ( 3562 darkred(_("upload/ignored")), 3563 blue(pkgatom), 3564 darkgreen(pkg_rel_path), 3565 ), 3566 importance = 0, 3567 type = "info", 3568 header = darkgreen(" # ") 3569 ) 3570 else: 3571 self.updateProgress( 3572 "[%s] %s :: %s" % ( 3573 brown(_("download")), 3574 blue(pkgatom), 3575 darkgreen(pkg_rel_path), 3576 ), 3577 importance = 0, 3578 type = "info", 3579 header = darkgreen(" # ") 3580 ) 3581 to_download.add((idpackage, pkg_path,)) 3582 3583 if ask: 3584 rc_question = self.askQuestion(_("Would you like to continue ?")) 3585 if rc_question == "No": 3586 return set(), set(), set(), set() 3587 3588 3589 fine = set() 3590 failed = set() 3591 downloaded_fine = set() 3592 downloaded_errors = set() 3593 3594 if to_download: 3595 3596 not_downloaded = set() 3597 mytxt = blue("%s ...") % (_("Starting to download missing files"),) 3598 self.updateProgress( 3599 mytxt, 3600 importance = 1, 3601 type = "info", 3602 header = " " 3603 ) 3604 for uri in self.get_remote_mirrors(repo): 3605 3606 if not_downloaded: 3607 mytxt = blue("%s ...") % ( 3608 _("Searching missing/broken files on another mirror"),) 3609 self.updateProgress( 3610 mytxt, 3611 importance = 1, 3612 type = "info", 3613 header = " " 3614 ) 3615 to_download = not_downloaded.copy() 3616 not_downloaded = set() 3617 3618 for idpackage, pkg_path in to_download: 3619 rc_down = self.MirrorsService.download_package(uri, 3620 pkg_path, repo = repo) 3621 if rc_down: 3622 downloaded_fine.add(idpackage) 3623 available.add(idpackage) 3624 else: 3625 not_downloaded.add(pkg_path) 3626 3627 if not not_downloaded: 3628 self.updateProgress( 3629 red(_("Binary packages downloaded successfully.")), 3630 importance = 1, 3631 type = "info", 3632 header = " " 3633 ) 3634 break 3635 3636 if not_downloaded: 3637 mytxt = blue("%s:") % ( 3638 _("These are the packages that cannot be found online"),) 3639 self.updateProgress( 3640 mytxt, 3641 importance = 1, 3642 type = "info", 3643 header = " " 3644 ) 3645 for pkg_path in not_downloaded: 3646 downloaded_errors.add(pkg_path) 3647 self.updateProgress( 3648 brown(pkg_path), 3649 importance = 1, 3650 type = "warning", 3651 header = red(" * ") 3652 ) 3653 downloaded_errors |= not_downloaded 3654 mytxt = "%s." % (_("They won't be checked"),) 3655 self.updateProgress( 3656 mytxt, 3657 importance = 1, 3658 type = "warning", 3659 header = " " 3660 ) 3661 3662 my_qa = self.QA() 3663 3664 totalcounter = str(len(available)) 3665 currentcounter = 0 3666 for idpackage in available: 3667 currentcounter += 1 3668 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3669 orig_branch = self.get_branch_from_download_relative_uri(pkg_path) 3670 pkgfile = os.path.basename(pkg_path) 3671 3672 self.updateProgress( 3673 "[branch:%s] %s %s" % ( 3674 brown(orig_branch), 3675 blue(_("checking status of")), 3676 darkgreen(pkgfile), 3677 ), 3678 importance = 1, 3679 type = "info", 3680 header = " ", 3681 back = True, 3682 count = (currentcounter, totalcounter,) 3683 ) 3684 3685 storedmd5 = dbconn.retrieveDigest(idpackage) 3686 pkgpath = os.path.join(self.get_local_packages_directory(repo), 3687 orig_branch, pkgfile) 3688 result = self.entropyTools.compare_md5(pkgpath, storedmd5) 3689 qa_fine = my_qa.entropy_package_checks(pkgpath) 3690 if result and qa_fine: 3691 fine.add(idpackage) 3692 else: 3693 failed.add(idpackage) 3694 self.updateProgress( 3695 "[branch:%s] %s %s %s: %s" % ( 3696 brown(orig_branch), 3697 blue(_("package")), 3698 darkgreen(pkg_path), 3699 blue(_("is corrupted, stored checksum")), 3700 brown(storedmd5), 3701 ), 3702 importance = 1, 3703 type = "info", 3704 header = " ", 3705 count = (currentcounter, totalcounter,) 3706 ) 3707 3708 if failed: 3709 mytxt = blue("%s:") % (_("This is the list of broken packages"),) 3710 self.updateProgress( 3711 mytxt, 3712 importance = 1, 3713 type = "warning", 3714 header = darkred(" # ") 3715 ) 3716 for idpackage in failed: 3717 atom = dbconn.retrieveAtom(idpackage) 3718 down_p = dbconn.retrieveDownloadURL(idpackage) 3719 self.updateProgress( 3720 blue("[atom:%s] %s" % (atom, down_p,)), 3721 importance = 0, 3722 type = "warning", 3723 header = brown(" # ") 3724 ) 3725 3726 # print stats 3727 self.updateProgress( 3728 red("Statistics:"), 3729 importance = 1, 3730 type = "info", 3731 header = blue(" * ") 3732 ) 3733 self.updateProgress( 3734 brown("%s => %s" % ( 3735 len(fine) + len(failed), 3736 _("checked packages"), 3737 ) 3738 ), 3739 importance = 0, 3740 type = "info", 3741 header = brown(" # ") 3742 ) 3743 self.updateProgress( 3744 darkgreen("%s => %s" % ( 3745 len(fine), 3746 _("healthy packages"), 3747 ) 3748 ), 3749 importance = 0, 3750 type = "info", 3751 header = brown(" # ") 3752 ) 3753 self.updateProgress( 3754 darkred("%s => %s" % ( 3755 len(failed), 3756 _("broken packages"), 3757 ) 3758 ), 3759 importance = 0, 3760 type = "info", 3761 header = brown(" # ") 3762 ) 3763 self.updateProgress( 3764 blue("%s => %s" % ( 3765 len(downloaded_fine), 3766 _("downloaded packages"), 3767 ) 3768 ), 3769 importance = 0, 3770 type = "info", 3771 header = brown(" # ") 3772 ) 3773 self.updateProgress( 3774 bold("%s => %s" % ( 3775 len(downloaded_errors), 3776 _("failed downloads"), 3777 ) 3778 ), 3779 importance = 0, 3780 type = "info", 3781 header = brown(" # ") 3782 ) 3783 3784 self.close_server_database(dbconn) 3785 return fine, failed, downloaded_fine, downloaded_errors

3786 3787

3788 - def switch_packages_branch(self, from_branch, to_branch, repo = None):

3789 3790 if repo == None: 3791 repo = self.default_repository 3792 3793 if to_branch != self.SystemSettings['repositories']['branch']: 3794 mytxt = "%s: %s %s" % ( 3795 blue(_("Please setup your branch to")), 3796 bold(to_branch), 3797 blue(_("and retry")), 3798 ) 3799 self.updateProgress( 3800 mytxt, 3801 importance = 1, 3802 type = "error", 3803 header = darkred(" !! ") 3804 ) 3805 return None 3806 3807 mytxt = red("%s ...") % (_("Copying database (if not exists)"),) 3808 self.updateProgress( 3809 mytxt, 3810 importance = 1, 3811 type = "info", 3812 header = darkgreen(" @@ ") 3813 ) 3814 branch_dbdir = self.get_local_database_dir(repo) 3815 old_branch_dbdir = self.get_local_database_dir(repo, from_branch) 3816 3817 # close all our databases 3818 self.close_server_databases() 3819 3820 # if database file did not exist got created as an empty file 3821 # we can just rm -rf it 3822 branch_dbfile = self.get_local_database_file(repo) 3823 if os.path.isfile(branch_dbfile): 3824 if self.entropyTools.get_file_size(branch_dbfile) == 0: 3825 shutil.rmtree(branch_dbdir, True) 3826 3827 if os.path.isdir(branch_dbdir): 3828 3829 while 1: 3830 rnd_num = self.entropyTools.get_random_number() 3831 backup_dbdir = branch_dbdir + str(rnd_num) 3832 if not os.path.isdir(backup_dbdir): 3833 break 3834 os.rename(branch_dbdir, backup_dbdir) 3835 3836 if os.path.isdir(old_branch_dbdir): 3837 shutil.copytree(old_branch_dbdir, branch_dbdir) 3838 3839 mytxt = red("%s ...") % (_("Switching packages"),) 3840 self.updateProgress( 3841 mytxt, 3842 importance = 1, 3843 type = "info", 3844 header = darkgreen(" @@ ") 3845 ) 3846 3847 dbconn = self.open_server_repository(read_only = False, 3848 no_upload = True, repo = repo, lock_remote = False) 3849 try: 3850 dbconn.validateDatabase() 3851 except SystemDatabaseError: 3852 self.handle_uninitialized_repository(repo) 3853 dbconn = self.open_server_repository(read_only = False, 3854 no_upload = True, repo = repo, lock_remote = False) 3855 3856 idpackages = dbconn.listAllIdpackages() 3857 already_switched = set() 3858 not_found = set() 3859 switched = set() 3860 ignored = set() 3861 no_checksum = set() 3862 3863 maxcount = len(idpackages) 3864 count = 0 3865 for idpackage in idpackages: 3866 count += 1 3867 3868 cur_branch = dbconn.retrieveBranch(idpackage) 3869 atom = dbconn.retrieveAtom(idpackage) 3870 if cur_branch == to_branch: 3871 already_switched.add(idpackage) 3872 self.updateProgress( 3873 red("%s %s, %s %s" % ( 3874 _("Ignoring"), 3875 bold(atom), 3876 _("already in branch"), 3877 cur_branch, 3878 ) 3879 ), 3880 importance = 0, 3881 type = "info", 3882 header = darkgreen(" @@ "), 3883 count = (count, maxcount,) 3884 ) 3885 ignored.add(idpackage) 3886 continue 3887 3888 self.updateProgress( 3889 "[%s=>%s] %s" % ( 3890 brown(cur_branch), 3891 bold(to_branch), 3892 darkgreen(atom), 3893 ), 3894 importance = 0, 3895 type = "info", 3896 header = darkgreen(" @@ "), 3897 back = True, 3898 count = (count, maxcount,) 3899 ) 3900 dbconn.switchBranch(idpackage, to_branch) 3901 dbconn.commitChanges() 3902 switched.add(idpackage) 3903 3904 dbconn.commitChanges() 3905 3906 # now migrate counters 3907 dbconn.moveSpmUidsToBranch(to_branch, from_branch = from_branch) 3908 3909 self.close_server_database(dbconn) 3910 mytxt = blue("%s.") % (_("migration loop completed"),) 3911 self.updateProgress( 3912 "[%s=>%s] %s" % ( 3913 brown(from_branch), 3914 bold(to_branch), 3915 mytxt, 3916 ), 3917 importance = 1, 3918 type = "info", 3919 header = darkgreen(" * ") 3920 ) 3921 3922 return switched, already_switched, ignored, not_found, no_checksum

3923

3924 - def get_entropy_sets(self, repo = None, branch = None):

3925 3926 if branch == None: 3927 branch = self.SystemSettings['repositories']['branch'] 3928 if repo == None: 3929 repo = self.default_repository 3930 3931 sets_dir = self.get_local_database_sets_dir(repo, branch) 3932 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 3933 return {} 3934 3935 mydata = {} 3936 items = os.listdir(sets_dir) 3937 for item in items: 3938 3939 try: 3940 item_clean = str(item) 3941 except (UnicodeEncodeError, UnicodeDecodeError,): 3942 continue 3943 item_path = os.path.join(sets_dir, item) 3944 if not (os.path.isfile(item_path) and \ 3945 os.access(item_path, os.R_OK)): 3946 continue 3947 item_elements = self.entropyTools.extract_packages_from_set_file( 3948 item_path) 3949 if item_elements: 3950 mydata[item_clean] = item_elements.copy() 3951 3952 return mydata

3953

3954 - def get_configured_package_sets(self, repo = None, branch = None, 3955 validate = True):

3956 3957 if branch == None: 3958 branch = self.SystemSettings['repositories']['branch'] 3959 if repo == None: 3960 repo = self.default_repository 3961 3962 # portage sets 3963 sets_data = self.Spm().get_package_sets(False) 3964 sets_data.update(self.get_entropy_sets(repo, branch)) 3965 3966 if validate: 3967 invalid_sets = set() 3968 # validate 3969 for setname in sets_data: 3970 good = True 3971 for atom in sets_data[setname]: 3972 dbconn = self.open_server_repository(just_reading = True, 3973 repo = repo) 3974 match = dbconn.atomMatch(atom) 3975 if match[0] == -1: 3976 good = False 3977 break 3978 if not good: 3979 invalid_sets.add(setname) 3980 for invalid_set in invalid_sets: 3981 del sets_data[invalid_set] 3982 3983 return sets_data

3984

3985 - def update_database_package_sets(self, repo = None, dbconn = None):

3986 3987 if repo == None: 3988 repo = self.default_repository 3989 package_sets = self.get_configured_package_sets(repo) 3990 if dbconn == None: 3991 dbconn = self.open_server_repository( 3992 read_only = False, no_upload = True, repo = repo, 3993 do_treeupdates = False) 3994 dbconn.clearPackageSets() 3995 if package_sets: 3996 dbconn.insertPackageSets(package_sets) 3997 dbconn.commitChanges()

3998

Source Code for Module entropy.server.interfaces.main