entropy.server.interfaces.main

1 # -*- coding: utf-8 -*- 2 ''' 3 # DESCRIPTION: 4 # Entropy Object Oriented Interface 5 6 Copyright (C) 2007-2009 Fabio Erculiani 7 8 This program is free software; you can redistribute it and/or modify 9 it under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 2 of the License, or 11 (at your option) any later version. 12 13 This program is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 GNU General Public License for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with this program; if not, write to the Free Software 20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 ''' 22 23 from __future__ import with_statement 24 import os 25 import shutil 26 from entropy.core import Singleton 27 from entropy.exceptions import OnlineMirrorError, PermissionDenied, \ 28 SystemDatabaseError 29 from entropy.const import etpConst, etpSys, const_setup_perms, \ 30 const_create_working_dirs, const_extract_srv_repo_params, etpUi 31 from entropy.output import TextInterface, purple, red, darkgreen, \ 32 bold, brown, blue, darkred 33 from entropy.server.interfaces.mirrors import Server as MirrorsServer 34 from entropy.i18n import _ 35 from entropy.core import SystemSettings, SystemSettingsPlugin 36 from entropy.transceivers import FtpInterface 37 from entropy.db import EntropyRepository 38

39 -class ServerSystemSettingsPlugin(SystemSettingsPlugin):

40

41 - def server_parser(self, sys_set):

42 43 """ 44 Parses Entropy server system configuration file. 45 46 @return dict data 47 """ 48 49 data = { 50 'repositories': etpConst['server_repositories'].copy(), 51 'default_repository_id': etpConst['officialserverrepositoryid'], 52 'packages_expiration_days': etpConst['packagesexpirationdays'], 53 'database_file_format': etpConst['etpdatabasefileformat'], 54 'disabled_eapis': set(), 55 'exp_based_scope': etpConst['expiration_based_scope'], 56 'rss': { 57 'enabled': etpConst['rss-feed'], 58 'name': etpConst['rss-name'], 59 'base_url': etpConst['rss-base-url'], 60 'website_url': etpConst['rss-website-url'], 61 'editor': etpConst['rss-managing-editor'], 62 'max_entries': etpConst['rss-max-entries'], 63 'light_max_entries': etpConst['rss-light-max-entries'], 64 }, 65 } 66 67 fake_instance = self._helper.fake_default_repo 68 69 if not os.access(etpConst['serverconf'], os.R_OK): 70 return data 71 72 with open(etpConst['serverconf'],"r") as server_f: 73 serverconf = [x.strip() for x in server_f.readlines() if x.strip()] 74 75 for line in serverconf: 76 77 split_line = line.split("|") 78 split_line_len = len(split_line) 79 80 if (line.find("officialserverrepositoryid|") != -1) and \ 81 (not line.startswith("#")) and (split_line_len == 2): 82 83 if not fake_instance: 84 data['default_repository_id'] = split_line[1].strip() 85 86 elif line.startswith("expiration-days|") and (split_line_len == 2): 87 88 mydays = split_line[1].strip() 89 try: 90 mydays = int(mydays) 91 data['packages_expiration_days'] = mydays 92 except ValueError: 93 continue 94 95 elif line.startswith("expiration-based-scope|") and \ 96 (split_line_len == 2): 97 98 exp_opt = split_line[1].strip().lower() 99 if exp_opt in ("enable", "enabled", "true", "1", "yes"): 100 data['exp_based_scope'] = True 101 else: 102 data['exp_based_scope'] = False 103 104 elif line.startswith("disabled-eapis|") and (split_line_len == 2): 105 106 mydis = split_line[1].strip().split(",") 107 try: 108 mydis = [int(x) for x in mydis] 109 mydis = set([x for x in mydis if x in (1, 2, 3,)]) 110 except ValueError: 111 continue 112 if (len(mydis) < 3) and mydis: 113 data['disabled_eapis'] = mydis 114 115 116 elif line.startswith("repository|") and (split_line_len in [5, 6]) \ 117 and (not fake_instance): 118 119 repoid, repodata = const_extract_srv_repo_params(line, 120 product = sys_set['repositories']['product']) 121 if repoid in data['repositories']: 122 # just update mirrors 123 data['repositories'][repoid]['mirrors'].extend( 124 repodata['mirrors']) 125 else: 126 data['repositories'][repoid] = repodata.copy() 127 128 elif line.startswith("database-format|") and (split_line_len == 2): 129 130 fmt = split_line[1] 131 if fmt in etpConst['etpdatabasesupportedcformats']: 132 data['database_file_format'] = fmt 133 134 elif line.startswith("rss-feed|") and (split_line_len == 2): 135 136 feed = split_line[1] 137 if feed in ("enable", "enabled", "true", "1"): 138 data['rss']['enabled'] = True 139 elif feed in ("disable", "disabled", "false", "0", "no",): 140 data['rss']['enabled'] = False 141 142 elif line.startswith("rss-name|") and (split_line_len == 2): 143 144 feedname = line.split("rss-name|")[1].strip() 145 data['rss']['name'] = feedname 146 147 elif line.startswith("rss-base-url|") and (split_line_len == 2): 148 149 data['rss']['base_url'] = line.split("rss-base-url|")[1].strip() 150 if not data['rss']['base_url'][-1] == "/": 151 data['rss']['base_url'] += "/" 152 153 elif line.startswith("rss-website-url|") and (split_line_len == 2): 154 155 data['rss']['website_url'] = split_line[1].strip() 156 157 elif line.startswith("managing-editor|") and (split_line_len == 2): 158 159 data['rss']['editor'] = split_line[1].strip() 160 161 elif line.startswith("max-rss-entries|") and (split_line_len == 2): 162 163 try: 164 entries = int(split_line[1].strip()) 165 data['rss']['max_entries'] = entries 166 except (ValueError, IndexError,): 167 continue 168 169 elif line.startswith("max-rss-light-entries|") and \ 170 (split_line_len == 2): 171 172 try: 173 entries = int(split_line[1].strip()) 174 data['rss']['light_max_entries'] = entries 175 except (ValueError, IndexError,): 176 continue 177 178 # add system database if community repository mode is enabled 179 if self._helper.community_repo: 180 data['repositories'][etpConst['clientserverrepoid']] = {} 181 mydata = {} 182 mydata['description'] = "Community Repositories System Database" 183 mydata['mirrors'] = [] 184 mydata['community'] = False 185 data['repositories'][etpConst['clientserverrepoid']].update(mydata) 186 187 # expand paths 188 for repoid in data['repositories']: 189 data['repositories'][repoid]['packages_dir'] = \ 190 os.path.join( etpConst['entropyworkdir'], 191 "server", 192 repoid, 193 "packages", 194 etpSys['arch'] 195 ) 196 data['repositories'][repoid]['store_dir'] = \ 197 os.path.join( etpConst['entropyworkdir'], 198 "server", 199 repoid, 200 "store", 201 etpSys['arch'] 202 ) 203 data['repositories'][repoid]['upload_dir'] = \ 204 os.path.join( etpConst['entropyworkdir'], 205 "server", 206 repoid, 207 "upload", 208 etpSys['arch'] 209 ) 210 data['repositories'][repoid]['database_dir'] = \ 211 os.path.join( etpConst['entropyworkdir'], 212 "server", 213 repoid, 214 "database", 215 etpSys['arch'] 216 ) 217 data['repositories'][repoid]['packages_relative_path'] = \ 218 os.path.join( sys_set['repositories']['product'], 219 repoid, 220 "packages", 221 etpSys['arch'] 222 )+"/" 223 data['repositories'][repoid]['database_relative_path'] = \ 224 os.path.join( sys_set['repositories']['product'], 225 repoid, 226 "database", 227 etpSys['arch'] 228 )+"/" 229 230 # Support for shell variables 231 shell_repoid = os.getenv('ETP_REPO') 232 if shell_repoid: 233 data['default_repository_id'] = shell_repoid 234 235 expiration_days = os.getenv('ETP_EXPIRATION_DAYS') 236 if expiration_days: 237 try: 238 expiration_days = int(expiration_days) 239 data['packages_expiration_days'] = expiration_days 240 except ValueError: 241 pass 242 243 return data

244

245 -class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

246 247 import entropy.tools as entropyTools 248

249 - def repos_parser(self, sys_set):

250 251 data = {} 252 srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin'] 253 # if support is not enabled, don't waste time scanning files 254 srv_parser_data = sys_set[srv_plug_id]['server'] 255 if not srv_parser_data['exp_based_scope']: 256 return data 257 258 # get expiration-based packages removal data from config files 259 for repoid in srv_parser_data['repositories']: 260 261 # filter out system repository if community repository 262 # mode is enabled 263 if repoid == etpConst['clientserverrepoid']: 264 continue 265 266 idpackages = set() 267 exp_fp = self._helper.get_local_exp_based_pkgs_rm_whitelist_file( 268 repo = repoid) 269 dbconn = self._helper.open_server_repository( 270 just_reading = True, repo = repoid) 271 272 if os.access(exp_fp, os.R_OK | os.F_OK): 273 pkgs = self.entropyTools.generic_file_content_parser(exp_fp) 274 if '*' in pkgs: # wildcard support 275 idpackages.add(-1) 276 else: 277 for pkg in pkgs: 278 idpackage, rc_match = dbconn.atomMatch(pkg) 279 if rc_match: 280 continue 281 idpackages.add(idpackage) 282 283 data[repoid] = idpackages 284 285 return data

286 287

288 -class Server(Singleton, TextInterface):

289

290 - def init_singleton(self, default_repository = None, save_repository = False, 291 community_repo = False, fake_default_repo = False, 292 fake_default_repo_id = '::fake::', 293 fake_default_repo_desc = 'this is a fake repository'):

294 295 self.__instance_destroyed = False 296 if etpConst['uid'] != 0: 297 mytxt = _("Entropy Server interface must be run as root") 298 raise PermissionDenied("PermissionDenied: %s" % (mytxt,)) 299 300 # settings 301 from entropy.misc import LogFile 302 self.SystemSettings = SystemSettings() 303 self.community_repo = community_repo 304 from entropy.db import dbapi2 305 self.dbapi2 = dbapi2 # export for third parties 306 etpSys['serverside'] = True 307 self._memory_db_instances = {} 308 self.fake_default_repo = fake_default_repo 309 self.indexing = False 310 self.xcache = False 311 self.MirrorsService = None 312 self.__server_dbcache = {} 313 self.repository_treeupdate_digests = {} 314 self.__settings_to_backup = [] 315 self.__do_save_repository = save_repository 316 self.__sync_lock_cache = set() 317 self.rssMessages = { 318 'added': {}, 319 'removed': {}, 320 'commitmessage': "", 321 'light': {}, 322 } 323 324 if fake_default_repo: 325 default_repository = fake_default_repo_id 326 etpConst['officialserverrepositoryid'] = fake_default_repo_id 327 self.init_generic_memory_server_repository(fake_default_repo_id, 328 fake_default_repo_desc) 329 330 # create our SystemSettings plugin 331 self.sys_settings_plugin_id = \ 332 etpConst['system_settings_plugins_ids']['server_plugin'] 333 self.sys_settings_plugin = ServerSystemSettingsPlugin( 334 self.sys_settings_plugin_id, self) 335 self.SystemSettings.add_plugin(self.sys_settings_plugin) 336 337 # Fatscope support SystemSettings plugin 338 self.sys_settings_fatscope_plugin_id = \ 339 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 340 self.sys_settings_fatscope_plugin = ServerFatscopeSystemSettingsPlugin( 341 self.sys_settings_fatscope_plugin_id, self) 342 self.SystemSettings.add_plugin(self.sys_settings_fatscope_plugin) 343 344 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 345 self.default_repository = default_repository 346 if self.default_repository == None: 347 self.default_repository = srv_set['default_repository_id'] 348 349 if self.default_repository in srv_set['repositories']: 350 self.ensure_paths(self.default_repository) 351 352 # NOW DISABLED, deprecated! 353 # self.migrate_repository_databases_to_new_branched_path() 354 355 if self.default_repository not in srv_set['repositories']: 356 raise PermissionDenied("PermissionDenied: %s %s" % ( 357 self.default_repository, 358 _("repository not configured"), 359 ) 360 ) 361 if etpConst['clientserverrepoid'] == self.default_repository: 362 raise PermissionDenied("PermissionDenied: %s %s" % ( 363 etpConst['clientserverrepoid'], 364 _("protected repository id, can't use this, sorry dude..."), 365 ) 366 ) 367 368 self.switch_default_repository(self.default_repository)

369

370 - def destroy(self):

371 self.__instance_destroyed = True 372 if hasattr(self,'ClientService'): 373 self.ClientService.destroy() 374 if hasattr(self,'sys_settings_server_plugin'): 375 try: 376 self.SystemSettings.remove_plugin(self.sys_settings_plugin_id) 377 except KeyError: 378 pass 379 if hasattr(self,'sys_settings_fatscope_plugin'): 380 try: 381 self.SystemSettings.remove_plugin( 382 self.sys_settings_fatscope_plugin) 383 except KeyError: 384 pass 385 self.close_server_databases()

386

387 - def is_destroyed(self):

388 return self.__instance_destroyed

389

390 - def __del__(self):

391 self.destroy()

392

393 - def ensure_paths(self, repo):

394 upload_dir = os.path.join(self.get_local_upload_directory(repo), 395 self.SystemSettings['repositories']['branch']) 396 db_dir = self.get_local_database_dir(repo) 397 for mydir in [upload_dir, db_dir]: 398 if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)): 399 os.makedirs(mydir) 400 const_setup_perms(mydir, etpConst['entropygid'])

401 402

403 - def migrate_repository_databases_to_new_branched_path(self):

404 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 405 migrated_filename = '.branch_migrated' 406 for repoid in srv_set['repositories'].keys(): 407 408 if repoid == etpConst['clientserverrepoid']: continue 409 mydir = srv_set['repositories'][repoid]['database_dir'] 410 if not os.path.isdir(mydir): # empty ? 411 continue 412 413 migrated_filepath = os.path.join(mydir, migrated_filename) 414 if os.path.isfile(migrated_filepath): 415 continue 416 417 my_branched_dir = self.get_local_database_dir(repoid) 418 if os.path.isdir(my_branched_dir): # wtf? do not touch 419 continue 420 421 self.updateProgress( 422 "[%s:%s] %s: %s, %s: %s" % ( 423 brown("repo"), 424 purple(repoid), 425 _("migrating database path from"), 426 brown(mydir), 427 _('to'), 428 brown(my_branched_dir), 429 ), 430 importance = 1, 431 type = "info", 432 header = bold(" @@ ") 433 ) 434 435 repo_files = [os.path.join(mydir, x) for x in os.listdir(mydir) if \ 436 (os.path.isfile(os.path.join(mydir, x)) and \ 437 os.access(os.path.join(mydir, x), os.W_OK)) 438 ] 439 os.makedirs(my_branched_dir) 440 const_setup_perms(my_branched_dir, etpConst['entropygid']) 441 442 for repo_file in repo_files: 443 repo_filename = os.path.basename(repo_file) 444 shutil.move(repo_file, 445 os.path.join(my_branched_dir, repo_filename)) 446 447 f_migrated = open(migrated_filepath,"w") 448 f_migrated.write("done\n") 449 f_migrated.flush() 450 f_migrated.close()

451

452 - def setup_services(self):

453 self.setup_entropy_settings() 454 cs_name = 'ClientService' 455 if hasattr(self, cs_name): 456 obj = getattr(self, cs_name) 457 obj.destroy() 458 from entropy.client.interfaces import Client 459 self.ClientService = Client( 460 indexing = self.indexing, 461 xcache = self.xcache, 462 repo_validation = False, 463 noclientdb = 1 464 ) 465 from entropy.cache import EntropyCacher 466 self.Cacher = EntropyCacher() 467 self.ClientService.updateProgress = self.updateProgress 468 self.validRepositories = self.ClientService.validRepositories 469 self.entropyTools = self.ClientService.entropyTools 470 self.dumpTools = self.ClientService.dumpTools 471 self.QA = self.ClientService.QA 472 self.backup_entropy_settings() 473 self.SpmService = self.ClientService.Spm() 474 self.MirrorsService = MirrorsServer(self)

475

476 - def setup_entropy_settings(self, repo = None):

477 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 478 curr_repoid = srv_set['default_repository_id'] 479 backup_list = [ 480 'etpdatabaseclientfilepath', 481 'clientdbid', 482 {'server': srv_set.copy()}, 483 ] 484 for setting in backup_list: 485 if setting not in self.__settings_to_backup: 486 self.__settings_to_backup.append(setting) 487 # setup client database 488 if not self.community_repo: 489 etpConst['etpdatabaseclientfilepath'] = \ 490 self.get_local_database_file(repo) 491 etpConst['clientdbid'] = etpConst['serverdbid'] 492 const_create_working_dirs()

493

494 - def close_server_databases(self):

495 if hasattr(self,'serverDbCache'): 496 for item in self.__server_dbcache: 497 try: 498 self.__server_dbcache[item].closeDB() 499 except self.dbapi2.ProgrammingError: # already closed? 500 pass 501 self.__server_dbcache.clear()

502

503 - def close_server_database(self, dbinstance):

504 found = None 505 for item in self.__server_dbcache: 506 if dbinstance == self.__server_dbcache[item]: 507 found = item 508 break 509 if found: 510 instance = self.__server_dbcache.pop(found) 511 instance.closeDB()

512

513 - def get_available_repositories(self):

514 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 515 return srv_set['repositories'].copy()

516

517 - def switch_default_repository(self, repoid, save = None, 518 handle_uninitialized = True):

519 520 # avoid setting __default__ as default server repo 521 if repoid == etpConst['clientserverrepoid']: 522 return 523 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 524 525 if save == None: 526 save = self.__do_save_repository 527 if repoid not in srv_set['repositories']: 528 raise PermissionDenied("PermissionDenied: %s %s" % ( 529 repoid, 530 _("repository not configured"), 531 ) 532 ) 533 self.close_server_databases() 534 srv_set['default_repository_id'] = repoid 535 self.default_repository = repoid 536 self.setup_services() 537 if save: 538 self.save_default_repository(repoid) 539 540 self.setup_community_repositories_settings() 541 self.show_interface_status() 542 if handle_uninitialized and etpUi['warn']: 543 self.handle_uninitialized_repository(repoid)

544

545 - def setup_community_repositories_settings(self):

546 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 547 if self.community_repo: 548 for repoid in srv_set['repositories']: 549 srv_set['repositories'][repoid]['community'] = True

550 551

552 - def handle_uninitialized_repository(self, repoid):

553 554 if not self.is_repository_initialized(repoid): 555 mytxt = blue("%s.") % ( 556 _("Your default repository is not initialized"),) 557 self.updateProgress( 558 "[%s:%s] %s" % ( 559 brown("repo"), 560 purple(repoid), 561 mytxt, 562 ), 563 importance = 1, 564 type = "warning", 565 header = darkred(" !!! ") 566 ) 567 answer = self.askQuestion( 568 _("Do you want to initialize your default repository ?")) 569 if answer == "No": 570 mytxt = red("%s.") % ( 571 _("Continuing with an uninitialized repository"),) 572 self.updateProgress( 573 "[%s:%s] %s" % ( 574 brown("repo"), 575 purple(repoid), 576 mytxt, 577 ), 578 importance = 1, 579 type = "warning", 580 header = darkred(" !!! ") 581 ) 582 else: 583 # move empty database for security sake 584 dbfile = self.get_local_database_file(repoid) 585 if os.path.isfile(dbfile): 586 shutil.move(dbfile, dbfile+".backup") 587 self.initialize_server_database(empty = True, 588 repo = repoid, warnings = False)

589 590

591 - def show_interface_status(self):

592 type_txt = _("server-side repository") 593 if self.community_repo: 594 type_txt = _("community repository") 595 # ..on repository: <repository_name> 596 mytxt = _("Entropy Server Interface Instance on repository") 597 self.updateProgress( 598 blue("%s: %s, %s: %s (%s: %s)" % ( 599 mytxt, 600 red(self.default_repository), 601 _("current branch"), 602 darkgreen(self.SystemSettings['repositories']['branch']), 603 purple(_("type")), 604 bold(type_txt), 605 ) 606 ), 607 importance = 2, 608 type = "info", 609 header = red(" @@ ") 610 ) 611 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 612 repos = srv_set['repositories'].keys() 613 mytxt = blue("%s:") % (_("Currently configured repositories"),) 614 self.updateProgress( 615 mytxt, 616 importance = 1, 617 type = "info", 618 header = red(" @@ ") 619 ) 620 for repo in repos: 621 self.updateProgress( 622 darkgreen(repo), 623 importance = 0, 624 type = "info", 625 header = brown(" # ") 626 )

627 628

629 - def save_default_repository(self, repoid):

630 631 # avoid setting __default__ as default server repo 632 if repoid == etpConst['clientserverrepoid']: 633 return 634 635 if os.path.isfile(etpConst['serverconf']): 636 f_srv = open(etpConst['serverconf'],"r") 637 content = f_srv.readlines() 638 f_srv.close() 639 content = [x.strip() for x in content] 640 found = False 641 new_content = [] 642 for line in content: 643 if line.strip().startswith("officialserverrepositoryid|"): 644 line = "officialserverrepositoryid|%s" % (repoid,) 645 found = True 646 new_content.append(line) 647 if not found: 648 new_content.append("officialserverrepositoryid|%s" % (repoid,)) 649 f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp","w") 650 for line in new_content: 651 f_srv_t.write(line+"\n") 652 f_srv_t.flush() 653 f_srv_t.close() 654 os.rename(etpConst['serverconf']+".save_default_repo_tmp", 655 etpConst['serverconf']) 656 else: 657 f_srv = open(etpConst['serverconf'],"w") 658 f_srv.write("officialserverrepositoryid|%s\n" % (repoid,)) 659 f_srv.flush() 660 f_srv.close()

661

662 - def toggle_repository(self, repoid, enable = True):

663 664 # avoid setting __default__ as default server repo 665 if repoid == etpConst['clientserverrepoid']: 666 return False 667 668 if not os.path.isfile(etpConst['serverconf']): 669 return None 670 f_srv = open(etpConst['serverconf']) 671 tmpfile = etpConst['serverconf']+".switch" 672 mycontent = [x.strip() for x in f_srv.readlines()] 673 f_srv.close() 674 f_tmp = open(tmpfile,"w") 675 st = "repository|%s" % (repoid,) 676 status = False 677 for line in mycontent: 678 if enable: 679 if (line.find(st) != -1) and line.startswith("#") and \ 680 (len(line.split("|")) == 5): 681 line = line[1:] 682 status = True 683 else: 684 if (line.find(st) != -1) and not line.startswith("#") and \ 685 (len(line.split("|")) == 5): 686 line = "#"+line 687 status = True 688 f_tmp.write(line+"\n") 689 f_tmp.flush() 690 f_tmp.close() 691 shutil.move(tmpfile, etpConst['serverconf']) 692 if status: 693 self.close_server_databases() 694 self.SystemSettings.clear() 695 self.setup_services() 696 self.show_interface_status() 697 return status

698

699 - def backup_entropy_settings(self):

700 for setting in self.__settings_to_backup: 701 if isinstance(setting, basestring): 702 self.ClientService.backup_constant(setting) 703 elif isinstance(setting, dict): 704 self.SystemSettings.set_persistent_setting(setting)

705

706 - def is_repository_initialized(self, repo):

707 708 def do_validate(dbc): 709 try: 710 dbc.validateDatabase() 711 return True 712 except SystemDatabaseError: 713 return False

714 715 dbc = self.open_server_repository(just_reading = True, repo = repo) 716 valid = do_validate(dbc) 717 self.close_server_database(dbc) 718 if not valid: # check online? 719 dbc = self.open_server_repository(read_only = False, 720 no_upload = True, repo = repo, is_new = True) 721 valid = do_validate(dbc) 722 self.close_server_database(dbc) 723 724 return valid

725

726 - def do_server_repository_sync_lock(self, repo, no_upload):

727 728 if repo == None: 729 repo = self.default_repository 730 731 # check if the database is locked locally 732 lock_file = self.MirrorsService.get_database_lockfile(repo) 733 if os.path.isfile(lock_file): 734 self.updateProgress( 735 red(_("Entropy database is already locked by you :-)")), 736 importance = 1, 737 type = "info", 738 header = red(" * ") 739 ) 740 else: 741 # check if the database is locked REMOTELY 742 mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),) 743 self.updateProgress( 744 red(mytxt), 745 importance = 1, 746 type = "info", 747 header = red(" * "), 748 back = True 749 ) 750 for uri in self.get_remote_mirrors(repo): 751 given_up = self.MirrorsService.mirror_lock_check(uri, 752 repo = repo) 753 if given_up: 754 crippled_uri = \ 755 self.entropyTools.extract_ftp_host_from_uri(uri) 756 mytxt = "%s:" % (_("Mirrors status table"),) 757 self.updateProgress( 758 darkgreen(mytxt), 759 importance = 1, 760 type = "info", 761 header = brown(" * ") 762 ) 763 dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo) 764 for db_uri, db_st1, db_st2 in dbstatus: 765 db_st1_info = darkgreen(_("Unlocked")) 766 if db_st1: 767 db_st1_info = red(_("Locked")) 768 db_st2_info = darkgreen(_("Unlocked")) 769 if db_st2: 770 db_st2_info = red(_("Locked")) 771 772 crippled_uri = \ 773 self.entropyTools.extract_ftp_host_from_uri(db_uri) 774 self.updateProgress( 775 "%s: [%s: %s] [%s: %s]" % ( 776 bold(crippled_uri), 777 brown(_("database")), 778 db_st1_info, 779 brown(_("download")), 780 db_st2_info, 781 ), 782 importance = 1, 783 type = "info", 784 header = "\t" 785 ) 786 787 raise OnlineMirrorError("OnlineMirrorError: %s %s" % ( 788 _("cannot lock mirror"), 789 crippled_uri, 790 ) 791 ) 792 793 # if we arrive here, it is because all the mirrors are unlocked 794 self.MirrorsService.lock_mirrors(True, repo = repo) 795 self.MirrorsService.sync_databases(no_upload, repo = repo)

796 797

798 - def init_generic_memory_server_repository(self, repoid, description, 799 mirrors = None, community_repo = False, service_url = None):

800 801 if mirrors is None: 802 mirrors = [] 803 product = self.SystemSettings['repositories']['product'] 804 dbc = self.open_memory_database(dbname = etpConst['serverdbid']+repoid) 805 self._memory_db_instances[repoid] = dbc 806 807 eapi3_port = int(etpConst['socket_service']['port']) 808 eapi3_ssl_port = int(etpConst['socket_service']['ssl_port']) 809 # add to settings 810 repodata = { 811 'repoid': repoid, 812 'description': description, 813 'mirrors': mirrors, 814 'community': community_repo, 815 'service_port': eapi3_port, 816 'ssl_service_port': eapi3_ssl_port, 817 'service_url': service_url, 818 'handler': '', # not supported 819 'in_memory': True, 820 } 821 822 etpConst['server_repositories'][repoid] = repodata 823 self.SystemSettings.clear() 824 825 return dbc

826

827 - def open_memory_database(self, dbname = None):

828 if dbname == None: 829 dbname = etpConst['genericdbid'] 830 dbc = EntropyRepository( 831 readOnly = False, 832 dbFile = ':memory:', 833 clientDatabase = True, 834 dbname = dbname, 835 xcache = False, 836 indexing = False, 837 OutputInterface = self, 838 skipChecks = True 839 ) 840 dbc.initializeDatabase() 841 return dbc

842

843 - def open_server_repository( 844 self, 845 read_only = True, 846 no_upload = True, 847 just_reading = False, 848 repo = None, 849 indexing = True, 850 warnings = True, 851 do_cache = True, 852 use_branch = None, 853 lock_remote = True, 854 is_new = False, 855 do_treeupdates = True 856 ):

857 858 if repo == None: 859 repo = self.default_repository 860 861 if repo == etpConst['clientserverrepoid'] and self.community_repo: 862 return self.ClientService.clientDbconn 863 864 # in-memory server repos 865 if repo in self._memory_db_instances: 866 return self._memory_db_instances.get(repo) 867 868 if just_reading: 869 read_only = True 870 no_upload = True 871 872 local_dbfile = self.get_local_database_file(repo, use_branch) 873 if do_cache: 874 cached = self.__server_dbcache.get( 875 (repo, etpConst['systemroot'], local_dbfile, read_only, 876 no_upload, just_reading, use_branch, lock_remote,) 877 ) 878 if cached != None: 879 return cached 880 881 local_dbfile_dir = os.path.dirname(local_dbfile) 882 if not os.path.isdir(local_dbfile_dir): 883 os.makedirs(local_dbfile_dir) 884 885 if (not read_only) and (lock_remote) and \ 886 (repo not in self.__sync_lock_cache): 887 self.do_server_repository_sync_lock(repo, no_upload) 888 self.__sync_lock_cache.add(repo) 889 890 local_dbfile_exists = os.path.lexists(local_dbfile) 891 conn = EntropyRepository( 892 readOnly = read_only, 893 dbFile = local_dbfile, 894 noUpload = no_upload, 895 OutputInterface = self, 896 dbname = etpConst['serverdbid']+repo, 897 useBranch = use_branch, 898 lockRemote = lock_remote 899 ) 900 if not local_dbfile_exists: 901 # better than having a completely broken db 902 conn.readOnly = False 903 conn.initializeDatabase() 904 conn.commitChanges() 905 906 valid = True 907 try: 908 conn.validateDatabase() 909 except SystemDatabaseError: 910 valid = False 911 912 # verify if we need to update the database to sync 913 # with portage updates, we just ignore being readonly in the case 914 if (repo not in etpConst['server_treeupdatescalled']) and \ 915 (not just_reading): 916 # sometimes, when filling a new server db 917 # we need to avoid tree updates 918 if valid: 919 if do_treeupdates: 920 self.repository_packages_spm_sync(conn, 921 branch = use_branch, repo = repo) 922 elif warnings and not is_new: 923 mytxt = _("Entropy database is corrupted!") 924 self.updateProgress( 925 darkred(mytxt), 926 importance = 1, 927 type = "warning", 928 header = bold(" !!! ") 929 ) 930 931 if not read_only and valid and indexing: 932 933 self.updateProgress( 934 "[repo:%s|%s] %s" % ( 935 blue(repo), 936 red(_("database")), 937 blue(_("indexing database")), 938 ), 939 importance = 1, 940 type = "info", 941 header = brown(" @@ "), 942 back = True 943 ) 944 conn.createAllIndexes() 945 946 if do_cache: 947 # !!! also cache just_reading otherwise there will be 948 # real issues if the connection is opened several times 949 self.__server_dbcache[ 950 (repo, etpConst['systemroot'], local_dbfile, read_only, 951 no_upload, just_reading, use_branch, lock_remote,)] = conn 952 953 # auto-update package sets 954 if (not read_only) and (not is_new): 955 cur_sets = conn.retrievePackageSets() 956 sys_sets = self.get_configured_package_sets(repo) 957 if cur_sets != sys_sets: 958 self.update_database_package_sets(repo, dbconn = conn) 959 conn.commitChanges() 960 961 return conn

962

963 - def repository_packages_spm_sync(self, repo_db, branch = None, repo = None):

964 """ 965 Service method used to sync package names with Source Package Manager. 966 Source Package Manager can change package names, categories or slot 967 and Entropy repositories must be kept in sync. 968 969 In other words, it checks for /usr/portage/profiles/updates changes. 970 """ 971 972 if branch == None: 973 branch = self.SystemSettings['repositories']['branch'] 974 if repo == None: 975 repo = self.default_repository 976 977 etpConst['server_treeupdatescalled'].add(repo) 978 979 repo_updates_file = self.get_local_database_treeupdates_file(repo) 980 doRescan = False 981 982 stored_digest = repo_db.retrieveRepositoryUpdatesDigest(repo) 983 if stored_digest == -1: 984 doRescan = True 985 986 # check portage files for changes if doRescan is still false 987 portage_dirs_digest = "0" 988 if not doRescan: 989 990 if self.repository_treeupdate_digests.has_key(repo): 991 portage_dirs_digest = self.repository_treeupdate_digests.get( 992 repo) 993 else: 994 995 spm = self.SpmService 996 # grab portdir 997 updates_dir = etpConst['systemroot'] + \ 998 spm.get_spm_setting("PORTDIR") + "/profiles/updates" 999 if os.path.isdir(updates_dir): 1000 # get checksum 1001 mdigest = self.entropyTools.md5obj_directory(updates_dir) 1002 # also checksum etpConst['etpdatabaseupdatefile'] 1003 if os.path.isfile(repo_updates_file): 1004 f = open(repo_updates_file) 1005 block = f.read(1024) 1006 while block: 1007 mdigest.update(block) 1008 block = f.read(1024) 1009 f.close() 1010 portage_dirs_digest = mdigest.hexdigest() 1011 self.repository_treeupdate_digests[repo] = \ 1012 portage_dirs_digest 1013 1014 if doRescan or (str(stored_digest) != str(portage_dirs_digest)): 1015 1016 # force parameters 1017 repo_db.readOnly = False 1018 repo_db.noUpload = True 1019 1020 # reset database tables 1021 repo_db.clearTreeupdatesEntries(repo) 1022 1023 updates_dir = etpConst['systemroot'] + \ 1024 self.SpmService.get_spm_setting("PORTDIR") + "/profiles/updates" 1025 update_files = self.entropyTools.sort_update_files( 1026 os.listdir(updates_dir)) 1027 update_files = [os.path.join(updates_dir, x) for x in update_files] 1028 # now load actions from files 1029 update_actions = [] 1030 for update_file in update_files: 1031 f = open(update_file, "r") 1032 mycontent = f.readlines() 1033 f.close() 1034 lines = [x.strip() for x in mycontent if x.strip()] 1035 update_actions.extend(lines) 1036 1037 # add entropy packages.db.repo_updates content 1038 if os.path.isfile(repo_updates_file): 1039 f = open(repo_updates_file, "r") 1040 mycontent = f.readlines() 1041 f.close() 1042 lines = [x.strip() for x in mycontent if x.strip() and \ 1043 not x.strip().startswith("#")] 1044 update_actions.extend(lines) 1045 # now filter the required actions 1046 update_actions = repo_db.filterTreeUpdatesActions(update_actions) 1047 if update_actions: 1048 1049 mytxt = "%s: %s. %s %s" % ( 1050 bold(_("ATTENTION")), 1051 red(_("forcing package updates")), 1052 red(_("Syncing with")), 1053 blue(updates_dir), 1054 ) 1055 self.updateProgress( 1056 mytxt, 1057 importance = 1, 1058 type = "info", 1059 header = brown(" * ") 1060 ) 1061 # lock database 1062 if repo_db.lockRemote: 1063 self.do_server_repository_sync_lock( 1064 repo, repo_db.noUpload) 1065 # now run queue 1066 try: 1067 quickpkg_list = repo_db.runTreeUpdatesActions( 1068 update_actions) 1069 except: 1070 # destroy digest 1071 repo_db.setRepositoryUpdatesDigest(repo, "-1") 1072 raise 1073 1074 if quickpkg_list: 1075 # quickpkg package and packages owning it as a dependency 1076 try: 1077 self._run_packages_spm_sync_quickpkg( 1078 quickpkg_list, repo_db, repo) 1079 except: 1080 self.entropyTools.print_traceback() 1081 mytxt = "%s: %s: %s, %s." % ( 1082 bold(_("WARNING")), 1083 red(_("Cannot complete quickpkg for atoms")), 1084 blue(str(sorted(quickpkg_list))), 1085 _("do it manually"), 1086 ) 1087 self.updateProgress( 1088 mytxt, 1089 importance = 1, 1090 type = "warning", 1091 header = darkred(" * ") 1092 ) 1093 repo_db.commitChanges() 1094 1095 # store new actions 1096 repo_db.addRepositoryUpdatesActions( 1097 repo, update_actions, branch) 1098 1099 # store new digest into database 1100 repo_db.setRepositoryUpdatesDigest( 1101 repo, portage_dirs_digest) 1102 repo_db.commitChanges()

1103

1104 - def _run_packages_spm_sync_quickpkg(self, atoms, repo_db, repo):

1105 """ 1106 Executes packages regeneration for given atoms. 1107 """ 1108 package_paths = set() 1109 runatoms = set() 1110 for myatom in atoms: 1111 mymatch = repo_db.atomMatch(myatom) 1112 if mymatch[0] == -1: 1113 continue 1114 myatom = repo_db.retrieveAtom(mymatch[0]) 1115 runatoms.add(myatom) 1116 1117 for myatom in runatoms: 1118 self.updateProgress( 1119 red("%s: " % (_("repackaging"),) )+blue(myatom), 1120 importance = 1, 1121 type = "warning", 1122 header = blue(" # ") 1123 ) 1124 mydest = self.get_local_store_directory(repo = repo) 1125 try: 1126 mypath = self.quickpkg(myatom, mydest) 1127 except: 1128 # remove broken bin before raising 1129 mypath = os.path.join(mydest, 1130 os.path.basename(myatom) + etpConst['packagesext']) 1131 if os.path.isfile(mypath): 1132 os.remove(mypath) 1133 self.entropyTools.print_traceback() 1134 mytxt = "%s: %s: %s, %s." % ( 1135 bold(_("WARNING")), 1136 red(_("Cannot complete quickpkg for atom")), 1137 blue(myatom), 1138 _("do it manually"), 1139 ) 1140 self.updateProgress( 1141 mytxt, 1142 importance = 1, 1143 type = "warning", 1144 header = darkred(" * ") 1145 ) 1146 continue 1147 package_paths.add(mypath) 1148 packages_data = [(x, False,) for x in package_paths] 1149 idpackages = self.add_packages_to_repository( 1150 packages_data, repo = repo) 1151 1152 if not idpackages: 1153 1154 mytxt = "%s: %s. %s." % ( 1155 bold(_("ATTENTION")), 1156 red(_("package files rebuild did not run properly")), 1157 red(_("Please update packages manually")), 1158 ) 1159 self.updateProgress( 1160 mytxt, 1161 importance = 1, 1162 type = "warning", 1163 header = darkred(" * ") 1164 )

1165

1166 - def deps_tester(self, default_repo = None):

1167 1168 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1169 server_repos = sys_set['repositories'].keys() 1170 installed_packages = set() 1171 # if a default repository is passed, we will just test against it 1172 if default_repo: 1173 server_repos = [default_repo] 1174 1175 for repo in server_repos: 1176 dbconn = self.open_server_repository(read_only = True, 1177 no_upload = True, repo = repo, do_treeupdates = False) 1178 installed_packages |= set([(x, repo) for x in \ 1179 dbconn.listAllIdpackages()]) 1180 1181 1182 deps_not_satisfied = set() 1183 length = str((len(installed_packages))) 1184 count = 0 1185 mytxt = _("Checking") 1186 1187 for idpackage, repo in installed_packages: 1188 count += 1 1189 dbconn = self.open_server_repository(read_only = True, 1190 no_upload = True, repo = repo, do_treeupdates = False) 1191 1192 if (count%150 == 0) or (count == length) or (count == 1): 1193 atom = dbconn.retrieveAtom(idpackage) 1194 self.updateProgress( 1195 darkgreen(mytxt)+" "+bold(atom), 1196 importance = 0, 1197 type = "info", 1198 back = True, 1199 count = (count, length), 1200 header = darkred(" @@ ") 1201 ) 1202 1203 xdeps = dbconn.retrieveDependencies(idpackage) 1204 for xdep in xdeps: 1205 xid, xuseless = self.atom_match(xdep) 1206 if xid == -1: 1207 deps_not_satisfied.add(xdep) 1208 1209 return deps_not_satisfied

1210

1211 - def dependencies_test(self, repo = None):

1212 1213 mytxt = "%s %s" % (blue(_("Running dependencies test")), red("...")) 1214 self.updateProgress( 1215 mytxt, 1216 importance = 2, 1217 type = "info", 1218 header = red(" @@ ") 1219 ) 1220 1221 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1222 server_repos = srv_set['repositories'].keys() 1223 deps_not_matched = self.deps_tester(repo) 1224 1225 if deps_not_matched: 1226 1227 crying_atoms = {} 1228 for atom in deps_not_matched: 1229 for repo in server_repos: 1230 dbconn = self.open_server_repository(just_reading = True, 1231 repo = repo, do_treeupdates = False) 1232 riddep = dbconn.searchDependency(atom) 1233 if riddep == -1: 1234 continue 1235 ridpackages = dbconn.searchIdpackageFromIddependency(riddep) 1236 for i in ridpackages: 1237 iatom = dbconn.retrieveAtom(i) 1238 if not crying_atoms.has_key(atom): 1239 crying_atoms[atom] = set() 1240 crying_atoms[atom].add((iatom, repo)) 1241 1242 mytxt = blue("%s:") % (_("These are the dependencies not found"),) 1243 self.updateProgress( 1244 mytxt, 1245 importance = 1, 1246 type = "info", 1247 header = red(" @@ ") 1248 ) 1249 mytxt = "%s:" % (_("Needed by"),) 1250 for atom in deps_not_matched: 1251 self.updateProgress( 1252 red(atom), 1253 importance = 1, 1254 type = "info", 1255 header = blue(" # ") 1256 ) 1257 if crying_atoms.has_key(atom): 1258 self.updateProgress( 1259 red(mytxt), 1260 importance = 0, 1261 type = "info", 1262 header = blue(" # ") 1263 ) 1264 for my_dep, myrepo in crying_atoms[atom]: 1265 self.updateProgress( 1266 "[%s:%s] %s" % ( 1267 blue(_("by repo")), 1268 darkred(myrepo), 1269 darkgreen(my_dep), 1270 ), 1271 importance = 0, 1272 type = "info", 1273 header = blue(" # ") 1274 ) 1275 else: 1276 1277 mytxt = blue(_("Every dependency is satisfied. It's all fine.")) 1278 self.updateProgress( 1279 mytxt, 1280 importance = 2, 1281 type = "info", 1282 header = red(" @@ ") 1283 ) 1284 1285 return deps_not_matched

1286

1287 - def test_shared_objects(self, get_files = False, repo = None):

1288 1289 # load db 1290 dbconn = self.open_server_repository(read_only = True, 1291 no_upload = True, repo = repo) 1292 QA = self.QA() 1293 packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn, 1294 broken_symbols = True) 1295 if status != 0: 1296 return 1, None 1297 1298 if get_files: 1299 return 0, brokenexecs 1300 1301 if (not brokenexecs) and (not packages_matched): 1302 mytxt = "%s." % (_("System is healthy"),) 1303 self.updateProgress( 1304 blue(mytxt), 1305 importance = 2, 1306 type = "info", 1307 header = red(" @@ ") 1308 ) 1309 return 0, None 1310 1311 mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),) 1312 self.updateProgress( 1313 blue(mytxt), 1314 importance = 1, 1315 type = "info", 1316 header = red(" @@ ") 1317 ) 1318 1319 packages = self.SpmService.query_belongs_multiple(brokenexecs) 1320 1321 if packages: 1322 mytxt = "%s:" % (_("These are the matched packages"),) 1323 self.updateProgress( 1324 red(mytxt), 1325 importance = 1, 1326 type = "info", 1327 header = red(" @@ ") 1328 ) 1329 for my_atom, my_elf_id in packages: 1330 package_slot = my_atom, my_elf_id 1331 self.updateProgress( 1332 "%s [elf:%s]" % ( 1333 purple(my_atom), 1334 darkgreen(str(my_elf_id)), 1335 ), 1336 importance = 0, 1337 type = "info", 1338 header = red(" # ") 1339 ) 1340 for filename in sorted(list(packages[package_slot])): 1341 self.updateProgress( 1342 darkgreen(filename), 1343 importance = 0, 1344 type = "info", 1345 header = brown(" => ") 1346 ) 1347 # print string 1348 pkgstring = ' '.join(["%s:%s" % ( 1349 self.entropyTools.dep_getkey(x[0]), x[1],) for x \ 1350 in sorted(packages)]) 1351 mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,) 1352 self.updateProgress( 1353 mytxt, 1354 importance = 1, 1355 type = "info", 1356 header = red(" @@ ") 1357 ) 1358 else: 1359 self.updateProgress( 1360 red(_("No matched packages")), 1361 importance = 1, 1362 type = "info", 1363 header = red(" @@ ") 1364 ) 1365 1366 return 0, packages

1367

1368 - def orphaned_spm_packages_test(self):

1369 1370 mytxt = "%s %s" % ( 1371 blue(_("Running orphaned SPM packages test")), red("..."),) 1372 self.updateProgress( 1373 mytxt, 1374 importance = 2, 1375 type = "info", 1376 header = red(" @@ ") 1377 ) 1378 installed_packages, length = self.SpmService.get_installed_packages() 1379 not_found = {} 1380 count = 0 1381 for installed_package in installed_packages: 1382 count += 1 1383 self.updateProgress( 1384 "%s: %s" % ( 1385 darkgreen(_("Scanning package")), 1386 brown(installed_package),), 1387 importance = 0, 1388 type = "info", 1389 back = True, 1390 count = (count, length), 1391 header = darkred(" @@ ") 1392 ) 1393 key, slot = (self.entropyTools.dep_getkey(installed_package), 1394 self.SpmService.get_installed_package_slot(installed_package),) 1395 pkg_atom = "%s:%s" % (key, slot,) 1396 tree_atom = self.SpmService.get_best_atom(pkg_atom) 1397 if not tree_atom: 1398 not_found[installed_package] = pkg_atom 1399 self.updateProgress( 1400 "%s: %s" % ( 1401 blue(pkg_atom), 1402 darkred(_("not found anymore")), 1403 ), 1404 importance = 0, 1405 type = "warning", 1406 count = (count, length), 1407 header = darkred(" @@ ") 1408 ) 1409 1410 if not_found: 1411 not_found_list = ' '.join([not_found[x] for x in sorted(not_found)]) 1412 self.updateProgress( 1413 "%s: %s" % ( 1414 blue(_("Packages string")), 1415 not_found_list, 1416 ), 1417 importance = 0, 1418 type = "warning", 1419 count = (count, length), 1420 header = darkred(" @@ ") 1421 ) 1422 1423 return not_found

1424

1425 - def depends_table_initialize(self, repo = None):

1426 dbconn = self.open_server_repository(read_only = False, 1427 no_upload = True, repo = repo) 1428 dbconn.regenerateDependsTable() 1429 dbconn.taintDatabase() 1430 dbconn.commitChanges()

1431

1432 - def library_paths_table_initialize(self, repo = None):

1433 dbconn = self.open_server_repository(read_only = False, 1434 no_upload = True, repo = repo) 1435 dbconn.regenerateLibrarypathsidpackageTable() 1436 dbconn.taintDatabase() 1437 dbconn.commitChanges()

1438

1439 - def create_empty_database(self, dbpath = None, repo = None):

1440 if dbpath == None: 1441 dbpath = self.get_local_database_file(repo) 1442 1443 dbdir = os.path.dirname(dbpath) 1444 if not os.path.isdir(dbdir): 1445 os.makedirs(dbdir) 1446 1447 mytxt = red("%s ...") % (_("Initializing an empty database"),) 1448 self.updateProgress( 1449 mytxt, 1450 importance = 1, 1451 type = "info", 1452 header = darkgreen(" * "), 1453 back = True 1454 ) 1455 dbconn = self.ClientService.open_generic_database(dbpath) 1456 dbconn.initializeDatabase() 1457 dbconn.commitChanges() 1458 dbconn.closeDB() 1459 mytxt = "%s %s %s." % ( 1460 red(_("Entropy database file")), 1461 bold(dbpath), 1462 red(_("successfully initialized")), 1463 ) 1464 self.updateProgress( 1465 mytxt, 1466 importance = 1, 1467 type = "info", 1468 header = darkgreen(" * ") 1469 )

1470

1471 - def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

1472 1473 # check package_tag "no spaces" 1474 1475 try: 1476 package_tag = str(package_tag) 1477 if " " in package_tag: 1478 raise ValueError 1479 except (UnicodeDecodeError, UnicodeEncodeError, ValueError,): 1480 self.updateProgress( 1481 "%s: %s" % ( 1482 blue(_("Invalid tag specified")), 1483 package_tag, 1484 ), 1485 importance = 1, type = "error", header = darkred(" !! ") 1486 ) 1487 return 1, package_tag 1488 1489 if repo == None: 1490 repo = self.default_repository 1491 1492 # sanity check 1493 invalid_atoms = [] 1494 dbconn = self.open_server_repository(read_only = True, 1495 no_upload = True, repo = repo) 1496 for idpackage in idpackages: 1497 ver_tag = dbconn.retrieveVersionTag(idpackage) 1498 if ver_tag: 1499 invalid_atoms.append(dbconn.retrieveAtom(idpackage)) 1500 1501 if invalid_atoms: 1502 self.updateProgress( 1503 "%s: %s" % ( 1504 blue(_("Packages already tagged, action aborted")), 1505 ', '.join([darkred(unicode(x)) for x in invalid_atoms]), 1506 ), 1507 importance = 1, type = "error", header = darkred(" !! ") 1508 ) 1509 return 2, invalid_atoms 1510 1511 matches = [(x, repo) for x in idpackages] 1512 status = 0 1513 data = self.move_packages( 1514 matches, to_repo = repo, from_repo = repo, ask = ask, 1515 do_copy = True, new_tag = package_tag 1516 ) 1517 return status, data

1518

1519 - def flushback_packages(self, from_branches, repo = None, ask = True):

1520 """ 1521 When creating a new branch, for space reasons, packages are not 1522 moved to a new location. This works fine until old branch is removed. 1523 To avoid inconsistences, before deciding to do that, all the packages 1524 in the old branch should be flushed back to the the currently configured 1525 branch. 1526 1527 @param from_branches -- list of branches to move packages from 1528 @type from_branches -- list 1529 @param repo -- repository to work on 1530 @type repo -- str 1531 @param ask -- user interactivity 1532 @type ask -- bool 1533 1534 @return status 1535 """ 1536 1537 status = True 1538 if repo == None: 1539 repo = self.default_repository 1540 branch = self.SystemSettings['repositories']['branch'] 1541 1542 if branch in from_branches: 1543 from_branches = [x for x in from_branches if x != branch] 1544 1545 self.updateProgress( 1546 "[%s=>%s|%s] %s" % ( 1547 darkgreen(', '.join(from_branches)), 1548 darkred(branch), 1549 brown(repo), 1550 blue(_("flushing back selected packages from branches")), 1551 ), 1552 importance = 2, 1553 type = "info", 1554 header = red(" @@ ") 1555 ) 1556 1557 dbconn = self.open_server_repository(read_only = True, 1558 no_upload = True, repo = repo) 1559 1560 idpackage_map = dict(((x, [],) for x in from_branches)) 1561 idpackages = dbconn.listAllIdpackages(order_by = 'atom') 1562 for idpackage in idpackages: 1563 download_url = dbconn.retrieveDownloadURL(idpackage) 1564 url_br = self.ClientService.get_branch_from_download_relative_uri( 1565 download_url) 1566 if url_br in from_branches: 1567 idpackage_map[url_br].append(idpackage) 1568 1569 mapped_branches = [x for x in idpackage_map if idpackage_map[x]] 1570 if not mapped_branches: 1571 self.updateProgress( 1572 "[%s=>%s|%s] %s !" % ( 1573 darkgreen(', '.join(from_branches)), 1574 darkred(branch), 1575 brown(repo), 1576 blue(_("nothing to do")), 1577 ), 1578 importance = 0, 1579 type = "warning", 1580 header = blue(" @@ ") 1581 ) 1582 return status 1583 1584 1585 all_fine = True 1586 tmp_down_dir = self.entropyTools.get_random_temp_file() 1587 os.makedirs(tmp_down_dir) 1588 1589 download_queue = {} 1590 local_up_dir = self.get_local_upload_directory(repo) 1591 local_basedir = os.path.join(local_up_dir, branch) 1592 dbconn = self.open_server_repository(read_only = False, 1593 no_upload = True, repo = repo) 1594 1595 def generate_queue(branch, repo, from_branch, down_q, idpackage_map): 1596 1597 self.updateProgress( 1598 "[%s=>%s|%s] %s" % ( 1599 darkgreen(from_branch), 1600 darkred(branch), 1601 brown(repo), 1602 brown(_("these are the packages that will be flushed")), 1603 ), 1604 importance = 1, 1605 type = "info", 1606 header = brown(" @@ ") 1607 ) 1608 1609 1610 for idpackage in idpackage_map[from_branch]: 1611 atom = dbconn.retrieveAtom(idpackage) 1612 self.updateProgress( 1613 "[%s=>%s|%s] %s" % ( 1614 darkgreen(from_branch), 1615 darkred(branch), 1616 brown(repo), 1617 purple(atom), 1618 ), 1619 importance = 0, 1620 type = "info", 1621 header = blue(" # ") 1622 ) 1623 pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage)) 1624 pkg_fp = os.path.join(tmp_down_dir, pkg_fp) 1625 down_q.append((pkg_fp, idpackage,))

1626 1627 1628 for from_branch in sorted(mapped_branches): 1629 1630 download_queue[from_branch] = [] 1631 all_fine = False 1632 generate_queue(branch, repo, from_branch, 1633 download_queue[from_branch], idpackage_map) 1634 1635 if ask: 1636 rc_question = self.askQuestion( 1637 _("Would you like to continue ?")) 1638 if rc_question == "No": 1639 continue 1640 1641 remote_relative_path = self.get_remote_packages_relative_path(repo) 1642 1643 for uri in self.get_remote_mirrors(repo): 1644 1645 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1646 ftp_basedir = os.path.join(remote_relative_path, from_branch) 1647 1648 downloader_queue = [x[0] for x in download_queue[from_branch]] 1649 downloader = self.MirrorsService.FtpServerHandler( 1650 FtpInterface, 1651 self, 1652 [uri], 1653 downloader_queue, 1654 critical_files = downloader_queue, 1655 use_handlers = True, 1656 ftp_basedir = ftp_basedir, 1657 local_basedir = tmp_down_dir, 1658 download = True, 1659 repo = repo 1660 ) 1661 1662 errors, m_fine_uris, m_broken_uris = downloader.go() 1663 1664 if not errors: 1665 for downloaded_path, idpackage in \ 1666 download_queue[from_branch]: 1667 1668 self.updateProgress( 1669 "[%s=>%s|%s|%s] %s: %s" % ( 1670 darkgreen(from_branch), 1671 darkred(branch), 1672 brown(repo), 1673 dbconn.retrieveAtom(idpackage), 1674 blue(_("checking package hash")), 1675 darkgreen(os.path.basename(downloaded_path)), 1676 ), 1677 importance = 0, 1678 type = "info", 1679 header = brown(" "), 1680 back = True 1681 ) 1682 1683 md5hash = self.entropyTools.md5sum(downloaded_path) 1684 db_md5hash = dbconn.retrieveDigest(idpackage) 1685 if md5hash != db_md5hash: 1686 errors = True 1687 self.updateProgress( 1688 "[%s=>%s|%s|%s] %s: %s" % ( 1689 darkgreen(from_branch), 1690 darkred(branch), 1691 brown(repo), 1692 dbconn.retrieveAtom(idpackage), 1693 blue(_("hash does not match for")), 1694 darkgreen(os.path.basename(downloaded_path)), 1695 ), 1696 importance = 0, 1697 type = "error", 1698 header = brown(" ") 1699 ) 1700 continue 1701 1702 1703 1704 if errors: 1705 reason = _("wrong md5") 1706 if m_broken_uris: 1707 my_broken_uris = [ 1708 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 1709 for x, y in m_broken_uris] 1710 reason = my_broken_uris[0][1] 1711 1712 self.updateProgress( 1713 "[%s=>%s|%s] %s, %s: %s" % ( 1714 darkgreen(from_branch), 1715 darkred(branch), 1716 brown(repo), 1717 blue(_("download errors")), 1718 blue(_("reason")), 1719 reason, 1720 ), 1721 importance = 1, 1722 type = "error", 1723 header = darkred(" !!! ") 1724 ) 1725 # continuing if possible 1726 continue 1727 1728 all_fine = True 1729 1730 self.updateProgress( 1731 "[%s=>%s|%s] %s: %s" % ( 1732 darkgreen(from_branch), 1733 darkred(branch), 1734 brown(repo), 1735 blue(_("download completed successfully")), 1736 darkgreen(crippled_uri), 1737 ), 1738 importance = 1, 1739 type = "info", 1740 header = darkgreen(" * ") 1741 ) 1742 1743 if not all_fine: 1744 self.updateProgress( 1745 "[%s=>%s|%s] %s" % ( 1746 darkgreen(', '.join(from_branches)), 1747 darkred(branch), 1748 brown(repo), 1749 blue(_("error downloading packages from mirrors")), 1750 ), 1751 importance = 2, 1752 type = "error", 1753 header = darkred(" !!! ") 1754 ) 1755 return False 1756 1757 tmp_db_path = self.entropyTools.get_random_temp_file() 1758 for from_branch in sorted(mapped_branches): 1759 1760 self.updateProgress( 1761 "[%s=>%s|%s] %s: %s" % ( 1762 darkgreen(from_branch), 1763 darkred(branch), 1764 brown(repo), 1765 blue(_("working on branch")), 1766 darkgreen(from_branch), 1767 ), 1768 importance = 1, 1769 type = "info", 1770 header = brown(" @@ ") 1771 ) 1772 1773 down_queue = download_queue[from_branch] 1774 for package_path, idpackage in down_queue: 1775 1776 self.updateProgress( 1777 "[%s=>%s|%s] %s: %s" % ( 1778 darkgreen(from_branch), 1779 darkred(branch), 1780 brown(repo), 1781 blue(_("updating package")), 1782 darkgreen(os.path.basename(package_path)), 1783 ), 1784 importance = 1, 1785 type = "info", 1786 header = brown(" "), 1787 back = True 1788 ) 1789 1790 # move files to upload 1791 package_name = os.path.basename(package_path) 1792 new_package_path = os.path.join(local_basedir, package_name) 1793 try: 1794 os.rename(package_path, new_package_path) 1795 except OSError: 1796 shutil.move(package_path, new_package_path) 1797 1798 # create md5 checksum 1799 self.entropyTools.create_md5_file(new_package_path) 1800 1801 # update database 1802 download_url = dbconn.retrieveDownloadURL(idpackage) 1803 download_url = \ 1804 self.ClientService.swap_branch_in_download_relative_uri( 1805 branch, download_url) 1806 dbconn.setDownloadURL(idpackage, download_url) 1807 dbconn.switchBranch(idpackage, branch) 1808 dbconn.commitChanges() 1809 1810 self.updateProgress( 1811 "[%s=>%s|%s] %s: %s" % ( 1812 darkgreen(from_branch), 1813 darkred(branch), 1814 brown(repo), 1815 blue(_("package flushed")), 1816 darkgreen(os.path.basename(package_path)), 1817 ), 1818 importance = 1, 1819 type = "info", 1820 header = brown(" ") 1821 ) 1822 1823 try: 1824 os.rmdir(tmp_down_dir) 1825 except OSError: 1826 pass 1827 1828 return True 1829

1830 - def move_packages(self, matches, to_repo, from_repo = None, ask = True, 1831 do_copy = False, new_tag = None, pull_deps = False):

1832 1833 if from_repo == None: 1834 from_repo = self.default_repository 1835 switched = set() 1836 1837 my_matches = list(matches) 1838 1839 # avoid setting __default__ as default server repo 1840 if etpConst['clientserverrepoid'] in (to_repo, from_repo): 1841 self.updateProgress( 1842 "%s: %s" % ( 1843 blue(_("Cannot touch system database")), 1844 red(etpConst['clientserverrepoid']), 1845 ), 1846 importance = 2, type = "warning", header = darkred(" @@ ") 1847 ) 1848 return switched 1849 1850 if not my_matches and from_repo: 1851 dbconn = self.open_server_repository(read_only = True, 1852 no_upload = True, repo = from_repo) 1853 my_matches = set( \ 1854 [(x, from_repo) for x in \ 1855 dbconn.listAllIdpackages()] 1856 ) 1857 1858 mytxt = _("Preparing to move selected packages to") 1859 if do_copy: 1860 mytxt = _("Preparing to copy selected packages to") 1861 self.updateProgress( 1862 "%s %s:" % ( 1863 blue(mytxt), 1864 red(to_repo), 1865 ), 1866 importance = 2, 1867 type = "info", 1868 header = red(" @@ ") 1869 ) 1870 self.updateProgress( 1871 "%s: %s" % ( 1872 bold(_("Note")), 1873 red(_("all old packages with conflicting scope will be " \ 1874 "removed from destination repo unless injected")), 1875 ), 1876 importance = 1, 1877 type = "info", 1878 header = red(" @@ ") 1879 ) 1880 1881 new_tag_string = '' 1882 if new_tag != None: 1883 new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")), 1884 brown(new_tag),) 1885 1886 my_qa = self.QA() 1887 branch = self.SystemSettings['repositories']['branch'] 1888 pull_deps_matches = [] 1889 for idpackage, repo in my_matches: 1890 dbconn = self.open_server_repository(read_only = True, 1891 no_upload = True, repo = repo) 1892 self.updateProgress( 1893 "[%s=>%s|%s] %s " % ( 1894 darkgreen(repo), 1895 darkred(to_repo), 1896 brown(branch), 1897 blue(dbconn.retrieveAtom(idpackage)), 1898 ) + new_tag_string, 1899 importance = 0, 1900 type = "info", 1901 header = brown(" # ") 1902 ) 1903 # do we want to pull in also package dependencies? 1904 if pull_deps: 1905 dep_idpackages = my_qa.get_deep_dependency_list(dbconn, 1906 idpackage) 1907 for dep_idpackage in dep_idpackages: 1908 1909 my_dep_match = (dep_idpackage, repo,) 1910 if my_dep_match in pull_deps_matches: 1911 continue 1912 if my_dep_match in my_matches: 1913 continue 1914 1915 pull_deps_matches.append(my_dep_match) 1916 self.updateProgress( 1917 "[%s|%s] %s" % ( 1918 brown(branch), 1919 blue(_("dependency")), 1920 purple(dbconn.retrieveAtom(dep_idpackage)), 1921 ), 1922 importance = 0, 1923 type = "info", 1924 header = purple(" >> ") 1925 ) 1926 1927 if pull_deps: 1928 # put deps first! 1929 my_matches = pull_deps_matches + [x for x in my_matches if x not \ 1930 in pull_deps_matches] 1931 1932 if ask: 1933 rc_question = self.askQuestion(_("Would you like to continue ?")) 1934 if rc_question == "No": 1935 return switched 1936 1937 for idpackage, repo in my_matches: 1938 dbconn = self.open_server_repository(read_only = False, 1939 no_upload = True, repo = repo) 1940 match_branch = dbconn.retrieveBranch(idpackage) 1941 match_atom = dbconn.retrieveAtom(idpackage) 1942 package_filename = os.path.basename( 1943 dbconn.retrieveDownloadURL(idpackage)) 1944 self.updateProgress( 1945 "[%s=>%s|%s] %s: %s" % ( 1946 darkgreen(repo), 1947 darkred(to_repo), 1948 brown(branch), 1949 blue(_("switching")), 1950 darkgreen(match_atom), 1951 ), 1952 importance = 0, 1953 type = "info", 1954 header = red(" @@ "), 1955 back = True 1956 ) 1957 # move binary file 1958 from_file = os.path.join(self.get_local_packages_directory(repo), 1959 match_branch, package_filename) 1960 if not os.path.isfile(from_file): 1961 from_file = os.path.join(self.get_local_upload_directory(repo), 1962 match_branch, package_filename) 1963 if not os.path.isfile(from_file): 1964 self.updateProgress( 1965 "[%s=>%s|%s] %s: %s -> %s" % ( 1966 darkgreen(repo), 1967 darkred(to_repo), 1968 brown(branch), 1969 bold(_("cannot switch, package not found, skipping")), 1970 darkgreen(), 1971 red(from_file), 1972 ), 1973 importance = 1, 1974 type = "warning", 1975 header = darkred(" !!! ") 1976 ) 1977 continue 1978 1979 if new_tag != None: 1980 match_category = dbconn.retrieveCategory(idpackage) 1981 match_name = dbconn.retrieveName(idpackage) 1982 match_version = dbconn.retrieveVersion(idpackage) 1983 tagged_package_filename = \ 1984 self.entropyTools.create_package_filename( 1985 match_category, match_name, match_version, new_tag) 1986 to_file = os.path.join(self.get_local_upload_directory(to_repo), 1987 match_branch, tagged_package_filename) 1988 else: 1989 to_file = os.path.join(self.get_local_upload_directory(to_repo), 1990 match_branch, package_filename) 1991 if not os.path.isdir(os.path.dirname(to_file)): 1992 os.makedirs(os.path.dirname(to_file)) 1993 1994 copy_data = [ 1995 (from_file, to_file,), 1996 (from_file + etpConst['packagesmd5fileext'], 1997 to_file + etpConst['packagesmd5fileext'],), 1998 (from_file + etpConst['packagesexpirationfileext'], 1999 to_file + etpConst['packagesexpirationfileext'],) 2000 ] 2001 2002 for from_item, to_item in copy_data: 2003 self.updateProgress( 2004 "[%s=>%s|%s] %s: %s" % ( 2005 darkgreen(repo), 2006 darkred(to_repo), 2007 brown(branch), 2008 blue(_("moving file")), 2009 darkgreen(os.path.basename(from_item)), 2010 ), 2011 importance = 0, 2012 type = "info", 2013 header = red(" @@ "), 2014 back = True 2015 ) 2016 if os.path.isfile(from_item): 2017 shutil.copy2(from_item, to_item) 2018 2019 self.updateProgress( 2020 "[%s=>%s|%s] %s: %s" % ( 2021 darkgreen(repo), 2022 darkred(to_repo), 2023 brown(branch), 2024 blue(_("loading data from source database")), 2025 darkgreen(repo), 2026 ), 2027 importance = 0, 2028 type = "info", 2029 header = red(" @@ "), 2030 back = True 2031 ) 2032 # install package into destination db 2033 data = dbconn.getPackageData(idpackage) 2034 if new_tag != None: 2035 data['versiontag'] = new_tag 2036 2037 todbconn = self.open_server_repository(read_only = False, 2038 no_upload = True, repo = to_repo) 2039 2040 self.updateProgress( 2041 "[%s=>%s|%s] %s: %s" % ( 2042 darkgreen(repo), 2043 darkred(to_repo), 2044 brown(branch), 2045 blue(_("injecting data to destination database")), 2046 darkgreen(to_repo), 2047 ), 2048 importance = 0, 2049 type = "info", 2050 header = red(" @@ "), 2051 back = True 2052 ) 2053 new_idpackage, new_revision, new_data = todbconn.handlePackage(data) 2054 del data 2055 todbconn.commitChanges() 2056 2057 if not do_copy: 2058 self.updateProgress( 2059 "[%s=>%s|%s] %s: %s" % ( 2060 darkgreen(repo), 2061 darkred(to_repo), 2062 brown(branch), 2063 blue(_("removing entry from source database")), 2064 darkgreen(repo), 2065 ), 2066 importance = 0, 2067 type = "info", 2068 header = red(" @@ "), 2069 back = True 2070 ) 2071 2072 # remove package from old db 2073 dbconn.removePackage(idpackage) 2074 dbconn.commitChanges() 2075 2076 self.updateProgress( 2077 "[%s=>%s|%s] %s: %s" % ( 2078 darkgreen(repo), 2079 darkred(to_repo), 2080 brown(branch), 2081 blue(_("successfully handled atom")), 2082 darkgreen(match_atom), 2083 ), 2084 importance = 0, 2085 type = "info", 2086 header = blue(" @@ ") 2087 ) 2088 switched.add((idpackage, repo,)) 2089 2090 # just run this to make dev aware 2091 self.dependencies_test(to_repo) 2092 2093 return switched

2094 2095

2096 - def package_injector(self, package_file, inject = False, repo = None):

2097 2098 if repo == None: 2099 repo = self.default_repository 2100 2101 upload_dir = os.path.join(self.get_local_upload_directory(repo), 2102 self.SystemSettings['repositories']['branch']) 2103 if not os.path.isdir(upload_dir): 2104 os.makedirs(upload_dir) 2105 2106 dbconn = self.open_server_repository(read_only = False, 2107 no_upload = True, repo = repo) 2108 self.updateProgress( 2109 red("[repo: %s] %s: %s" % ( 2110 darkgreen(repo), 2111 _("adding package"), 2112 bold(os.path.basename(package_file)), 2113 ) 2114 ), 2115 importance = 1, 2116 type = "info", 2117 header = brown(" * "), 2118 back = True 2119 ) 2120 mydata = self.SpmService.extract_pkg_metadata(package_file, 2121 inject = inject) 2122 idpackage, revision, mydata = dbconn.handlePackage(mydata) 2123 2124 # set trashed counters 2125 trashing_counters = set() 2126 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2127 myserver_repos = srv_set['repositories'].keys() 2128 for myrepo in myserver_repos: 2129 mydbconn = self.open_server_repository(read_only = True, 2130 no_upload = True, repo = myrepo) 2131 mylist = mydbconn.retrieve_packages_to_remove( 2132 mydata['name'], 2133 mydata['category'], 2134 mydata['slot'], 2135 mydata['injected'] 2136 ) 2137 for myitem in mylist: 2138 trashing_counters.add(mydbconn.retrieveCounter(myitem)) 2139 2140 for mycounter in trashing_counters: 2141 dbconn.setTrashedCounter(mycounter) 2142 2143 # add package info to our current server repository 2144 dbconn.removePackageFromInstalledTable(idpackage) 2145 dbconn.addPackageToInstalledTable(idpackage, repo) 2146 atom = dbconn.retrieveAtom(idpackage) 2147 2148 self.updateProgress( 2149 "[repo:%s] %s: %s %s: %s" % ( 2150 darkgreen(repo), 2151 blue(_("added package")), 2152 darkgreen(atom), 2153 blue(_("rev")), # as in revision 2154 bold(str(revision)), 2155 ), 2156 importance = 1, 2157 type = "info", 2158 header = red(" @@ ") 2159 ) 2160 2161 manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage)) 2162 if manual_deps: 2163 self.updateProgress( 2164 "[repo:%s] %s: %s" % ( 2165 darkgreen(repo), 2166 blue(_("manual dependencies for")), 2167 darkgreen(atom), 2168 ), 2169 importance = 1, 2170 type = "warning", 2171 header = darkgreen(" ## ") 2172 ) 2173 for m_dep in manual_deps: 2174 self.updateProgress( 2175 brown(m_dep), 2176 importance = 1, 2177 type = "warning", 2178 header = darkred(" # ") 2179 ) 2180 2181 download_url = self._setup_repository_package_filename(idpackage, 2182 repo = repo) 2183 downloadfile = os.path.basename(download_url) 2184 destination_path = os.path.join(upload_dir, downloadfile) 2185 try: 2186 os.rename(package_file, destination_path) 2187 except OSError: 2188 shutil.move(package_file, destination_path) 2189 2190 dbconn.commitChanges() 2191 return idpackage, destination_path

2192 2193 # this function changes the final repository package filename

2194 - def _setup_repository_package_filename(self, idpackage, repo = None):

2195 2196 dbconn = self.open_server_repository(read_only = False, 2197 no_upload = True, repo = repo) 2198 2199 downloadurl = dbconn.retrieveDownloadURL(idpackage) 2200 packagerev = dbconn.retrieveRevision(idpackage) 2201 downloaddir = os.path.dirname(downloadurl) 2202 downloadfile = os.path.basename(downloadurl) 2203 # add revision 2204 downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev, 2205 etpConst['packagesext'],) 2206 downloadurl = os.path.join(downloaddir, downloadfile) 2207 2208 # update url 2209 dbconn.setDownloadURL(idpackage, downloadurl) 2210 2211 return downloadurl

2212

2213 - def add_packages_to_repository(self, packages_data, ask = True, 2214 repo = None):

2215 2216 if repo == None: 2217 repo = self.default_repository 2218 2219 mycount = 0 2220 maxcount = len(packages_data) 2221 idpackages_added = set() 2222 to_be_injected = set() 2223 my_qa = self.QA() 2224 missing_deps_taint = False 2225 for package_filepath, inject in packages_data: 2226 2227 mycount += 1 2228 self.updateProgress( 2229 "[repo:%s] %s: %s" % ( 2230 darkgreen(repo), 2231 blue(_("adding package")), 2232 darkgreen(os.path.basename(package_filepath)), 2233 ), 2234 importance = 1, 2235 type = "info", 2236 header = blue(" @@ "), 2237 count = (mycount, maxcount,) 2238 ) 2239 2240 try: 2241 # add to database 2242 idpackage, destination_path = self.package_injector( 2243 package_filepath, 2244 inject = inject, 2245 repo = repo 2246 ) 2247 idpackages_added.add(idpackage) 2248 to_be_injected.add((idpackage, destination_path)) 2249 except Exception, err: 2250 self.entropyTools.print_traceback() 2251 self.updateProgress( 2252 "[repo:%s] %s: %s" % ( 2253 darkgreen(repo), 2254 darkred(_("Exception caught, closing tasks")), 2255 darkgreen(unicode(err)), 2256 ), 2257 importance = 1, 2258 type = "error", 2259 header = bold(" !!! "), 2260 count = (mycount, maxcount,) 2261 ) 2262 # reinit depends table 2263 self.depends_table_initialize(repo) 2264 # reinit librarypathsidpackage table 2265 self.library_paths_table_initialize(repo) 2266 if idpackages_added: 2267 dbconn = self.open_server_repository(read_only = False, 2268 no_upload = True, repo = repo) 2269 missing_deps_taint = my_qa.test_missing_dependencies( 2270 idpackages_added, 2271 dbconn, 2272 ask = ask, 2273 repo = repo, 2274 self_check = True, 2275 black_list = \ 2276 self.get_missing_dependencies_blacklist( 2277 repo = repo), 2278 black_list_adder = \ 2279 self.add_missing_dependencies_blacklist_items 2280 ) 2281 my_qa.test_depends_linking(idpackages_added, dbconn, 2282 repo = repo) 2283 if to_be_injected: 2284 self.inject_database_into_packages(to_be_injected, 2285 repo = repo) 2286 # reinit depends table 2287 if missing_deps_taint: 2288 self.depends_table_initialize(repo) 2289 self.close_server_databases() 2290 raise 2291 2292 # reinit depends table 2293 self.depends_table_initialize(repo) 2294 # reinit librarypathsidpackage table 2295 self.library_paths_table_initialize(repo) 2296 2297 if idpackages_added: 2298 dbconn = self.open_server_repository(read_only = False, 2299 no_upload = True, repo = repo) 2300 missing_deps_taint = my_qa.test_missing_dependencies( 2301 idpackages_added, 2302 dbconn, 2303 ask = ask, 2304 repo = repo, 2305 self_check = True, 2306 black_list = \ 2307 self.get_missing_dependencies_blacklist(repo = repo), 2308 black_list_adder = \ 2309 self.add_missing_dependencies_blacklist_items 2310 ) 2311 my_qa.test_depends_linking(idpackages_added, dbconn, repo = repo) 2312 2313 # reinit depends table 2314 if missing_deps_taint: 2315 self.depends_table_initialize(repo) 2316 2317 # inject database into packages 2318 self.inject_database_into_packages(to_be_injected, repo = repo) 2319 2320 return idpackages_added

2321 2322

2323 - def inject_database_into_packages(self, injection_data, repo = None):

2324 2325 if repo == None: 2326 repo = self.default_repository 2327 2328 # now inject metadata into tbz2 packages 2329 self.updateProgress( 2330 "[repo:%s] %s:" % ( 2331 darkgreen(repo), 2332 blue(_("Injecting entropy metadata into built packages")), 2333 ), 2334 importance = 1, 2335 type = "info", 2336 header = red(" @@ ") 2337 ) 2338 2339 dbconn = self.open_server_repository(read_only = False, 2340 no_upload = True, repo = repo) 2341 for idpackage, package_path in injection_data: 2342 self.updateProgress( 2343 "[repo:%s|%s] %s: %s" % ( 2344 darkgreen(repo), 2345 brown(str(idpackage)), 2346 blue(_("injecting entropy metadata")), 2347 darkgreen(os.path.basename(package_path)), 2348 ), 2349 importance = 1, 2350 type = "info", 2351 header = blue(" @@ "), 2352 back = True 2353 ) 2354 data = dbconn.getPackageData(idpackage) 2355 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2356 dbpath = self.ClientService.inject_entropy_database_into_package( 2357 package_path, data, treeupdates_actions) 2358 digest = self.entropyTools.md5sum(package_path) 2359 # update digest 2360 dbconn.setDigest(idpackage, digest) 2361 # update signatures 2362 signatures = data['signatures'].copy() 2363 for hash_key in sorted(signatures): 2364 hash_func = getattr(self.entropyTools, hash_key) 2365 signatures[hash_key] = hash_func(package_path) 2366 dbconn.setSignatures(idpackage, signatures['sha1'], 2367 signatures['sha256'], signatures['sha512']) 2368 self.entropyTools.create_md5_file(package_path) 2369 # remove garbage 2370 os.remove(dbpath) 2371 self.updateProgress( 2372 "[repo:%s|%s] %s: %s" % ( 2373 darkgreen(repo), 2374 brown(str(idpackage)), 2375 blue(_("injection complete")), 2376 darkgreen(os.path.basename(package_path)), 2377 ), 2378 importance = 1, 2379 type = "info", 2380 header = red(" @@ ") 2381 ) 2382 dbconn.commitChanges()

2383

2384 - def check_config_file_updates(self):

2385 self.updateProgress( 2386 "[%s] %s" % ( 2387 red(_("config files")), # something short please 2388 blue(_("checking system")), 2389 ), 2390 importance = 1, 2391 type = "info", 2392 header = blue(" @@ "), 2393 back = True 2394 ) 2395 # scanning for config files not updated 2396 scandata = self.ClientService.FileUpdates.scanfs(dcache = False) 2397 if scandata: 2398 self.updateProgress( 2399 "[%s] %s" % ( 2400 red(_("config files")), # something short please 2401 blue(_("there are configuration files not updated yet")), 2402 ), 2403 importance = 1, 2404 type = "error", 2405 header = darkred(" @@ ") 2406 ) 2407 for key in scandata: 2408 self.updateProgress( 2409 "%s" % (brown(etpConst['systemroot'] + \ 2410 scandata[key]['destination'])), 2411 importance = 1, 2412 type = "info", 2413 header = "\t" 2414 ) 2415 return True 2416 return False

2417

2418 - def quickpkg(self, atom, storedir):

2419 return self.SpmService.quickpkg(atom, storedir)

2420 2421

2422 - def remove_packages(self, idpackages, repo = None):

2423 2424 if repo == None: 2425 repo = self.default_repository 2426 2427 dbconn = self.open_server_repository(read_only = False, 2428 no_upload = True, repo = repo) 2429 for idpackage in idpackages: 2430 atom = dbconn.retrieveAtom(idpackage) 2431 self.updateProgress( 2432 "[repo:%s] %s: %s" % ( 2433 darkgreen(repo), 2434 blue(_("removing package")), 2435 darkgreen(atom), 2436 ), 2437 importance = 1, 2438 type = "info", 2439 header = brown(" @@ ") 2440 ) 2441 dbconn.removePackage(idpackage) 2442 self.close_server_database(dbconn) 2443 self.updateProgress( 2444 "[repo:%s] %s" % ( 2445 darkgreen(repo), 2446 blue(_("removal complete")), 2447 ), 2448 importance = 1, 2449 type = "info", 2450 header = brown(" @@ ") 2451 )

2452 2453

2454 - def bump_database(self, repo = None):

2455 dbconn = self.open_server_repository(read_only = False, 2456 no_upload = True, repo = repo) 2457 dbconn.taintDatabase() 2458 self.close_server_database(dbconn)

2459

2460 - def get_remote_mirrors(self, repo = None):

2461 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2462 if repo == None: 2463 repo = self.default_repository 2464 return srv_set['repositories'][repo]['mirrors'][:]

2465

2466 - def get_remote_packages_relative_path(self, repo = None):

2467 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2468 if repo == None: 2469 repo = self.default_repository 2470 return srv_set['repositories'][repo]['packages_relative_path']

2471

2472 - def get_remote_database_relative_path(self, repo = None):

2473 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2474 if repo == None: 2475 repo = self.default_repository 2476 return srv_set['repositories'][repo]['database_relative_path']

2477

2478 - def get_local_database_file(self, repo = None, branch = None):

2479 if repo == None: 2480 repo = self.default_repository 2481 return os.path.join(self.get_local_database_dir(repo, branch), 2482 etpConst['etpdatabasefile'])

2483

2484 - def get_local_store_directory(self, repo = None):

2485 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2486 if repo == None: 2487 repo = self.default_repository 2488 return srv_set['repositories'][repo]['store_dir']

2489

2490 - def get_local_upload_directory(self, repo = None):

2491 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2492 if repo == None: 2493 repo = self.default_repository 2494 return srv_set['repositories'][repo]['upload_dir']

2495

2496 - def get_local_packages_directory(self, repo = None):

2497 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2498 if repo == None: 2499 repo = self.default_repository 2500 return srv_set['repositories'][repo]['packages_dir']

2501

2502 - def get_local_database_taint_file(self, repo = None, branch = None):

2503 if repo == None: 2504 repo = self.default_repository 2505 return os.path.join(self.get_local_database_dir(repo, branch), 2506 etpConst['etpdatabasetaintfile'])

2507

2508 - def get_local_database_revision_file(self, repo = None, branch = None):

2509 if repo == None: 2510 repo = self.default_repository 2511 return os.path.join(self.get_local_database_dir(repo, branch), 2512 etpConst['etpdatabaserevisionfile'])

2513

2514 - def get_local_database_timestamp_file(self, repo = None, branch = None):

2515 if repo == None: 2516 repo = self.default_repository 2517 return os.path.join(self.get_local_database_dir(repo, branch), 2518 etpConst['etpdatabasetimestampfile'])

2519

2520 - def get_local_database_ca_cert_file(self, repo = None, branch = None):

2521 if repo == None: 2522 repo = self.default_repository 2523 return os.path.join(self.get_local_database_dir(repo, branch), 2524 etpConst['etpdatabasecacertfile'])

2525

2526 - def get_local_database_server_cert_file(self, repo = None, branch = None):

2527 if repo == None: 2528 repo = self.default_repository 2529 return os.path.join(self.get_local_database_dir(repo, branch), 2530 etpConst['etpdatabaseservercertfile'])

2531

2532 - def get_local_database_mask_file(self, repo = None, branch = None):

2533 if repo == None: 2534 repo = self.default_repository 2535 return os.path.join(self.get_local_database_dir(repo, branch), 2536 etpConst['etpdatabasemaskfile'])

2537

2538 - def get_local_database_system_mask_file(self, repo = None, branch = None):

2539 if repo == None: 2540 repo = self.default_repository 2541 return os.path.join(self.get_local_database_dir(repo, branch), 2542 etpConst['etpdatabasesytemmaskfile'])

2543

2544 - def get_local_database_confl_tagged_file(self, repo = None, branch = None):

2545 if repo == None: 2546 repo = self.default_repository 2547 return os.path.join(self.get_local_database_dir(repo, branch), 2548 etpConst['etpdatabaseconflictingtaggedfile'])

2549

2550 - def get_local_database_licensewhitelist_file(self, repo = None, 2551 branch = None):

2552 2553 if repo == None: 2554 repo = self.default_repository 2555 return os.path.join(self.get_local_database_dir(repo, branch), 2556 etpConst['etpdatabaselicwhitelistfile'])

2557

2558 - def get_local_database_rss_file(self, repo = None, branch = None):

2559 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2560 if repo == None: 2561 repo = self.default_repository 2562 return os.path.join(self.get_local_database_dir(repo, branch), 2563 srv_set['rss']['name'])

2564

2565 - def get_local_database_rsslight_file(self, repo = None, branch = None):

2566 if repo == None: 2567 repo = self.default_repository 2568 return os.path.join(self.get_local_database_dir(repo, branch), 2569 etpConst['rss-light-name'])

2570

2571 - def get_local_database_notice_board_file(self, repo = None, branch = None):

2572 if repo == None: 2573 repo = self.default_repository 2574 return os.path.join(self.get_local_database_dir(repo, branch), 2575 etpConst['rss-notice-board'])

2576

2577 - def get_local_database_treeupdates_file(self, repo = None, branch = None):

2578 if repo == None: 2579 repo = self.default_repository 2580 return os.path.join(self.get_local_database_dir(repo, branch), 2581 etpConst['etpdatabaseupdatefile'])

2582

2583 - def get_local_database_compressed_metafiles_file(self, repo = None, 2584 branch = None):

2585 2586 if repo == None: 2587 repo = self.default_repository 2588 return os.path.join(self.get_local_database_dir(repo, branch), 2589 etpConst['etpdatabasemetafilesfile'])

2590

2591 - def get_local_database_metafiles_not_found_file(self, repo = None, 2592 branch = None):

2593 2594 if repo == None: 2595 repo = self.default_repository 2596 return os.path.join(self.get_local_database_dir(repo, branch), 2597 etpConst['etpdatabasemetafilesnotfound'])

2598

2599 - def get_local_exp_based_pkgs_rm_whitelist_file(self, repo = None, 2600 branch = None):

2601 if repo == None: 2602 repo = self.default_repository 2603 return os.path.join(self.get_local_database_dir(repo, branch), 2604 etpConst['etpdatabaseexpbasedpkgsrm'])

2605

2606 - def get_local_pkglist_file(self, repo = None, branch = None):

2607 if repo == None: 2608 repo = self.default_repository 2609 return os.path.join(self.get_local_database_dir(repo, branch), 2610 etpConst['etpdatabasepkglist'])

2611

2612 - def get_local_database_sets_dir(self, repo = None, branch = None):

2613 if repo == None: 2614 repo = self.default_repository 2615 return os.path.join(self.get_local_database_dir(repo, branch), 2616 etpConst['confsetsdirname'])

2617

2618 - def get_local_post_branch_mig_script(self, repo = None, branch = None):

2619 if repo == None: 2620 repo = self.default_repository 2621 return os.path.join(self.get_local_database_dir(repo, branch), 2622 etpConst['etp_post_branch_hop_script'])

2623

2624 - def get_local_post_branch_upg_script(self, repo = None, branch = None):

2625 if repo == None: 2626 repo = self.default_repository 2627 return os.path.join(self.get_local_database_dir(repo, branch), 2628 etpConst['etp_post_branch_upgrade_script'])

2629

2630 - def get_local_database_dir(self, repo = None, branch = None):

2631 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2632 if repo == None: 2633 repo = self.default_repository 2634 if branch == None: 2635 branch = self.SystemSettings['repositories']['branch'] 2636 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2637 branch)

2638

2639 - def get_missing_dependencies_blacklist_file(self, repo = None, 2640 branch = None):

2641 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2642 if repo == None: 2643 repo = self.default_repository 2644 if branch == None: 2645 branch = self.SystemSettings['repositories']['branch'] 2646 return os.path.join(srv_set['repositories'][repo]['database_dir'], 2647 branch, etpConst['etpdatabasemissingdepsblfile'])

2648

2649 - def get_missing_dependencies_blacklist(self, repo = None, branch = None):

2650 if repo == None: 2651 repo = self.default_repository 2652 if branch == None: 2653 branch = self.SystemSettings['repositories']['branch'] 2654 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2655 wl_data = [] 2656 if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK): 2657 f_wl = open(wl_file,"r") 2658 wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \ 2659 not x.strip().startswith("#")] 2660 f_wl.close() 2661 return set(wl_data)

2662

2663 - def add_missing_dependencies_blacklist_items(self, items, repo = None, 2664 branch = None):

2665 2666 if repo == None: 2667 repo = self.default_repository 2668 if branch == None: 2669 branch = self.SystemSettings['repositories']['branch'] 2670 wl_file = self.get_missing_dependencies_blacklist_file(repo, branch) 2671 wl_dir = os.path.dirname(wl_file) 2672 if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)): 2673 return 2674 if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK): 2675 return 2676 f_wl = open(wl_file,"a+") 2677 f_wl.write('\n'.join(items)+'\n') 2678 f_wl.flush() 2679 f_wl.close()

2680

2681 - def get_local_database_revision(self, repo = None):

2682 2683 if repo == None: 2684 repo = self.default_repository 2685 2686 dbrev_file = self.get_local_database_revision_file(repo) 2687 if os.path.isfile(dbrev_file): 2688 f_rev = open(dbrev_file) 2689 rev = f_rev.readline().strip() 2690 f_rev.close() 2691 try: 2692 rev = int(rev) 2693 except ValueError: 2694 self.updateProgress( 2695 "[repo:%s] %s: %s - %s" % ( 2696 darkgreen(repo), 2697 blue(_("invalid database revision")), 2698 bold(rev), 2699 blue(_("defaulting to 0")), 2700 ), 2701 importance = 2, 2702 type = "error", 2703 header = darkred(" !!! ") 2704 ) 2705 rev = 0 2706 return rev 2707 else: 2708 return 0

2709

2710 - def get_remote_database_revision(self, repo = None):

2711 2712 if repo == None: 2713 repo = self.default_repository 2714 2715 remote_status = self.MirrorsService.get_remote_databases_status(repo) 2716 if not [x for x in remote_status if x[1]]: 2717 remote_revision = 0 2718 else: 2719 remote_revision = max([x[1] for x in remote_status]) 2720 2721 return remote_revision

2722

2723 - def get_branch_from_download_relative_uri(self, mypath):

2724 return self.ClientService.get_branch_from_download_relative_uri(mypath)

2725

2726 - def get_current_timestamp(self):

2727 from datetime import datetime 2728 import time 2729 return "%s" % (datetime.fromtimestamp(time.time()),)

2730

2731 - def create_repository_pkglist(self, repo = None, branch = None):

2732 pkglist_file = self.get_local_pkglist_file(repo = repo, branch = branch) 2733 2734 tmp_pkglist_file = pkglist_file + ".tmp" 2735 dbconn = self.open_server_repository(repo = repo, just_reading = True, 2736 do_treeupdates = False) 2737 pkglist = dbconn.listAllDownloads(do_sort = True, full_path = True) 2738 2739 with open(tmp_pkglist_file, "w") as pkg_f: 2740 for pkg in pkglist: 2741 pkg_f.write(pkg + "\n") 2742 pkg_f.flush() 2743 2744 os.rename(tmp_pkglist_file, pkglist_file)

2745

2746 - def package_set_list(self, *args, **kwargs):

2747 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2748 repos = srv_set['repositories'].keys() 2749 kwargs['server_repos'] = repos 2750 kwargs['serverInstance'] = self 2751 return self.ClientService.package_set_list(*args, **kwargs)

2752

2753 - def package_set_search(self, *args, **kwargs):

2754 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2755 repos = srv_set['repositories'].keys() 2756 kwargs['server_repos'] = repos 2757 kwargs['serverInstance'] = self 2758 return self.ClientService.package_set_search(*args, **kwargs)

2759

2760 - def package_set_match(self, *args, **kwargs):

2761 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2762 repos = srv_set['repositories'].keys() 2763 kwargs['server_repos'] = repos 2764 kwargs['serverInstance'] = self 2765 return self.ClientService.package_set_match(*args, **kwargs)

2766

2767 - def atom_match(self, *args, **kwargs):

2768 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2769 repos = srv_set['repositories'].keys() 2770 kwargs['server_repos'] = repos 2771 kwargs['serverInstance'] = self 2772 return self.ClientService.atom_match(*args, **kwargs)

2773

2774 - def scan_package_changes(self):

2775 2776 installed_packages = self.SpmService.get_installed_packages_counter() 2777 installed_counters = set() 2778 to_be_added = set() 2779 to_be_removed = set() 2780 to_be_injected = set() 2781 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2782 exp_based_scope = my_settings['exp_based_scope'] 2783 2784 server_repos = my_settings['repositories'].keys() 2785 exp_pkgs_cache = {} 2786 2787 # packages to be added 2788 for spm_atom, spm_counter in installed_packages: 2789 found = False 2790 for server_repo in server_repos: 2791 installed_counters.add(spm_counter) 2792 server_dbconn = self.open_server_repository(read_only = True, 2793 no_upload = True, repo = server_repo) 2794 counter = server_dbconn.isCounterAvailable(spm_counter) 2795 if counter: 2796 found = True 2797 break 2798 if not found: 2799 to_be_added.add((spm_atom, spm_counter,)) 2800 2801 # packages to be removed from the database 2802 database_counters = {} 2803 for server_repo in server_repos: 2804 server_dbconn = self.open_server_repository(read_only = True, 2805 no_upload = True, repo = server_repo) 2806 database_counters[server_repo] = server_dbconn.listAllCounters( 2807 branch = self.SystemSettings['repositories']['branch']) 2808 2809 ordered_counters = set() 2810 for server_repo in database_counters: 2811 for data in database_counters[server_repo]: 2812 ordered_counters.add((data, server_repo)) 2813 database_counters = ordered_counters 2814 2815 for (counter, idpackage,), xrepo in database_counters: 2816 2817 if counter < 0: 2818 continue # skip packages without valid counter 2819 2820 if counter in installed_counters: 2821 continue 2822 2823 dbconn = self.open_server_repository(read_only = True, 2824 no_upload = True, repo = xrepo) 2825 2826 dorm = True 2827 # check if the package is in to_be_added 2828 if to_be_added: 2829 2830 dorm = False 2831 atom = dbconn.retrieveAtom(idpackage) 2832 atomkey = self.entropyTools.dep_getkey(atom) 2833 atomtag = self.entropyTools.dep_gettag(atom) 2834 atomslot = dbconn.retrieveSlot(idpackage) 2835 2836 add = True 2837 for spm_atom, spm_counter in to_be_added: 2838 addslot = self.SpmService.get_installed_package_slot( 2839 spm_atom) 2840 addkey = self.entropyTools.dep_getkey(spm_atom) 2841 # workaround for ebuilds not having slot 2842 if addslot == None: 2843 addslot = '0' 2844 # atomtag != None is for handling tagged pkgs correctly 2845 if (atomkey == addkey) and \ 2846 ((str(atomslot) == str(addslot)) or (atomtag != None)): 2847 # do not add to to_be_removed 2848 add = False 2849 break 2850 2851 if not add: 2852 continue 2853 dorm = True 2854 2855 # checking if we are allowed to remove stuff on this repo 2856 # it xrepo is not the default one, we MUST skip this to 2857 # avoid touching what developer doesn't expect 2858 if dorm and (xrepo == self.default_repository): 2859 trashed = self.is_counter_trashed(counter) 2860 if trashed: 2861 # search into portage then 2862 try: 2863 key, slot = dbconn.retrieveKeySlot(idpackage) 2864 trashed = self.SpmService.get_installed_atom( 2865 key+":"+slot) 2866 except TypeError: # referred to retrieveKeySlot 2867 trashed = True 2868 if not trashed: 2869 2870 dbtag = dbconn.retrieveVersionTag(idpackage) 2871 if dbtag: 2872 is_injected = dbconn.isInjected(idpackage) 2873 if not is_injected: 2874 to_be_injected.add((idpackage, xrepo)) 2875 2876 elif exp_based_scope: 2877 2878 # check if support for this is set 2879 plg_id = self.sys_settings_fatscope_plugin_id 2880 exp_data = self.SystemSettings[plg_id]['repos'].get( 2881 xrepo, set()) 2882 2883 # only some packages are set, check if our is 2884 # in the list 2885 if (idpackage not in exp_data) and (-1 not in exp_data): 2886 to_be_removed.add((idpackage, xrepo)) 2887 continue 2888 2889 idpackage_expired = self.is_match_expired((idpackage, 2890 xrepo,)) 2891 2892 if idpackage_expired: 2893 # expired !!! 2894 # add this and its depends (reverse deps) 2895 to_be_removed.add((idpackage, xrepo)) 2896 for my_id in dbconn.retrieveDepends(idpackage): 2897 to_be_removed.add((my_id, xrepo)) 2898 2899 else: 2900 to_be_removed.add((idpackage, xrepo)) 2901 2902 return to_be_added, to_be_removed, to_be_injected

2903

2904 - def is_match_expired(self, match):

2905 2906 idpackage, repoid = match 2907 dbconn = self.open_server_repository(repo = repoid, just_reading = True) 2908 # 3600 * 24 = 86400 2909 my_settings = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2910 pkg_exp_secs = my_settings['packages_expiration_days'] * 86400 2911 cur_unix_time = self.entropyTools.get_current_unix_time() 2912 # if packages removal is triggered by expiration 2913 # we will have to check if our package is really 2914 # expired and remove its reverse deps too 2915 mydate = dbconn.retrieveDateCreation(idpackage) 2916 # cross fingers hoping that time is set correctly 2917 mydelta = cur_unix_time - float(mydate) 2918 if mydelta > pkg_exp_secs: 2919 return True 2920 return False

2921

2922 - def is_counter_trashed(self, counter):

2923 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2924 server_repos = srv_set['repositories'].keys() 2925 for repo in server_repos: 2926 dbconn = self.open_server_repository(read_only = True, 2927 no_upload = True, repo = repo) 2928 if dbconn.isCounterTrashed(counter): 2929 return True 2930 return False

2931

2932 - def transform_package_into_injected(self, idpackage, repo = None):

2933 dbconn = self.open_server_repository(read_only = False, 2934 no_upload = True, repo = repo) 2935 counter = dbconn.getNewNegativeCounter() 2936 dbconn.setCounter(idpackage, counter) 2937 dbconn.setInjected(idpackage)

2938

2939 - def initialize_server_database(self, empty = True, repo = None, 2940 warnings = True):

2941 2942 if repo == None: 2943 repo = self.default_repository 2944 2945 self.close_server_databases() 2946 revisions_match = {} 2947 treeupdates_actions = [] 2948 injected_packages = set() 2949 idpackages = set() 2950 idpackages_added = set() 2951 2952 mytxt = red("%s ...") % (_("Initializing Entropy database"),) 2953 self.updateProgress( 2954 mytxt, importance = 1, 2955 type = "info", header = darkgreen(" * "), 2956 back = True 2957 ) 2958 2959 if os.path.isfile(self.get_local_database_file(repo)): 2960 2961 dbconn = self.open_server_repository(read_only = True, 2962 no_upload = True, repo = repo, warnings = warnings) 2963 2964 if dbconn.doesTableExist("baseinfo") and \ 2965 dbconn.doesTableExist("extrainfo"): 2966 idpackages = dbconn.listAllIdpackages() 2967 2968 if dbconn.doesTableExist("treeupdatesactions"): 2969 treeupdates_actions = dbconn.listAllTreeUpdatesActions() 2970 2971 # save list of injected packages 2972 if dbconn.doesTableExist("injected") and \ 2973 dbconn.doesTableExist("extrainfo"): 2974 injected_packages = dbconn.listAllInjectedPackages( 2975 justFiles = True) 2976 injected_packages = set([os.path.basename(x) for x \ 2977 in injected_packages]) 2978 2979 for idpackage in idpackages: 2980 url = dbconn.retrieveDownloadURL(idpackage) 2981 package = os.path.basename(url) 2982 branch = dbconn.retrieveBranch(idpackage) 2983 revision = dbconn.retrieveRevision(idpackage) 2984 revisions_match[package] = (branch, revision,) 2985 2986 self.close_server_database(dbconn) 2987 2988 mytxt = "%s: %s: %s" % ( 2989 bold(_("WARNING")), 2990 red(_("database already exists")), 2991 self.get_local_database_file(repo), 2992 ) 2993 self.updateProgress( 2994 mytxt, 2995 importance = 1, 2996 type = "warning", 2997 header = darkred(" !!! ") 2998 ) 2999 3000 rc_question = self.askQuestion(_("Do you want to continue ?")) 3001 if rc_question == "No": 3002 return 3003 try: 3004 os.remove(self.get_local_database_file(repo)) 3005 except OSError: 3006 pass 3007 3008 3009 # initialize 3010 dbconn = self.open_server_repository(read_only = False, 3011 no_upload = True, repo = repo, is_new = True) 3012 dbconn.initializeDatabase() 3013 3014 if not empty: 3015 3016 revisions_file = "/entropy-revisions-dump.txt" 3017 # dump revisions - as a backup 3018 if revisions_match: 3019 self.updateProgress( 3020 "%s: %s" % ( 3021 red(_("Dumping current revisions to file")), 3022 darkgreen(revisions_file), 3023 ), 3024 importance = 1, 3025 type = "info", 3026 header = darkgreen(" * ") 3027 ) 3028 f_rev = open(revisions_file,"w") 3029 f_rev.write(str(revisions_match)) 3030 f_rev.flush() 3031 f_rev.close() 3032 3033 # dump treeupdates - as a backup 3034 treeupdates_file = "/entropy-treeupdates-dump.txt" 3035 if treeupdates_actions: 3036 self.updateProgress( 3037 "%s: %s" % ( 3038 # do not translate treeupdates 3039 red(_("Dumping current 'treeupdates' actions to file")), 3040 bold(treeupdates_file), 3041 ), 3042 importance = 1, 3043 type = "info", 3044 header = darkgreen(" * ") 3045 ) 3046 f_tree = open(treeupdates_file,"w") 3047 f_tree.write(str(treeupdates_actions)) 3048 f_tree.flush() 3049 f_tree.close() 3050 3051 rc_question = self.askQuestion( 3052 _("Would you like to sync packages first (important!) ?")) 3053 if rc_question == "Yes": 3054 self.MirrorsService.sync_packages(repo = repo) 3055 3056 # fill tree updates actions 3057 if treeupdates_actions: 3058 dbconn.bumpTreeUpdatesActions(treeupdates_actions) 3059 3060 # now fill the database 3061 pkg_branch_dir = os.path.join( 3062 self.get_local_packages_directory(repo), 3063 self.SystemSettings['repositories']['branch']) 3064 pkglist = os.listdir(pkg_branch_dir) 3065 # filter .md5 and .expired packages 3066 pkg_ext_len = len(etpConst['packagesext']) 3067 pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \ 3068 etpConst['packagesext']) and not \ 3069 os.path.isfile(os.path.join(pkg_branch_dir, 3070 x + etpConst['packagesexpirationfileext']))] 3071 3072 if pkglist: 3073 self.updateProgress( 3074 "%s '%s' %s %s" % ( 3075 red(_("Reinitializing Entropy database for branch")), 3076 bold(self.SystemSettings['repositories']['branch']), 3077 red(_("using Packages in the repository")), 3078 red("..."), 3079 ), 3080 importance = 1, 3081 type = "info", 3082 header = darkgreen(" * ") 3083 ) 3084 3085 counter = 0 3086 maxcount = len(pkglist) 3087 branch = self.SystemSettings['repositories']['branch'] 3088 for pkg in pkglist: 3089 counter += 1 3090 3091 self.updateProgress( 3092 "[repo:%s|%s] %s: %s" % ( 3093 darkgreen(repo), 3094 brown(branch), 3095 blue(_("analyzing")), 3096 bold(pkg), 3097 ), 3098 importance = 1, 3099 type = "info", 3100 header = " ", 3101 back = True, 3102 count = (counter, maxcount,) 3103 ) 3104 3105 doinject = False 3106 if pkg in injected_packages: 3107 doinject = True 3108 3109 pkg_path = os.path.join(self.get_local_packages_directory(repo), 3110 branch, pkg) 3111 mydata = self.SpmService.extract_pkg_metadata(pkg_path, 3112 inject = doinject) 3113 3114 # get previous revision 3115 revision_avail = revisions_match.get(pkg) 3116 add_revision = 0 3117 if (revision_avail != None): 3118 if branch == revision_avail[0]: 3119 add_revision = revision_avail[1] 3120 3121 idpackage, revision, mydata_upd = dbconn.addPackage(mydata, 3122 revision = add_revision) 3123 idpackages_added.add(idpackage) 3124 3125 self.updateProgress( 3126 "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % ( 3127 repo, 3128 brown(branch), 3129 darkgreen(str(counter)), 3130 blue(str(maxcount)), 3131 red(_("added package")), 3132 darkgreen(pkg), 3133 red(_("revision")), 3134 brown(str(revision)), 3135 ), 3136 importance = 1, 3137 type = "info", 3138 header = " ", 3139 back = True 3140 ) 3141 3142 self.depends_table_initialize(repo) 3143 self.library_paths_table_initialize(repo) 3144 3145 my_qa = self.QA() 3146 3147 if idpackages_added: 3148 dbconn = self.open_server_repository(read_only = False, 3149 no_upload = True, repo = repo) 3150 my_qa.test_missing_dependencies( 3151 idpackages_added, dbconn, ask = True, 3152 repo = repo, self_check = True, 3153 black_list = \ 3154 self.get_missing_dependencies_blacklist(repo = repo), 3155 black_list_adder = \ 3156 self.add_missing_dependencies_blacklist_items 3157 ) 3158 3159 dbconn.commitChanges() 3160 self.close_server_databases() 3161 3162 return 0

3163

3164 - def match_packages(self, packages, repo = None):

3165 3166 dbconn = self.open_server_repository(read_only = True, 3167 no_upload = True, repo = repo) 3168 if ("world" in packages) or not packages: 3169 return dbconn.listAllIdpackages(), True 3170 else: 3171 idpackages = set() 3172 for package in packages: 3173 matches = dbconn.atomMatch(package, multiMatch = True) 3174 if matches[1] == 0: 3175 idpackages |= matches[0] 3176 else: 3177 mytxt = "%s: %s: %s" % ( 3178 red(_("Attention")), 3179 blue(_("cannot match")), 3180 bold(package), 3181 ) 3182 self.updateProgress( 3183 mytxt, 3184 importance = 1, 3185 type = "warning", 3186 header = darkred(" !!! ") 3187 ) 3188 return idpackages, False

3189

3190 - def get_remote_package_checksum(self, repo, filename, branch):

3191 3192 import urllib2 3193 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3194 if not srv_set['repositories'][repo].has_key('handler'): 3195 return None 3196 url = srv_set['repositories'][repo]['handler'] 3197 3198 # does the package has "#" (== tag) ? hackish thing that works 3199 filename = filename.replace("#","%23") 3200 # "+" 3201 filename = filename.replace("+","%2b") 3202 request = os.path.join(url, etpConst['handlers']['md5sum']) 3203 request += filename+"&branch="+branch 3204 3205 proxy_settings = self.SystemSettings['system']['proxy'] 3206 try: 3207 mydict = {} 3208 if proxy_settings['ftp']: 3209 mydict['ftp'] = proxy_settings['ftp'] 3210 if proxy_settings['http']: 3211 mydict['http'] = proxy_settings['http'] 3212 if mydict: 3213 mydict['username'] = proxy_settings['username'] 3214 mydict['password'] = proxy_settings['password'] 3215 self.entropyTools.add_proxy_opener(urllib2, mydict) 3216 else: 3217 # unset 3218 urllib2._opener = None 3219 item = urllib2.urlopen(request) 3220 result = item.readline().strip() 3221 item.close() 3222 del item 3223 return result 3224 except (urllib2.URLError, urllib2.HTTPError,): # no HTTP support? 3225 return None

3226

3227 - def verify_remote_packages(self, packages, ask = True, repo = None):

3228 3229 if repo == None: 3230 repo = self.default_repository 3231 3232 self.updateProgress( 3233 "[%s] %s:" % ( 3234 red("remote"), 3235 blue(_("Integrity verification of the selected packages")), 3236 ), 3237 importance = 1, 3238 type = "info", 3239 header = blue(" @@ ") 3240 ) 3241 3242 idpackages, world = self.match_packages(packages) 3243 dbconn = self.open_server_repository(read_only = True, no_upload = True, 3244 repo = repo) 3245 branch = self.SystemSettings['repositories']['branch'] 3246 3247 if world: 3248 self.updateProgress( 3249 blue( 3250 _("All the packages in repository will be checked.")), 3251 importance = 1, 3252 type = "info", 3253 header = " " 3254 ) 3255 else: 3256 mytxt = red("%s:") % ( 3257 _("This is the list of the packages that would be checked"),) 3258 self.updateProgress( 3259 mytxt, 3260 importance = 1, 3261 type = "info", 3262 header = " " 3263 ) 3264 for idpackage in idpackages: 3265 pkgatom = dbconn.retrieveAtom(idpackage) 3266 down_url = dbconn.retrieveDownloadURL(idpackage) 3267 pkgfile = os.path.basename(down_url) 3268 self.updateProgress( 3269 red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)), 3270 importance = 1, 3271 type = "info", 3272 header = darkgreen(" - ") 3273 ) 3274 3275 if ask: 3276 rc_question = self.askQuestion( 3277 _("Would you like to continue ?")) 3278 if rc_question == "No": 3279 return set(), set(), {} 3280 3281 match = set() 3282 not_match = set() 3283 broken_packages = {} 3284 3285 for uri in self.get_remote_mirrors(repo): 3286 3287 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3288 self.updateProgress( 3289 "[repo:%s] %s: %s" % ( 3290 darkgreen(repo), 3291 blue(_("Working on mirror")), 3292 brown(crippled_uri), 3293 ), 3294 importance = 1, 3295 type = "info", 3296 header = red(" @@ ") 3297 ) 3298 3299 3300 totalcounter = len(idpackages) 3301 currentcounter = 0 3302 for idpackage in idpackages: 3303 3304 currentcounter += 1 3305 pkgfile = dbconn.retrieveDownloadURL(idpackage) 3306 orig_branch = self.get_branch_from_download_relative_uri( 3307 pkgfile) 3308 3309 self.updateProgress( 3310 "[%s] %s: %s" % ( 3311 brown(crippled_uri), 3312 blue(_("checking hash")), 3313 darkgreen(pkgfile), 3314 ), 3315 importance = 1, 3316 type = "info", 3317 header = blue(" @@ "), 3318 back = True, 3319 count = (currentcounter, totalcounter,) 3320 ) 3321 3322 checksum_ok = False 3323 ck_remote = self.get_remote_package_checksum(repo, 3324 os.path.basename(pkgfile), orig_branch) 3325 if ck_remote == None: 3326 self.updateProgress( 3327 "[%s] %s: %s %s" % ( 3328 brown(crippled_uri), 3329 blue(_("digest verification of")), 3330 bold(pkgfile), 3331 blue(_("not supported")), 3332 ), 3333 importance = 1, 3334 type = "info", 3335 header = blue(" @@ "), 3336 count = (currentcounter, totalcounter,) 3337 ) 3338 elif len(ck_remote) == 32: 3339 pkghash = dbconn.retrieveDigest(idpackage) 3340 if ck_remote == pkghash: 3341 checksum_ok = True 3342 else: 3343 self.updateProgress( 3344 "[%s] %s: %s %s" % ( 3345 brown(crippled_uri), 3346 blue(_("digest verification of")), 3347 bold(pkgfile), 3348 blue(_("failed for unknown reasons")), 3349 ), 3350 importance = 1, 3351 type = "info", 3352 header = blue(" @@ "), 3353 count = (currentcounter, totalcounter,) 3354 ) 3355 3356 if checksum_ok: 3357 match.add(idpackage) 3358 else: 3359 not_match.add(idpackage) 3360 self.updateProgress( 3361 "[%s] %s: %s %s" % ( 3362 brown(crippled_uri), 3363 blue(_("package")), 3364 bold(pkgfile), 3365 red(_("NOT healthy")), 3366 ), 3367 importance = 1, 3368 type = "warning", 3369 header = darkred(" !!! "), 3370 count = (currentcounter, totalcounter,) 3371 ) 3372 if not broken_packages.has_key(crippled_uri): 3373 broken_packages[crippled_uri] = [] 3374 broken_packages[crippled_uri].append(pkgfile) 3375 3376 if broken_packages: 3377 mytxt = blue("%s:") % ( 3378 _("This is the list of broken packages"),) 3379 self.updateProgress( 3380 mytxt, 3381 importance = 1, 3382 type = "info", 3383 header = red(" * ") 3384 ) 3385 for mirror in broken_packages.keys(): 3386 mytxt = "%s: %s" % ( 3387 brown(_("Mirror")), 3388 bold(mirror), 3389 ) 3390 self.updateProgress( 3391 mytxt, 3392 importance = 1, 3393 type = "info", 3394 header = red(" <> ") 3395 ) 3396 for broken_package in broken_packages[mirror]: 3397 self.updateProgress( 3398 blue(broken_package), 3399 importance = 1, 3400 type = "info", 3401 header = red(" - ") 3402 ) 3403 3404 self.updateProgress( 3405 "%s:" % ( 3406 blue(_("Statistics")), 3407 ), 3408 importance = 1, 3409 type = "info", 3410 header = red(" @@ ") 3411 ) 3412 self.updateProgress( 3413 "[%s] %s:\t%s" % ( 3414 red(crippled_uri), 3415 brown(_("Number of checked packages")), 3416 brown(str(len(match) + len(not_match))), 3417 ), 3418 importance = 1, 3419 type = "info", 3420 header = brown(" # ") 3421 ) 3422 self.updateProgress( 3423 "[%s] %s:\t%s" % ( 3424 red(crippled_uri), 3425 darkgreen(_("Number of healthy packages")), 3426 darkgreen(str(len(match))), 3427 ), 3428 importance = 1, 3429 type = "info", 3430 header = brown(" # ") 3431 ) 3432 self.updateProgress( 3433 "[%s] %s:\t%s" % ( 3434 red(crippled_uri), 3435 darkred(_("Number of broken packages")), 3436 darkred(str(len(not_match))), 3437 ), 3438 importance = 1, 3439 type = "info", 3440 header = brown(" # ") 3441 ) 3442 3443 return match, not_match, broken_packages

3444 3445

3446 - def verify_local_packages(self, packages, ask = True, repo = None):

3447 3448 if repo == None: 3449 repo = self.default_repository 3450 3451 self.updateProgress( 3452 "[%s] %s:" % ( 3453 red(_("local")), 3454 blue(_("Integrity verification of the selected packages")), 3455 ), 3456 importance = 1, 3457 type = "info", 3458 header = darkgreen(" * ") 3459 ) 3460 3461 idpackages, world = self.match_packages(packages) 3462 dbconn = self.open_server_repository(read_only = True, 3463 no_upload = True, repo = repo) 3464 3465 if world: 3466 self.updateProgress( 3467 blue(_("All the packages in repository will be checked.")), 3468 importance = 1, 3469 type = "info", 3470 header = " " 3471 ) 3472 3473 to_download = set() 3474 available = set() 3475 for idpackage in idpackages: 3476 3477 pkgatom = dbconn.retrieveAtom(idpackage) 3478 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3479 pkg_rel_path = '/'.join(pkg_path.split("/")[2:]) 3480 3481 bindir_path = os.path.join(self.get_local_packages_directory(repo), 3482 pkg_rel_path) 3483 uploaddir_path = os.path.join(self.get_local_upload_directory(repo), 3484 pkg_rel_path) 3485 3486 if os.path.isfile(bindir_path): 3487 if not world: 3488 self.updateProgress( 3489 "[%s] %s :: %s" % ( 3490 darkgreen(_("available")), 3491 blue(pkgatom), 3492 darkgreen(pkg_rel_path), 3493 ), 3494 importance = 0, 3495 type = "info", 3496 header = darkgreen(" # ") 3497 ) 3498 available.add(idpackage) 3499 elif os.path.isfile(uploaddir_path): 3500 if not world: 3501 self.updateProgress( 3502 "[%s] %s :: %s" % ( 3503 darkred(_("upload/ignored")), 3504 blue(pkgatom), 3505 darkgreen(pkg_rel_path), 3506 ), 3507 importance = 0, 3508 type = "info", 3509 header = darkgreen(" # ") 3510 ) 3511 else: 3512 self.updateProgress( 3513 "[%s] %s :: %s" % ( 3514 brown(_("download")), 3515 blue(pkgatom), 3516 darkgreen(pkg_rel_path), 3517 ), 3518 importance = 0, 3519 type = "info", 3520 header = darkgreen(" # ") 3521 ) 3522 to_download.add((idpackage, pkg_path,)) 3523 3524 if ask: 3525 rc_question = self.askQuestion(_("Would you like to continue ?")) 3526 if rc_question == "No": 3527 return set(), set(), set(), set() 3528 3529 3530 fine = set() 3531 failed = set() 3532 downloaded_fine = set() 3533 downloaded_errors = set() 3534 3535 if to_download: 3536 3537 not_downloaded = set() 3538 mytxt = blue("%s ...") % (_("Starting to download missing files"),) 3539 self.updateProgress( 3540 mytxt, 3541 importance = 1, 3542 type = "info", 3543 header = " " 3544 ) 3545 for uri in self.get_remote_mirrors(repo): 3546 3547 if not_downloaded: 3548 mytxt = blue("%s ...") % ( 3549 _("Searching missing/broken files on another mirror"),) 3550 self.updateProgress( 3551 mytxt, 3552 importance = 1, 3553 type = "info", 3554 header = " " 3555 ) 3556 to_download = not_downloaded.copy() 3557 not_downloaded = set() 3558 3559 for idpackage, pkg_path in to_download: 3560 rc_down = self.MirrorsService.download_package(uri, 3561 pkg_path, repo = repo) 3562 if rc_down: 3563 downloaded_fine.add(idpackage) 3564 available.add(idpackage) 3565 else: 3566 not_downloaded.add(pkg_path) 3567 3568 if not not_downloaded: 3569 self.updateProgress( 3570 red(_("Binary packages downloaded successfully.")), 3571 importance = 1, 3572 type = "info", 3573 header = " " 3574 ) 3575 break 3576 3577 if not_downloaded: 3578 mytxt = blue("%s:") % ( 3579 _("These are the packages that cannot be found online"),) 3580 self.updateProgress( 3581 mytxt, 3582 importance = 1, 3583 type = "info", 3584 header = " " 3585 ) 3586 for pkg_path in not_downloaded: 3587 downloaded_errors.add(pkg_path) 3588 self.updateProgress( 3589 brown(pkg_path), 3590 importance = 1, 3591 type = "warning", 3592 header = red(" * ") 3593 ) 3594 downloaded_errors |= not_downloaded 3595 mytxt = "%s." % (_("They won't be checked"),) 3596 self.updateProgress( 3597 mytxt, 3598 importance = 1, 3599 type = "warning", 3600 header = " " 3601 ) 3602 3603 my_qa = self.QA() 3604 3605 totalcounter = str(len(available)) 3606 currentcounter = 0 3607 for idpackage in available: 3608 currentcounter += 1 3609 pkg_path = dbconn.retrieveDownloadURL(idpackage) 3610 orig_branch = self.get_branch_from_download_relative_uri(pkg_path) 3611 pkgfile = os.path.basename(pkg_path) 3612 3613 self.updateProgress( 3614 "[branch:%s] %s %s" % ( 3615 brown(orig_branch), 3616 blue(_("checking status of")), 3617 darkgreen(pkgfile), 3618 ), 3619 importance = 1, 3620 type = "info", 3621 header = " ", 3622 back = True, 3623 count = (currentcounter, totalcounter,) 3624 ) 3625 3626 storedmd5 = dbconn.retrieveDigest(idpackage) 3627 pkgpath = os.path.join(self.get_local_packages_directory(repo), 3628 orig_branch, pkgfile) 3629 result = self.entropyTools.compare_md5(pkgpath, storedmd5) 3630 qa_fine = my_qa.entropy_package_checks(pkgpath) 3631 if result and qa_fine: 3632 fine.add(idpackage) 3633 else: 3634 failed.add(idpackage) 3635 self.updateProgress( 3636 "[branch:%s] %s %s %s: %s" % ( 3637 brown(orig_branch), 3638 blue(_("package")), 3639 darkgreen(pkg_path), 3640 blue(_("is corrupted, stored checksum")), 3641 brown(storedmd5), 3642 ), 3643 importance = 1, 3644 type = "info", 3645 header = " ", 3646 count = (currentcounter, totalcounter,) 3647 ) 3648 3649 if failed: 3650 mytxt = blue("%s:") % (_("This is the list of broken packages"),) 3651 self.updateProgress( 3652 mytxt, 3653 importance = 1, 3654 type = "warning", 3655 header = darkred(" # ") 3656 ) 3657 for idpackage in failed: 3658 atom = dbconn.retrieveAtom(idpackage) 3659 down_p = dbconn.retrieveDownloadURL(idpackage) 3660 self.updateProgress( 3661 blue("[atom:%s] %s" % (atom, down_p,)), 3662 importance = 0, 3663 type = "warning", 3664 header = brown(" # ") 3665 ) 3666 3667 # print stats 3668 self.updateProgress( 3669 red("Statistics:"), 3670 importance = 1, 3671 type = "info", 3672 header = blue(" * ") 3673 ) 3674 self.updateProgress( 3675 brown("%s => %s" % ( 3676 len(fine) + len(failed), 3677 _("checked packages"), 3678 ) 3679 ), 3680 importance = 0, 3681 type = "info", 3682 header = brown(" # ") 3683 ) 3684 self.updateProgress( 3685 darkgreen("%s => %s" % ( 3686 len(fine), 3687 _("healthy packages"), 3688 ) 3689 ), 3690 importance = 0, 3691 type = "info", 3692 header = brown(" # ") 3693 ) 3694 self.updateProgress( 3695 darkred("%s => %s" % ( 3696 len(failed), 3697 _("broken packages"), 3698 ) 3699 ), 3700 importance = 0, 3701 type = "info", 3702 header = brown(" # ") 3703 ) 3704 self.updateProgress( 3705 blue("%s => %s" % ( 3706 len(downloaded_fine), 3707 _("downloaded packages"), 3708 ) 3709 ), 3710 importance = 0, 3711 type = "info", 3712 header = brown(" # ") 3713 ) 3714 self.updateProgress( 3715 bold("%s => %s" % ( 3716 len(downloaded_errors), 3717 _("failed downloads"), 3718 ) 3719 ), 3720 importance = 0, 3721 type = "info", 3722 header = brown(" # ") 3723 ) 3724 3725 self.close_server_database(dbconn) 3726 return fine, failed, downloaded_fine, downloaded_errors

3727 3728

3729 - def switch_packages_branch(self, from_branch, to_branch, repo = None):

3730 3731 if repo == None: 3732 repo = self.default_repository 3733 3734 if to_branch != self.SystemSettings['repositories']['branch']: 3735 mytxt = "%s: %s %s" % ( 3736 blue(_("Please setup your branch to")), 3737 bold(to_branch), 3738 blue(_("and retry")), 3739 ) 3740 self.updateProgress( 3741 mytxt, 3742 importance = 1, 3743 type = "error", 3744 header = darkred(" !! ") 3745 ) 3746 return None 3747 3748 mytxt = red("%s ...") % (_("Copying database (if not exists)"),) 3749 self.updateProgress( 3750 mytxt, 3751 importance = 1, 3752 type = "info", 3753 header = darkgreen(" @@ ") 3754 ) 3755 branch_dbdir = self.get_local_database_dir(repo) 3756 old_branch_dbdir = self.get_local_database_dir(repo, from_branch) 3757 3758 # close all our databases 3759 self.close_server_databases() 3760 3761 # if database file did not exist got created as an empty file 3762 # we can just rm -rf it 3763 branch_dbfile = self.get_local_database_file(repo) 3764 if os.path.isfile(branch_dbfile): 3765 if self.entropyTools.get_file_size(branch_dbfile) == 0: 3766 shutil.rmtree(branch_dbdir, True) 3767 3768 if os.path.isdir(branch_dbdir): 3769 3770 while 1: 3771 rnd_num = self.entropyTools.get_random_number() 3772 backup_dbdir = branch_dbdir + str(rnd_num) 3773 if not os.path.isdir(backup_dbdir): 3774 break 3775 os.rename(branch_dbdir, backup_dbdir) 3776 3777 if os.path.isdir(old_branch_dbdir): 3778 shutil.copytree(old_branch_dbdir, branch_dbdir) 3779 3780 mytxt = red("%s ...") % (_("Switching packages"),) 3781 self.updateProgress( 3782 mytxt, 3783 importance = 1, 3784 type = "info", 3785 header = darkgreen(" @@ ") 3786 ) 3787 3788 dbconn = self.open_server_repository(read_only = False, 3789 no_upload = True, repo = repo, lock_remote = False) 3790 try: 3791 dbconn.validateDatabase() 3792 except SystemDatabaseError: 3793 self.handle_uninitialized_repository(repo) 3794 dbconn = self.open_server_repository(read_only = False, 3795 no_upload = True, repo = repo, lock_remote = False) 3796 3797 idpackages = dbconn.listAllIdpackages() 3798 already_switched = set() 3799 not_found = set() 3800 switched = set() 3801 ignored = set() 3802 no_checksum = set() 3803 3804 maxcount = len(idpackages) 3805 count = 0 3806 for idpackage in idpackages: 3807 count += 1 3808 3809 cur_branch = dbconn.retrieveBranch(idpackage) 3810 atom = dbconn.retrieveAtom(idpackage) 3811 if cur_branch == to_branch: 3812 already_switched.add(idpackage) 3813 self.updateProgress( 3814 red("%s %s, %s %s" % ( 3815 _("Ignoring"), 3816 bold(atom), 3817 _("already in branch"), 3818 cur_branch, 3819 ) 3820 ), 3821 importance = 0, 3822 type = "info", 3823 header = darkgreen(" @@ "), 3824 count = (count, maxcount,) 3825 ) 3826 ignored.add(idpackage) 3827 continue 3828 3829 self.updateProgress( 3830 "[%s=>%s] %s" % ( 3831 brown(cur_branch), 3832 bold(to_branch), 3833 darkgreen(atom), 3834 ), 3835 importance = 0, 3836 type = "info", 3837 header = darkgreen(" @@ "), 3838 back = True, 3839 count = (count, maxcount,) 3840 ) 3841 dbconn.switchBranch(idpackage, to_branch) 3842 dbconn.commitChanges() 3843 switched.add(idpackage) 3844 3845 dbconn.commitChanges() 3846 3847 # now migrate counters 3848 dbconn.moveCountersToBranch(to_branch, from_branch = from_branch) 3849 3850 self.close_server_database(dbconn) 3851 mytxt = blue("%s.") % (_("migration loop completed"),) 3852 self.updateProgress( 3853 "[%s=>%s] %s" % ( 3854 brown(from_branch), 3855 bold(to_branch), 3856 mytxt, 3857 ), 3858 importance = 1, 3859 type = "info", 3860 header = darkgreen(" * ") 3861 ) 3862 3863 return switched, already_switched, ignored, not_found, no_checksum

3864

3865 - def get_entropy_sets(self, repo = None, branch = None):

3866 3867 if branch == None: 3868 branch = self.SystemSettings['repositories']['branch'] 3869 if repo == None: 3870 repo = self.default_repository 3871 3872 sets_dir = self.get_local_database_sets_dir(repo, branch) 3873 if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)): 3874 return {} 3875 3876 mydata = {} 3877 items = os.listdir(sets_dir) 3878 for item in items: 3879 3880 try: 3881 item_clean = str(item) 3882 except (UnicodeEncodeError, UnicodeDecodeError,): 3883 continue 3884 item_path = os.path.join(sets_dir, item) 3885 if not (os.path.isfile(item_path) and \ 3886 os.access(item_path, os.R_OK)): 3887 continue 3888 item_elements = self.entropyTools.extract_packages_from_set_file( 3889 item_path) 3890 if item_elements: 3891 mydata[item_clean] = item_elements.copy() 3892 3893 return mydata

3894

3895 - def get_configured_package_sets(self, repo = None, branch = None, 3896 validate = True):

3897 3898 if branch == None: 3899 branch = self.SystemSettings['repositories']['branch'] 3900 if repo == None: 3901 repo = self.default_repository 3902 3903 # portage sets 3904 sets_data = self.SpmService.get_sets_expanded(builtin_sets = False) 3905 sets_data.update(self.get_entropy_sets(repo, branch)) 3906 3907 if validate: 3908 invalid_sets = set() 3909 # validate 3910 for setname in sets_data: 3911 good = True 3912 for atom in sets_data[setname]: 3913 dbconn = self.open_server_repository(just_reading = True, 3914 repo = repo) 3915 match = dbconn.atomMatch(atom) 3916 if match[0] == -1: 3917 good = False 3918 break 3919 if not good: 3920 invalid_sets.add(setname) 3921 for invalid_set in invalid_sets: 3922 del sets_data[invalid_set] 3923 3924 return sets_data

3925

3926 - def update_database_package_sets(self, repo = None, dbconn = None):

3927 3928 if repo == None: 3929 repo = self.default_repository 3930 package_sets = self.get_configured_package_sets(repo) 3931 if dbconn == None: 3932 dbconn = self.open_server_repository( 3933 read_only = False, no_upload = True, repo = repo, 3934 do_treeupdates = False) 3935 dbconn.clearPackageSets() 3936 if package_sets: 3937 dbconn.insertPackageSets(package_sets) 3938 dbconn.commitChanges()

3939

Source Code for Module entropy.server.interfaces.main