entropy.server.interfaces.mirrors

28 29 import socket 30 import entropy.dump as dumpTools 31 import entropy.tools as entropyTools

32 - def __init__(self, ServerInstance, repo = None):

33 34 from entropy.cache import EntropyCacher 35 from entropy.server.transceivers import TransceiverServerHandler 36 from entropy.server.interfaces.main import Server as MainServer 37 38 if not isinstance(ServerInstance, MainServer): 39 mytxt = _("entropy.server.interfaces.main.Server needed") 40 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 41 42 self.Entropy = ServerInstance 43 self.TransceiverServerHandler = TransceiverServerHandler 44 self.Cacher = EntropyCacher() 45 self.sys_settings_plugin_id = \ 46 etpConst['system_settings_plugins_ids']['server_plugin'] 47 self.SystemSettings = self.Entropy.SystemSettings 48 49 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 50 self.Entropy.updateProgress( 51 mytxt, 52 importance = 2, 53 type = "info", 54 header = red(" @@ ") 55 ) 56 mytxt = _("mirror") 57 for mirror in self.Entropy.get_remote_mirrors(repo): 58 mirror = EntropyTransceiver.hide_sensible_data(mirror) 59 self.Entropy.updateProgress( 60 blue("%s: %s") % (mytxt, darkgreen(mirror),), 61 importance = 0, 62 type = "info", 63 header = brown(" # ") 64 )

65

66 - def get_remote_branches(self, repo = None):

67 """ 68 Returns a list of remotely available branches for the provided 69 repository identifier. 70 71 @keyword repo: repository identifier 72 @type repo: string 73 @return: list of valid, available remote branches 74 @rtype: set 75 """ 76 77 if repo is None: 78 repo = self.Entropy.default_repository 79 80 remote_branches = set() 81 # this is used to validate Entropy repository path 82 ts_file = etpConst['etpdatabasetimestampfile'] 83 mirrors = self.Entropy.get_remote_mirrors(repo) 84 for uri in mirrors: 85 86 crippled_uri = EntropyTransceiver.get_uri_name(uri) 87 88 self.Entropy.updateProgress( 89 "[repo:%s] %s: %s" % ( 90 brown(repo), 91 blue(_("listing branches in mirror")), 92 darkgreen(crippled_uri), 93 ), 94 importance = 1, 95 type = "info", 96 header = brown(" @@ ") 97 ) 98 99 txc = self.Entropy.Transceiver(uri) 100 txc.set_verbosity(False) 101 with txc as handler: 102 branches_path = self.Entropy.get_remote_database_relative_path( 103 repo) 104 105 try: 106 branches = handler.list_content(branches_path) 107 except ValueError: 108 branches = [] # dir is empty 109 110 for branch in branches: 111 mypath = os.path.join("/", branches_path, branch) 112 if handler.is_dir(mypath): 113 remote_branches.add(branch) 114 115 return remote_branches

116

117 - def read_remote_file_in_branches(self, filename, repo = None, 118 excluded_branches = None):

119 """ 120 Reads a file remotely located in all the available branches. 121 122 @param filename: name of the file that should be located inside 123 repository database directory 124 @type filename: string 125 @keyword repo: repository identifier 126 @type repo: string 127 @keyword excluded_branches: list of branch identifiers excluded or None 128 @type excluded_branches: list or None 129 @return: dictionary with branches as key and raw file content as value: 130 {'4': 'abcd\n', '5': 'defg\n'} 131 @rtype: dict 132 """ 133 if repo is None: 134 repo = self.Entropy.default_repository 135 if excluded_branches is None: 136 excluded_branches = [] 137 138 branch_data = {} 139 mirrors = self.Entropy.get_remote_mirrors(repo) 140 for uri in mirrors: 141 142 crippled_uri = EntropyTransceiver.get_uri_name(uri) 143 144 self.Entropy.updateProgress( 145 "[repo:%s] %s: %s => %s" % ( 146 brown(repo), 147 blue(_("looking for file in mirror")), 148 darkgreen(crippled_uri), 149 filename, 150 ), 151 importance = 1, 152 type = "info", 153 header = brown(" @@ ") 154 ) 155 156 branches_path = self.Entropy.get_remote_database_relative_path(repo) 157 txc = self.Entropy.Transceiver(uri) 158 txc.set_verbosity(False) 159 160 with txc as handler: 161 162 branches = handler.list_content(branches_path) 163 for branch in branches: 164 165 # is branch excluded ? 166 if branch in excluded_branches: 167 continue 168 169 if branch_data.get(branch) != None: 170 # already read 171 continue 172 173 mypath = os.path.join("/", branches_path, branch, filename) 174 if not handler.is_file(mypath): 175 # nothing to do, not a file 176 continue 177 178 tmp_dir = tempfile.mkdtemp() 179 down_path = os.path.join(tmp_dir, 180 os.path.basename(filename)) 181 tries = 4 182 success = False 183 while tries: 184 downloaded = handler.download(mypath, down_path) 185 if not downloaded: 186 tries -= 1 187 continue # argh! 188 success = True 189 break 190 191 if success and os.path.isfile(down_path): 192 down_f = open(down_path) 193 branch_data[branch] = down_f.read() 194 down_f.close() 195 196 shutil.rmtree(tmp_dir, True) 197 198 return branch_data

199

200 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

201 202 if repo is None: 203 repo = self.Entropy.default_repository 204 205 if not mirrors: 206 mirrors = self.Entropy.get_remote_mirrors(repo) 207 208 issues = False 209 for uri in mirrors: 210 211 crippled_uri = EntropyTransceiver.get_uri_name(uri) 212 213 lock_text = _("unlocking") 214 if lock: 215 lock_text = _("locking") 216 self.Entropy.updateProgress( 217 "[repo:%s|%s] %s %s" % ( 218 brown(repo), 219 darkgreen(crippled_uri), 220 bold(lock_text), 221 blue("%s...") % (_("mirror"),), 222 ), 223 importance = 1, 224 type = "info", 225 header = brown(" * "), 226 back = True 227 ) 228 229 base_path = os.path.join( 230 self.Entropy.get_remote_database_relative_path(repo), 231 self.SystemSettings['repositories']['branch']) 232 lock_file = os.path.join(base_path, 233 etpConst['etpdatabaselockfile']) 234 235 txc = self.Entropy.Transceiver(uri) 236 txc.set_verbosity(False) 237 238 with txc as handler: 239 240 if lock and handler.is_file(lock_file): 241 self.Entropy.updateProgress( 242 "[repo:%s|%s] %s" % ( 243 brown(repo), 244 darkgreen(crippled_uri), 245 blue(_("mirror already locked")), 246 ), 247 importance = 1, 248 type = "info", 249 header = darkgreen(" * ") 250 ) 251 continue 252 253 elif not lock and not handler.is_file(lock_file): 254 self.Entropy.updateProgress( 255 "[repo:%s|%s] %s" % ( 256 brown(repo), 257 darkgreen(crippled_uri), 258 blue(_("mirror already unlocked")), 259 ), 260 importance = 1, 261 type = "info", 262 header = darkgreen(" * ") 263 ) 264 continue 265 266 if lock: 267 rc_lock = self._do_mirror_lock(uri, handler, repo = repo) 268 else: 269 rc_lock = self._do_mirror_unlock(uri, handler, repo = repo) 270 271 if not rc_lock: 272 issues = True 273 274 if not issues: 275 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 276 if os.path.isfile(db_taint_file): 277 os.remove(db_taint_file) 278 279 return issues

280 281

282 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 283 repo = None):

284 """ 285 This functions makes entropy clients to not download anything 286 from the chosen mirrors. it is used to avoid clients to 287 download databases while we're uploading a new one 288 """ 289 290 if mirrors is None: 291 mirrors = [] 292 293 if repo is None: 294 repo = self.Entropy.default_repository 295 296 if not mirrors: 297 mirrors = self.Entropy.get_remote_mirrors(repo) 298 299 issues = False 300 for uri in mirrors: 301 302 crippled_uri = EntropyTransceiver.get_uri_name(uri) 303 304 lock_text = _("unlocking") 305 if lock: 306 lock_text = _("locking") 307 self.Entropy.updateProgress( 308 "[repo:%s|%s] %s %s..." % ( 309 blue(repo), 310 red(crippled_uri), 311 bold(lock_text), 312 blue(_("mirror for download")), 313 ), 314 importance = 1, 315 type = "info", 316 header = red(" @@ "), 317 back = True 318 ) 319 320 lock_file = etpConst['etpdatabasedownloadlockfile'] 321 my_path = os.path.join( 322 self.Entropy.get_remote_database_relative_path(repo), 323 self.SystemSettings['repositories']['branch']) 324 lock_file = os.path.join(my_path, lock_file) 325 326 txc = self.Entropy.Transceiver(uri) 327 txc.set_verbosity(False) 328 329 with txc as handler: 330 331 if lock and handler.is_file(lock_file): 332 self.Entropy.updateProgress( 333 "[repo:%s|%s] %s" % ( 334 blue(repo), 335 red(crippled_uri), 336 blue(_("mirror already locked for download")), 337 ), 338 importance = 1, 339 type = "info", 340 header = red(" @@ ") 341 ) 342 continue 343 344 elif not lock and not handler.is_file(lock_file): 345 self.Entropy.updateProgress( 346 "[repo:%s|%s] %s" % ( 347 blue(repo), 348 red(crippled_uri), 349 blue(_("mirror already unlocked for download")), 350 ), 351 importance = 1, 352 type = "info", 353 header = red(" @@ ") 354 ) 355 continue 356 357 if lock: 358 rc_lock = self._do_mirror_lock(uri, handler, dblock = False, 359 repo = repo) 360 else: 361 rc_lock = self._do_mirror_unlock(uri, handler, 362 dblock = False, repo = repo) 363 if not rc_lock: 364 issues = True 365 366 return issues

367

368 - def _do_mirror_lock(self, uri, txc_handler, dblock = True, repo = None):

369 370 if repo is None: 371 repo = self.Entropy.default_repository 372 373 my_path = os.path.join( 374 self.Entropy.get_remote_database_relative_path(repo), 375 self.SystemSettings['repositories']['branch']) 376 377 # create path to lock file if it doesn't exist 378 if not txc_handler.is_dir(my_path): 379 txc_handler.makedirs(my_path) 380 381 crippled_uri = EntropyTransceiver.get_uri_name(uri) 382 lock_string = '' 383 384 if dblock: 385 self.create_local_database_lockfile(repo) 386 lock_file = self.get_database_lockfile(repo) 387 else: 388 # locking/unlocking mirror1 for download 389 lock_string = _('for download') 390 self.create_local_database_download_lockfile(repo) 391 lock_file = self.get_database_download_lockfile(repo) 392 393 remote_path = os.path.join(my_path, os.path.basename(lock_file)) 394 395 rc_upload = txc_handler.upload(lock_file, remote_path) 396 if rc_upload: 397 self.Entropy.updateProgress( 398 "[repo:%s|%s] %s %s" % ( 399 blue(repo), 400 red(crippled_uri), 401 blue(_("mirror successfully locked")), 402 blue(lock_string), 403 ), 404 importance = 1, 405 type = "info", 406 header = red(" @@ ") 407 ) 408 else: 409 self.Entropy.updateProgress( 410 "[repo:%s|%s] %s: %s - %s %s" % ( 411 blue(repo), 412 red(crippled_uri), 413 blue("lock error"), 414 rc_upload, 415 blue(_("mirror not locked")), 416 blue(lock_string), 417 ), 418 importance = 1, 419 type = "error", 420 header = darkred(" * ") 421 ) 422 self.remove_local_database_lockfile(repo) 423 424 return rc_upload

425 426

427 - def _do_mirror_unlock(self, uri, txc_handler, dblock = True, repo = None):

428 429 if repo is None: 430 repo = self.Entropy.default_repository 431 432 my_path = os.path.join( 433 self.Entropy.get_remote_database_relative_path(repo), 434 self.SystemSettings['repositories']['branch']) 435 436 crippled_uri = EntropyTransceiver.get_uri_name(uri) 437 438 if dblock: 439 dbfile = etpConst['etpdatabaselockfile'] 440 else: 441 dbfile = etpConst['etpdatabasedownloadlockfile'] 442 443 # make sure 444 remote_path = os.path.join(my_path, os.path.basename(dbfile)) 445 446 rc_delete = txc_handler.delete(remote_path) 447 if rc_delete: 448 self.Entropy.updateProgress( 449 "[repo:%s|%s] %s" % ( 450 blue(repo), 451 red(crippled_uri), 452 blue(_("mirror successfully unlocked")), 453 ), 454 importance = 1, 455 type = "info", 456 header = darkgreen(" * ") 457 ) 458 if dblock: 459 self.remove_local_database_lockfile(repo) 460 else: 461 self.remove_local_database_download_lockfile(repo) 462 else: 463 self.Entropy.updateProgress( 464 "[repo:%s|%s] %s: %s - %s" % ( 465 blue(repo), 466 red(crippled_uri), 467 blue(_("unlock error")), 468 rc_delete, 469 blue(_("mirror not unlocked")), 470 ), 471 importance = 1, 472 type = "error", 473 header = darkred(" * ") 474 ) 475 476 return rc_delete

477

478 - def get_database_lockfile(self, repo = None):

479 if repo is None: 480 repo = self.Entropy.default_repository 481 return os.path.join(self.Entropy.get_local_database_dir(repo), 482 etpConst['etpdatabaselockfile'])

483

484 - def get_database_download_lockfile(self, repo = None):

485 if repo is None: 486 repo = self.Entropy.default_repository 487 return os.path.join(self.Entropy.get_local_database_dir(repo), 488 etpConst['etpdatabasedownloadlockfile'])

489

490 - def create_local_database_download_lockfile(self, repo = None):

491 if repo is None: 492 repo = self.Entropy.default_repository 493 lock_file = self.get_database_download_lockfile(repo) 494 f_lock = open(lock_file, "w") 495 f_lock.write("download locked") 496 f_lock.flush() 497 f_lock.close()

498

499 - def create_local_database_lockfile(self, repo = None):

500 if repo is None: 501 repo = self.Entropy.default_repository 502 lock_file = self.get_database_lockfile(repo) 503 f_lock = open(lock_file, "w") 504 f_lock.write("database locked") 505 f_lock.flush() 506 f_lock.close()

507

508 - def remove_local_database_lockfile(self, repo = None):

509 if repo is None: 510 repo = self.Entropy.default_repository 511 lock_file = self.get_database_lockfile(repo) 512 if os.path.isfile(lock_file): 513 os.remove(lock_file)

514

515 - def remove_local_database_download_lockfile(self, repo = None):

516 if repo is None: 517 repo = self.Entropy.default_repository 518 lock_file = self.get_database_download_lockfile(repo) 519 if os.path.isfile(lock_file): 520 os.remove(lock_file)

521

522 - def download_package(self, uri, pkg_relative_path, repo = None):

523 524 if repo is None: 525 repo = self.Entropy.default_repository 526 527 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 528 pkgfile = os.path.basename(pkg_relative_path) 529 crippled_uri = EntropyTransceiver.get_uri_name(uri) 530 531 tries = 0 532 while tries < 5: 533 534 tries += 1 535 txc = self.Entropy.Transceiver(uri) 536 with txc as handler: 537 538 self.Entropy.updateProgress( 539 "[repo:%s|%s|#%s] %s: %s" % ( 540 brown(repo), 541 darkgreen(crippled_uri), 542 brown(str(tries)), 543 blue(_("connecting to download package")), 544 darkgreen(pkg_to_join_path), 545 ), 546 importance = 1, 547 type = "info", 548 header = darkgreen(" * "), 549 back = True 550 ) 551 552 remote_path = os.path.join( 553 self.Entropy.get_remote_packages_relative_path(repo), 554 pkg_to_join_path) 555 download_path = os.path.join( 556 self.Entropy.get_local_packages_directory(repo), 557 pkg_to_join_path) 558 download_dir = os.path.dirname(download_path) 559 560 self.Entropy.updateProgress( 561 "[repo:%s|%s|#%s] %s: %s" % ( 562 brown(repo), 563 darkgreen(crippled_uri), 564 brown(str(tries)), 565 blue(_("downloading package")), 566 darkgreen(remote_path), 567 ), 568 importance = 1, 569 type = "info", 570 header = darkgreen(" * ") 571 ) 572 573 if (not os.path.isdir(download_dir)) and \ 574 (not os.access(download_dir, os.R_OK)): 575 os.makedirs(download_dir) 576 577 rc_download = handler.download(remote_path, download_path) 578 if not rc_download: 579 self.Entropy.updateProgress( 580 "[repo:%s|%s|#%s] %s: %s %s" % ( 581 brown(repo), 582 darkgreen(crippled_uri), 583 brown(str(tries)), 584 blue(_("package")), 585 darkgreen(pkg_to_join_path), 586 blue(_("does not exist")), 587 ), 588 importance = 1, 589 type = "error", 590 header = darkred(" !!! ") 591 ) 592 return rc_download 593 594 dbconn = self.Entropy.open_server_repository(read_only = True, 595 no_upload = True, repo = repo) 596 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 597 if idpackage == -1: 598 self.Entropy.updateProgress( 599 "[repo:%s|%s|#%s] %s: %s %s" % ( 600 brown(repo), 601 darkgreen(crippled_uri), 602 brown(str(tries)), 603 blue(_("package")), 604 darkgreen(pkg_relative_path), 605 blue(_("is not listed in the repository !")), 606 ), 607 importance = 1, 608 type = "error", 609 header = darkred(" !!! ") 610 ) 611 return False 612 613 storedmd5 = dbconn.retrieveDigest(idpackage) 614 self.Entropy.updateProgress( 615 "[repo:%s|%s|#%s] %s: %s" % ( 616 brown(repo), 617 darkgreen(crippled_uri), 618 brown(str(tries)), 619 blue(_("verifying checksum of package")), 620 darkgreen(pkg_relative_path), 621 ), 622 importance = 1, 623 type = "info", 624 header = darkgreen(" * "), 625 back = True 626 ) 627 628 md5check = self.entropyTools.compare_md5(download_path, storedmd5) 629 if md5check: 630 self.Entropy.updateProgress( 631 "[repo:%s|%s|#%s] %s: %s %s" % ( 632 brown(repo), 633 darkgreen(crippled_uri), 634 brown(str(tries)), 635 blue(_("package")), 636 darkgreen(pkg_relative_path), 637 blue(_("downloaded successfully")), 638 ), 639 importance = 1, 640 type = "info", 641 header = darkgreen(" * ") 642 ) 643 return True 644 else: 645 self.Entropy.updateProgress( 646 "[repo:%s|%s|#%s] %s: %s %s" % ( 647 brown(repo), 648 darkgreen(crippled_uri), 649 brown(str(tries)), 650 blue(_("package")), 651 darkgreen(pkg_relative_path), 652 blue(_("checksum does not match. re-downloading...")), 653 ), 654 importance = 1, 655 type = "warning", 656 header = darkred(" * ") 657 ) 658 if os.path.isfile(download_path): 659 os.remove(download_path) 660 661 # if we get here it means the files hasn't been downloaded properly 662 self.Entropy.updateProgress( 663 "[repo:%s|%s|#%s] %s: %s %s" % ( 664 brown(repo), 665 darkgreen(crippled_uri), 666 brown(str(tries)), 667 blue(_("package")), 668 darkgreen(pkg_relative_path), 669 blue(_("seems broken. Consider to re-package it. Giving up!")), 670 ), 671 importance = 1, 672 type = "error", 673 header = darkred(" !!! ") 674 ) 675 return False

676

677 - def _get_remote_db_status(self, uri, repo):

678 679 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 680 db_format = sys_set['database_file_format'] 681 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 682 if cmethod is None: 683 raise InvalidDataType("InvalidDataType: %s." % ( 684 _("Wrong database compression method passed"), 685 ) 686 ) 687 remote_dir = os.path.join( 688 self.Entropy.get_remote_database_relative_path(repo), 689 self.SystemSettings['repositories']['branch']) 690 691 # let raise exception if connection is impossible 692 txc = self.Entropy.Transceiver(uri) 693 with txc as handler: 694 695 compressedfile = etpConst[cmethod[2]] 696 rc1 = handler.is_file(os.path.join(remote_dir, compressedfile)) 697 698 rev_file = self.Entropy.get_local_database_revision_file(repo) 699 revfilename = os.path.basename(rev_file) 700 rc2 = handler.is_file(os.path.join(remote_dir, revfilename)) 701 702 revision = 0 703 if not (rc1 and rc2): 704 return [uri, revision] 705 706 tmp_fd, rev_tmp_path = tempfile.mkstemp() 707 708 dlcount = 5 709 dled = False 710 while dlcount: 711 remote_rev_path = os.path.join(remote_dir, revfilename) 712 dled = handler.download(remote_rev_path, rev_tmp_path) 713 if dled: 714 break 715 dlcount -= 1 716 os.close(tmp_fd) 717 718 crippled_uri = EntropyTransceiver.get_uri_name(uri) 719 720 if os.access(rev_tmp_path, os.R_OK) and \ 721 os.path.isfile(rev_tmp_path): 722 723 f_rev = open(rev_tmp_path, "r") 724 try: 725 revision = int(f_rev.readline().strip()) 726 except ValueError: 727 mytxt = _("mirror hasn't valid database revision file") 728 self.Entropy.updateProgress( 729 "[repo:%s|%s] %s: %s" % ( 730 brown(repo), 731 darkgreen(crippled_uri), 732 blue(mytxt), 733 bold(revision), 734 ), 735 importance = 1, 736 type = "error", 737 header = darkred(" !!! ") 738 ) 739 revision = 0 740 f_rev.close() 741 742 elif dlcount == 0: 743 self.Entropy.updateProgress( 744 "[repo:%s|%s] %s: %s" % ( 745 brown(repo), 746 darkgreen(crippled_uri), 747 blue(_("unable to download repository revision")), 748 bold(revision), 749 ), 750 importance = 1, 751 type = "error", 752 header = darkred(" !!! ") 753 ) 754 revision = 0 755 756 else: 757 self.Entropy.updateProgress( 758 "[repo:%s|%s] %s: %s" % ( 759 brown(repo), 760 darkgreen(crippled_uri), 761 blue(_("mirror doesn't have valid revision file")), 762 bold(revision), 763 ), 764 importance = 1, 765 type = "error", 766 header = darkred(" !!! ") 767 ) 768 revision = 0 769 770 os.remove(rev_tmp_path) 771 return [uri, revision]

772

773 - def get_remote_databases_status(self, repo = None, mirrors = None):

774 775 if repo is None: 776 repo = self.Entropy.default_repository 777 if not mirrors: 778 mirrors = self.Entropy.get_remote_mirrors(repo) 779 780 data = [] 781 for uri in mirrors: 782 data.append(self._get_remote_db_status(uri, repo)) 783 784 return data

785

786 - def is_local_database_locked(self, repo = None):

787 local_repo = repo 788 if local_repo is None: 789 local_repo = self.Entropy.default_repository 790 lock_file = self.get_database_lockfile(local_repo) 791 return os.path.isfile(lock_file)

792

793 - def get_mirrors_lock(self, repo = None):

794 795 dbstatus = [] 796 remote_dir = os.path.join( 797 self.Entropy.get_remote_database_relative_path(repo), 798 self.SystemSettings['repositories']['branch']) 799 lock_file = os.path.join(remote_dir, etpConst['etpdatabaselockfile']) 800 down_lock_file = os.path.join(remote_dir, 801 etpConst['etpdatabasedownloadlockfile']) 802 803 for uri in self.Entropy.get_remote_mirrors(repo): 804 data = [uri, False, False] 805 806 # let raise exception if connection is impossible 807 txc = self.Entropy.Transceiver(uri) 808 with txc as handler: 809 if handler.is_file(lock_file): 810 # upload locked 811 data[1] = True 812 if handler.is_file(down_lock_file): 813 # download locked 814 data[2] = True 815 dbstatus.append(data) 816 817 return dbstatus

818

819 - def download_notice_board(self, repo = None):

820 821 if repo is None: 822 repo = self.Entropy.default_repository 823 mirrors = self.Entropy.get_remote_mirrors(repo) 824 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 825 mytmpdir = self.entropyTools.get_random_temp_file() 826 os.makedirs(mytmpdir) 827 828 self.Entropy.updateProgress( 829 "[repo:%s] %s %s" % ( 830 brown(repo), 831 blue(_("downloading notice board from mirrors to")), 832 red(rss_path), 833 ), 834 importance = 1, 835 type = "info", 836 header = blue(" @@ ") 837 ) 838 839 downloaded = False 840 for uri in mirrors: 841 crippled_uri = EntropyTransceiver.get_uri_name(uri) 842 downloader = self.TransceiverServerHandler( 843 self.Entropy, [uri], 844 [rss_path], download = True, 845 local_basedir = mytmpdir, critical_files = [rss_path], 846 repo = repo 847 ) 848 errors, m_fine_uris, m_broken_uris = downloader.go() 849 if not errors: 850 self.Entropy.updateProgress( 851 "[repo:%s] %s: %s" % ( 852 brown(repo), 853 blue(_("notice board downloaded successfully from")), 854 red(crippled_uri), 855 ), 856 importance = 1, 857 type = "info", 858 header = blue(" @@ ") 859 ) 860 downloaded = True 861 break 862 863 if downloaded: 864 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 865 rss_path) 866 867 return downloaded

868

869 - def upload_notice_board(self, repo = None):

870 871 if repo is None: 872 repo = self.Entropy.default_repository 873 mirrors = self.Entropy.get_remote_mirrors(repo) 874 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 875 876 self.Entropy.updateProgress( 877 "[repo:%s] %s %s" % ( 878 brown(repo), 879 blue(_("uploading notice board from")), 880 red(rss_path), 881 ), 882 importance = 1, 883 type = "info", 884 header = blue(" @@ ") 885 ) 886 887 uploader = self.TransceiverServerHandler( 888 self.Entropy, 889 mirrors, 890 [rss_path], 891 critical_files = [rss_path], 892 repo = repo 893 ) 894 errors, m_fine_uris, m_broken_uris = uploader.go() 895 if errors: 896 m_broken_uris = sorted(m_broken_uris) 897 m_broken_uris = [EntropyTransceiver.get_uri_name(x) \ 898 for x in m_broken_uris] 899 self.Entropy.updateProgress( 900 "[repo:%s] %s %s" % ( 901 brown(repo), 902 blue(_("notice board upload failed on")), 903 red(', '.join(m_broken_uris)), 904 ), 905 importance = 1, 906 type = "info", 907 header = blue(" @@ ") 908 ) 909 return False 910 self.Entropy.updateProgress( 911 "[repo:%s] %s" % ( 912 brown(repo), 913 blue(_("notice board upload success")), 914 ), 915 importance = 1, 916 type = "info", 917 header = blue(" @@ ") 918 ) 919 return True

920 921

922 - def update_notice_board(self, title, notice_text, link = None, repo = None):

923 924 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 925 rss_description = "Inform about important distribution activities." 926 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 927 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 928 if not link: 929 link = srv_set['rss']['website_url'] 930 931 self.download_notice_board(repo) 932 rss_main = RSS(rss_path, rss_title, rss_description, 933 maxentries = 20) 934 rss_main.add_item(title, link, description = notice_text) 935 rss_main.write_changes() 936 status = self.upload_notice_board(repo) 937 return status

938

939 - def read_notice_board(self, do_download = True, repo = None):

940 941 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 942 if do_download: 943 self.download_notice_board(repo) 944 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 945 return None 946 rss_main = RSS(rss_path, '', '') 947 return rss_main.get_entries()

948

949 - def remove_from_notice_board(self, identifier, repo = None):

950 951 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 952 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 953 rss_description = "Inform about important distribution activities." 954 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 955 return 0 956 rss_main = RSS(rss_path, rss_title, rss_description) 957 data = rss_main.remove_entry(identifier) 958 rss_main.write_changes() 959 return data

960

961 - def update_rss_feed(self, repo = None):

962 963 if repo is None: 964 repo = self.Entropy.default_repository 965 966 product = self.SystemSettings['repositories']['product'] 967 #db_dir = self.Entropy.get_local_database_dir(repo) 968 rss_path = self.Entropy.get_local_database_rss_file(repo) 969 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 970 rss_dump_name = repo + etpConst['rss-dump-name'] 971 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 972 973 rss_title = "%s Online Repository Status" % ( 974 self.SystemSettings['system']['name'],) 975 rss_description = \ 976 "Keep you updated on what's going on in the %s Repository." % ( 977 self.SystemSettings['system']['name'],) 978 979 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 980 981 rss_main = RSS(rss_path, rss_title, rss_description, 982 maxentries = srv_set['rss']['max_entries']) 983 # load dump 984 db_actions = self.Cacher.pop(rss_dump_name) 985 if db_actions: 986 try: 987 f_rev = open(db_revision_path) 988 revision = f_rev.readline().strip() 989 f_rev.close() 990 except (IOError, OSError): 991 revision = "N/A" 992 commitmessage = '' 993 if self.Entropy.rssMessages['commitmessage']: 994 commitmessage = ' :: ' + \ 995 self.Entropy.rssMessages['commitmessage'] 996 997 title = ": " + self.SystemSettings['system']['name'] + " " + \ 998 product[0].upper() + product[1:] + " " + \ 999 self.SystemSettings['repositories']['branch'] + \ 1000 " :: Revision: " + revision + commitmessage 1001 1002 link = srv_set['rss']['base_url'] 1003 # create description 1004 added_items = db_actions.get("added") 1005 1006 if added_items: 1007 for atom in sorted(added_items): 1008 mylink = link + "?search=" + atom.split("~")[0] + \ 1009 "&arch=" + etpConst['currentarch'] + "&product="+product 1010 description = atom + ": " + added_items[atom]['description'] 1011 rss_main.add_item(title = "Added/Updated" + title, 1012 link = mylink, description = description) 1013 removed_items = db_actions.get("removed") 1014 1015 if removed_items: 1016 for atom in sorted(removed_items): 1017 description = atom + ": " + \ 1018 removed_items[atom]['description'] 1019 rss_main.add_item(title = "Removed" + title, link = link, 1020 description = description) 1021 1022 light_items = db_actions.get('light') 1023 if light_items: 1024 rss_light = RSS(rss_light_path, rss_title, rss_description, 1025 maxentries = srv_set['rss']['light_max_entries']) 1026 for atom in sorted(light_items): 1027 mylink = link + "?search=" + atom.split("~")[0] + \ 1028 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1029 product 1030 description = light_items[atom]['description'] 1031 rss_light.add_item(title = "[" + revision + "] " + atom, 1032 link = mylink, description = description) 1033 rss_light.write_changes() 1034 1035 rss_main.write_changes() 1036 self.Entropy.rssMessages.clear() 1037 self.dumpTools.removeobj(rss_dump_name)

1038 1039

1040 - def dump_database_to_file(self, db_path, destination_path, opener, 1041 exclude_tables = None, repo = None):

1042 1043 if not exclude_tables: 1044 exclude_tables = [] 1045 1046 f_out = opener(destination_path, "wb") 1047 dbconn = self.Entropy.open_server_repository(db_path, 1048 just_reading = True, repo = repo, do_treeupdates = False) 1049 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1050 self.Entropy.close_server_database(dbconn) 1051 f_out.close()

1052

1053 - def create_file_checksum(self, file_path, checksum_path):

1054 mydigest = self.entropyTools.md5sum(file_path) 1055 f_ck = open(checksum_path, "w") 1056 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1057 f_ck.write(mystring) 1058 f_ck.flush() 1059 f_ck.close()

1060

1061 - def compress_file(self, file_path, destination_path, opener):

1062 f_out = opener(destination_path, "wb") 1063 f_in = open(file_path, "rb") 1064 data = f_in.read(8192) 1065 while data: 1066 f_out.write(data) 1067 data = f_in.read(8192) 1068 f_in.close() 1069 if hasattr(f_out, 'flush'): 1070 f_out.flush() 1071 f_out.close()

1072

1073 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1074 disabled_eapis = None):

1075 1076 if disabled_eapis is None: 1077 disabled_eapis = [] 1078 1079 critical = [] 1080 extra_text_files = [] 1081 data = {} 1082 data['database_revision_file'] = \ 1083 self.Entropy.get_local_database_revision_file(repo) 1084 extra_text_files.append(data['database_revision_file']) 1085 critical.append(data['database_revision_file']) 1086 1087 1088 # branch migration support scripts 1089 post_branch_mig_file = self.Entropy.get_local_post_branch_mig_script( 1090 repo) 1091 if os.path.isfile(post_branch_mig_file) or download: 1092 data['database_post_branch_hop_script'] = post_branch_mig_file 1093 extra_text_files.append(data['database_post_branch_hop_script']) 1094 1095 post_branch_upg_file = self.Entropy.get_local_post_branch_upg_script( 1096 repo) 1097 if os.path.isfile(post_branch_upg_file) or download: 1098 data['database_post_branch_upgrade_script'] = post_branch_upg_file 1099 extra_text_files.append(data['database_post_branch_upgrade_script']) 1100 1101 post_repo_update_file = self.Entropy.get_local_post_repo_update_script( 1102 repo) 1103 if os.path.isfile(post_repo_update_file) or download: 1104 data['database_post_repo_update_script'] = post_repo_update_file 1105 extra_text_files.append(data['database_post_repo_update_script']) 1106 1107 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1108 if os.path.isfile(database_ts_file) or download: 1109 data['database_timestamp_file'] = database_ts_file 1110 if not download: 1111 critical.append(database_ts_file) 1112 1113 database_package_mask_file = \ 1114 self.Entropy.get_local_database_mask_file(repo) 1115 extra_text_files.append(database_package_mask_file) 1116 if os.path.isfile(database_package_mask_file) or download: 1117 data['database_package_mask_file'] = database_package_mask_file 1118 if not download: 1119 critical.append(data['database_package_mask_file']) 1120 1121 database_package_system_mask_file = \ 1122 self.Entropy.get_local_database_system_mask_file(repo) 1123 extra_text_files.append(database_package_system_mask_file) 1124 if os.path.isfile(database_package_system_mask_file) or download: 1125 data['database_package_system_mask_file'] = \ 1126 database_package_system_mask_file 1127 if not download: 1128 critical.append(data['database_package_system_mask_file']) 1129 1130 database_package_confl_tagged_file = \ 1131 self.Entropy.get_local_database_confl_tagged_file(repo) 1132 extra_text_files.append(database_package_confl_tagged_file) 1133 if os.path.isfile(database_package_confl_tagged_file) or download: 1134 data['database_package_confl_tagged_file'] = \ 1135 database_package_confl_tagged_file 1136 if not download: 1137 critical.append(data['database_package_confl_tagged_file']) 1138 1139 database_license_whitelist_file = \ 1140 self.Entropy.get_local_database_licensewhitelist_file(repo) 1141 extra_text_files.append(database_license_whitelist_file) 1142 if os.path.isfile(database_license_whitelist_file) or download: 1143 data['database_license_whitelist_file'] = \ 1144 database_license_whitelist_file 1145 if not download: 1146 critical.append(data['database_license_whitelist_file']) 1147 1148 exp_based_pkgs_removal_file = \ 1149 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1150 extra_text_files.append(exp_based_pkgs_removal_file) 1151 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1152 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1153 if not download: 1154 critical.append(data['exp_based_pkgs_removal_file']) 1155 1156 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1157 if os.path.isfile(database_rss_file) or download: 1158 data['database_rss_file'] = database_rss_file 1159 if not download: 1160 critical.append(data['database_rss_file']) 1161 database_rss_light_file = \ 1162 self.Entropy.get_local_database_rsslight_file(repo) 1163 1164 extra_text_files.append(database_rss_light_file) 1165 if os.path.isfile(database_rss_light_file) or download: 1166 data['database_rss_light_file'] = database_rss_light_file 1167 if not download: 1168 critical.append(data['database_rss_light_file']) 1169 1170 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1171 data['pkglist_file'] = pkglist_file 1172 if not download: 1173 critical.append(data['pkglist_file']) 1174 1175 critical_updates_file = self.Entropy.get_local_critical_updates_file( 1176 repo) 1177 if os.path.isfile(critical_updates_file) or download: 1178 data['critical_updates_file'] = critical_updates_file 1179 extra_text_files.append(data['critical_updates_file']) 1180 if not download: 1181 critical.append(data['critical_updates_file']) 1182 1183 keywords_file = self.Entropy.get_local_database_keywords_file( 1184 repo) 1185 if os.path.isfile(keywords_file) or download: 1186 data['keywords_file'] = keywords_file 1187 extra_text_files.append(data['keywords_file']) 1188 if not download: 1189 critical.append(data['keywords_file']) 1190 1191 # EAPI 2,3 1192 if not download: # we don't need to get the dump 1193 1194 if 3 not in disabled_eapis: 1195 1196 data['metafiles_path'] = \ 1197 self.Entropy.get_local_database_compressed_metafiles_file( 1198 repo) 1199 critical.append(data['metafiles_path']) 1200 1201 if 2 not in disabled_eapis: 1202 1203 data['dump_path_light'] = os.path.join( 1204 self.Entropy.get_local_database_dir(repo), 1205 etpConst[cmethod[5]]) 1206 critical.append(data['dump_path_light']) 1207 1208 data['dump_path_digest_light'] = os.path.join( 1209 self.Entropy.get_local_database_dir(repo), 1210 etpConst[cmethod[6]]) 1211 critical.append(data['dump_path_digest_light']) 1212 1213 # EAPI 1 1214 if 1 not in disabled_eapis: 1215 1216 data['compressed_database_path'] = os.path.join( 1217 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1218 critical.append(data['compressed_database_path']) 1219 data['compressed_database_path_light'] = os.path.join( 1220 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1221 1222 data['database_path_digest'] = os.path.join( 1223 self.Entropy.get_local_database_dir(repo), 1224 etpConst['etpdatabasehashfile'] 1225 ) 1226 critical.append(data['database_path_digest']) 1227 1228 data['compressed_database_path_digest'] = os.path.join( 1229 self.Entropy.get_local_database_dir(repo), 1230 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1231 ) 1232 critical.append(data['compressed_database_path_digest']) 1233 1234 data['compressed_database_path_digest_light'] = os.path.join( 1235 self.Entropy.get_local_database_dir(repo), 1236 etpConst[cmethod[8]] 1237 ) 1238 critical.append(data['compressed_database_path_digest_light']) 1239 1240 1241 # SSL cert file, just for reference 1242 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1243 extra_text_files.append(ssl_ca_cert) 1244 if os.path.isfile(ssl_ca_cert): 1245 data['ssl_ca_cert_file'] = ssl_ca_cert 1246 if not download: 1247 critical.append(ssl_ca_cert) 1248 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1249 extra_text_files.append(ssl_server_cert) 1250 if os.path.isfile(ssl_server_cert): 1251 data['ssl_server_cert_file'] = ssl_server_cert 1252 if not download: 1253 critical.append(ssl_server_cert) 1254 1255 # Some information regarding how packages are built 1256 spm_files_map = self.Entropy.Spm_class().config_files_map() 1257 spm_syms = {} 1258 for myname, myfile in spm_files_map.items(): 1259 if os.path.islink(myfile): 1260 spm_syms[myname] = myfile 1261 continue # we don't want symlinks 1262 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1263 data[myname] = myfile 1264 extra_text_files.append(myfile) 1265 1266 # XXX/FIXME: for symlinks, we read their link and send a file with that 1267 # content. This is the default behaviour for now and allows to send 1268 # /etc/make.profile link pointer correctly. 1269 for symname, symfile in spm_syms.items(): 1270 1271 tmp_file = entropy.tools.get_random_temp_file() 1272 mytmpdir = os.path.dirname(tmp_file) 1273 mytmpfile = os.path.join(mytmpdir, os.path.basename(symfile)) 1274 extra_text_files.append(mytmpfile) 1275 1276 mylink = os.readlink(symfile) 1277 f_mkp = open(mytmpfile, "w") 1278 f_mkp.write(mylink) 1279 f_mkp.flush() 1280 f_mkp.close() 1281 data[symname] = mytmpfile 1282 1283 return data, critical, extra_text_files

1284

1285 - def _show_package_sets_messages(self, repo):

1286 1287 self.Entropy.updateProgress( 1288 "[repo:%s] %s:" % ( 1289 brown(repo), 1290 blue(_("configured package sets")), 1291 ), 1292 importance = 0, 1293 type = "info", 1294 header = darkgreen(" * ") 1295 ) 1296 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1297 if not sets_data: 1298 self.Entropy.updateProgress( 1299 "%s" % (_("None configured"),), 1300 importance = 0, 1301 type = "info", 1302 header = brown(" # ") 1303 ) 1304 return 1305 for s_repo, s_name, s_sets in sets_data: 1306 self.Entropy.updateProgress( 1307 blue("%s" % (s_name,)), 1308 importance = 0, 1309 type = "info", 1310 header = brown(" # ") 1311 )

1312

1313 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1314 1315 self.Entropy.updateProgress( 1316 "[repo:%s|%s|%s:%s] %s" % ( 1317 brown(repo), 1318 darkgreen(crippled_uri), 1319 red("EAPI"), 1320 bold("3"), 1321 blue(_("preparing uncompressed database for the upload")), 1322 ), 1323 importance = 0, 1324 type = "info", 1325 header = darkgreen(" * ") 1326 ) 1327 self.Entropy.updateProgress( 1328 "%s: %s" % (_("database path"), blue(database_path),), 1329 importance = 0, 1330 type = "info", 1331 header = brown(" # ") 1332 )

1333

1334 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1335 upload_data, cmethod, repo):

1336 1337 if repo is None: 1338 repo = self.Entropy.default_repository 1339 1340 self.Entropy.updateProgress( 1341 "[repo:%s|%s|%s:%s] %s" % ( 1342 brown(repo), 1343 darkgreen(crippled_uri), 1344 red("EAPI"), 1345 bold("2"), 1346 blue(_("creating compressed database dump + checksum")), 1347 ), 1348 importance = 0, 1349 type = "info", 1350 header = darkgreen(" * ") 1351 ) 1352 self.Entropy.updateProgress( 1353 "%s: %s" % (_("database path"), blue(database_path),), 1354 importance = 0, 1355 type = "info", 1356 header = brown(" # ") 1357 ) 1358 self.Entropy.updateProgress( 1359 "%s: %s" % ( 1360 _("dump light"), 1361 blue(upload_data['dump_path_light']), 1362 ), 1363 importance = 0, 1364 type = "info", 1365 header = brown(" # ") 1366 ) 1367 self.Entropy.updateProgress( 1368 "%s: %s" % ( 1369 _("dump light checksum"), 1370 blue(upload_data['dump_path_digest_light']), 1371 ), 1372 importance = 0, 1373 type = "info", 1374 header = brown(" # ") 1375 ) 1376 1377 self.Entropy.updateProgress( 1378 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1379 importance = 0, 1380 type = "info", 1381 header = brown(" # ") 1382 )

1383

1384 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1385 upload_data, cmethod, repo):

1386 1387 self.Entropy.updateProgress( 1388 "[repo:%s|%s|%s:%s] %s" % ( 1389 brown(repo), 1390 darkgreen(crippled_uri), 1391 red("EAPI"), 1392 bold("1"), 1393 blue(_("compressing database + checksum")), 1394 ), 1395 importance = 0, 1396 type = "info", 1397 header = darkgreen(" * "), 1398 back = True 1399 ) 1400 self.Entropy.updateProgress( 1401 "%s: %s" % (_("database path"), blue(database_path),), 1402 importance = 0, 1403 type = "info", 1404 header = brown(" # ") 1405 ) 1406 self.Entropy.updateProgress( 1407 "%s: %s" % ( 1408 _("compressed database path"), 1409 blue(upload_data['compressed_database_path']), 1410 ), 1411 importance = 0, 1412 type = "info", 1413 header = brown(" # ") 1414 ) 1415 self.Entropy.updateProgress( 1416 "%s: %s" % ( 1417 _("database checksum"), 1418 blue(upload_data['database_path_digest']), 1419 ), 1420 importance = 0, 1421 type = "info", 1422 header = brown(" # ") 1423 ) 1424 self.Entropy.updateProgress( 1425 "%s: %s" % ( 1426 _("compressed checksum"), 1427 blue(upload_data['compressed_database_path_digest']), 1428 ), 1429 importance = 0, 1430 type = "info", 1431 header = brown(" # ") 1432 ) 1433 self.Entropy.updateProgress( 1434 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1435 importance = 0, 1436 type = "info", 1437 header = brown(" # ") 1438 )

1439

1440 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1441 1442 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1443 os.path.isfile(x) and os.access(x, os.R_OK)] 1444 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1445 file_list if x not in found_file_list] 1446 metafile_not_found_file = \ 1447 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1448 1449 f_meta = open(metafile_not_found_file, "w") 1450 f_meta.writelines(not_found_file_list) 1451 f_meta.flush() 1452 f_meta.close() 1453 found_file_list.append(metafile_not_found_file) 1454 if os.path.isfile(compressed_dest_path): 1455 os.remove(compressed_dest_path) 1456 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1457

1458 - def mirror_lock_check(self, uri, repo = None):

1459 """ 1460 Return whether mirror is locked. 1461 """ 1462 1463 if repo is None: 1464 repo = self.Entropy.default_repository 1465 gave_up = False 1466 1467 lock_file = self.get_database_lockfile(repo) 1468 lock_filename = os.path.basename(lock_file) 1469 1470 remote_dir = os.path.join( 1471 self.Entropy.get_remote_database_relative_path(repo), 1472 self.SystemSettings['repositories']['branch']) 1473 remote_lock_file = os.path.join(remote_dir, lock_filename) 1474 1475 txc = self.Entropy.Transceiver(uri) 1476 with txc as handler: 1477 1478 if not os.path.isfile(lock_file) and \ 1479 handler.is_file(remote_lock_file): 1480 1481 crippled_uri = EntropyTransceiver.get_uri_name(uri) 1482 self.Entropy.updateProgress( 1483 "[repo:%s|%s|%s] %s, %s" % ( 1484 brown(str(repo)), 1485 darkgreen(crippled_uri), 1486 red(_("locking")), 1487 darkblue(_("mirror already locked")), 1488 blue(_("waiting up to 2 minutes before giving up")), 1489 ), 1490 importance = 1, 1491 type = "warning", 1492 header = brown(" * "), 1493 back = True 1494 ) 1495 1496 unlocked = False 1497 count = 0 1498 while count < 120: 1499 count += 1 1500 time.sleep(1) 1501 if not handler.is_file(remote_lock_file): 1502 self.Entropy.updateProgress( 1503 red("[repo:%s|%s|%s] %s !" % ( 1504 repo, 1505 crippled_uri, 1506 _("locking"), 1507 _("mirror unlocked"), 1508 ) 1509 ), 1510 importance = 1, 1511 type = "info", 1512 header = darkgreen(" * ") 1513 ) 1514 unlocked = True 1515 break 1516 1517 if not unlocked: 1518 gave_up = True 1519 1520 return gave_up

1521

1522 - def shrink_database_and_close(self, repo = None):

1523 dbconn = self.Entropy.open_server_repository(read_only = False, 1524 no_upload = True, repo = repo, indexing = False, 1525 do_treeupdates = False) 1526 dbconn.dropAllIndexes() 1527 dbconn.vacuum() 1528 dbconn.vacuum() 1529 dbconn.commitChanges() 1530 self.Entropy.close_server_database(dbconn)

1531

1532 - def update_repository_timestamp(self, repo = None):

1533 if repo is None: 1534 repo = self.Entropy.default_repository 1535 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1536 current_ts = self.Entropy.get_current_timestamp() 1537 ts_f = open(ts_file, "w") 1538 ts_f.write(current_ts) 1539 ts_f.flush() 1540 ts_f.close()

1541

1542 - def sync_database_treeupdates(self, repo = None):

1543 1544 if repo is None: 1545 repo = self.Entropy.default_repository 1546 dbconn = self.Entropy.open_server_repository(read_only = False, 1547 no_upload = True, repo = repo, do_treeupdates = False) 1548 # grab treeupdates from other databases and inject 1549 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1550 server_repos = list(srv_set['repositories'].keys()) 1551 all_actions = set() 1552 for myrepo in server_repos: 1553 1554 # avoid __default__ 1555 if myrepo == etpConst['clientserverrepoid']: 1556 continue 1557 1558 mydbc = self.Entropy.open_server_repository(just_reading = True, 1559 repo = myrepo) 1560 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1561 for data in actions: 1562 all_actions.add(data) 1563 if not actions: 1564 continue 1565 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1566 try: 1567 # clear first 1568 dbconn.removeTreeUpdatesActions(repo) 1569 dbconn.insertTreeUpdatesActions(all_actions, repo) 1570 except dbapi2.Error as err: 1571 self.entropyTools.print_traceback() 1572 mytxt = "%s, %s: %s. %s" % ( 1573 _("Troubles with treeupdates"), 1574 _("error"), 1575 err, 1576 _("Bumping old data back"), 1577 ) 1578 self.Entropy.updateProgress( 1579 mytxt, 1580 importance = 1, 1581 type = "warning" 1582 ) 1583 # restore previous data 1584 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1585 1586 dbconn.commitChanges() 1587 self.Entropy.close_server_database(dbconn)

1588

1589 - def upload_database(self, uris, lock_check = False, pretend = False, 1590 repo = None):

1591 1592 if repo is None: 1593 repo = self.Entropy.default_repository 1594 1595 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1596 if srv_set['rss']['enabled']: 1597 self.update_rss_feed(repo = repo) 1598 1599 upload_errors = False 1600 broken_uris = set() 1601 fine_uris = set() 1602 1603 disabled_eapis = sorted(srv_set['disabled_eapis']) 1604 1605 for uri in uris: 1606 1607 db_format = srv_set['database_file_format'] 1608 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1609 if cmethod is None: 1610 raise InvalidDataType("InvalidDataType: %s." % ( 1611 _("wrong database compression method passed"), 1612 ) 1613 ) 1614 1615 crippled_uri = EntropyTransceiver.get_uri_name(uri) 1616 database_path = self.Entropy.get_local_database_file(repo) 1617 1618 if disabled_eapis: 1619 self.Entropy.updateProgress( 1620 "[repo:%s|%s|%s] %s: %s" % ( 1621 blue(repo), 1622 red(crippled_uri), 1623 darkgreen(_("upload")), 1624 darkred(_("disabled EAPI")), 1625 bold(', '.join([str(x) for x in disabled_eapis])), 1626 ), 1627 importance = 1, 1628 type = "warning", 1629 header = darkgreen(" * ") 1630 ) 1631 1632 # create/update timestamp file 1633 self.update_repository_timestamp(repo) 1634 # create pkglist service file 1635 self.Entropy.create_repository_pkglist(repo) 1636 1637 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1638 repo = repo, disabled_eapis = disabled_eapis) 1639 1640 if lock_check: 1641 given_up = self.mirror_lock_check(uri, repo = repo) 1642 if given_up: 1643 upload_errors = True 1644 broken_uris.add(uri) 1645 continue 1646 1647 self.lock_mirrors_for_download(True, [uri], repo = repo) 1648 1649 self.Entropy.updateProgress( 1650 "[repo:%s|%s|%s] %s" % ( 1651 blue(repo), 1652 red(crippled_uri), 1653 darkgreen(_("upload")), 1654 darkgreen(_("preparing to upload database to mirror")), 1655 ), 1656 importance = 1, 1657 type = "info", 1658 header = darkgreen(" * ") 1659 ) 1660 1661 # Package Sets info 1662 self._show_package_sets_messages(repo) 1663 self.sync_database_treeupdates(repo) 1664 self.Entropy.update_database_package_sets(repo) 1665 self.Entropy.close_server_databases() 1666 1667 # backup current database to avoid re-indexing 1668 old_dbpath = self.Entropy.get_local_database_file(repo) 1669 backup_dbpath = old_dbpath + ".up_backup" 1670 copy_back = False 1671 if not pretend: 1672 try: 1673 if os.access(backup_dbpath, os.R_OK) and \ 1674 os.path.isfile(backup_dbpath): 1675 os.remove(backup_dbpath) 1676 1677 shutil.copy2(old_dbpath, backup_dbpath) 1678 copy_back = True 1679 except shutil.Error: 1680 copy_back = False 1681 1682 self.shrink_database_and_close(repo) 1683 1684 # EAPI 3 1685 if 3 not in disabled_eapis: 1686 self._create_metafiles_file(upload_data['metafiles_path'], 1687 text_files, repo) 1688 self._show_eapi3_upload_messages(crippled_uri, database_path, 1689 repo) 1690 1691 # EAPI 2 1692 if 2 not in disabled_eapis: 1693 self._show_eapi2_upload_messages(crippled_uri, database_path, 1694 upload_data, cmethod, repo) 1695 1696 # create compressed dump + checksum 1697 self.dump_database_to_file(database_path, 1698 upload_data['dump_path_light'], cmethod[0], 1699 exclude_tables = ["content"], repo = repo) 1700 self.create_file_checksum(upload_data['dump_path_light'], 1701 upload_data['dump_path_digest_light']) 1702 1703 1704 # EAPI 1 1705 if 1 not in disabled_eapis: 1706 self._show_eapi1_upload_messages(crippled_uri, database_path, 1707 upload_data, cmethod, repo) 1708 1709 # compress the database and create uncompressed 1710 # database checksum -- DEPRECATED 1711 self.compress_file(database_path, 1712 upload_data['compressed_database_path'], cmethod[0]) 1713 self.create_file_checksum(database_path, 1714 upload_data['database_path_digest']) 1715 1716 # create compressed database checksum 1717 self.create_file_checksum( 1718 upload_data['compressed_database_path'], 1719 upload_data['compressed_database_path_digest']) 1720 1721 # create light version of the compressed db 1722 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1723 temp_eapi1_dbfile = eapi1_dbfile+".light" 1724 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1725 # open and remove content table 1726 eapi1_tmp_dbconn = \ 1727 self.Entropy.ClientService.open_generic_database( 1728 temp_eapi1_dbfile, indexing_override = False, 1729 xcache = False) 1730 eapi1_tmp_dbconn.dropContent() 1731 eapi1_tmp_dbconn.commitChanges() 1732 eapi1_tmp_dbconn.vacuum() 1733 eapi1_tmp_dbconn.closeDB() 1734 1735 # compress 1736 self.compress_file(temp_eapi1_dbfile, 1737 upload_data['compressed_database_path_light'], cmethod[0]) 1738 # go away, we don't need you anymore 1739 os.remove(temp_eapi1_dbfile) 1740 # create compressed light database checksum 1741 self.create_file_checksum( 1742 upload_data['compressed_database_path_light'], 1743 upload_data['compressed_database_path_digest_light']) 1744 1745 if not pretend: 1746 # upload 1747 uploader = self.TransceiverServerHandler( 1748 self.Entropy, 1749 [uri], 1750 [upload_data[x] for x in upload_data], 1751 critical_files = critical, 1752 repo = repo 1753 ) 1754 errors, m_fine_uris, m_broken_uris = uploader.go() 1755 if errors: 1756 my_broken_uris = sorted([ 1757 (EntropyTransceiver.get_uri_name(x[0]), 1758 x[1]) for x in m_broken_uris]) 1759 self.Entropy.updateProgress( 1760 "[repo:%s|%s|%s] %s" % ( 1761 repo, 1762 crippled_uri, 1763 _("errors"), 1764 _("upload failed, not unlocking and continuing"), 1765 ), 1766 importance = 0, 1767 type = "error", 1768 header = darkred(" !!! ") 1769 ) 1770 # get reason 1771 reason = my_broken_uris[0][1] 1772 self.Entropy.updateProgress( 1773 blue("%s: %s" % (_("reason"), reason,)), 1774 importance = 0, 1775 type = "error", 1776 header = blue(" # ") 1777 ) 1778 upload_errors = True 1779 broken_uris |= m_broken_uris 1780 continue 1781 1782 # copy db back 1783 if copy_back and os.path.isfile(backup_dbpath): 1784 self.Entropy.close_server_databases() 1785 further_backup_dbpath = old_dbpath+".security_backup" 1786 if os.path.isfile(further_backup_dbpath): 1787 os.remove(further_backup_dbpath) 1788 shutil.copy2(old_dbpath, further_backup_dbpath) 1789 shutil.move(backup_dbpath, old_dbpath) 1790 1791 # unlock 1792 self.lock_mirrors_for_download(False, [uri], repo = repo) 1793 fine_uris |= m_fine_uris 1794 1795 if not fine_uris: 1796 upload_errors = True 1797 return upload_errors, broken_uris, fine_uris

1798 1799

1800 - def download_database(self, uris, lock_check = False, pretend = False, 1801 repo = None):

1802 1803 if repo is None: 1804 repo = self.Entropy.default_repository 1805 1806 download_errors = False 1807 broken_uris = set() 1808 fine_uris = set() 1809 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1810 disabled_eapis = sorted(srv_set['disabled_eapis']) 1811 1812 for uri in uris: 1813 1814 db_format = srv_set['database_file_format'] 1815 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1816 if cmethod is None: 1817 raise InvalidDataType("InvalidDataType: %s." % ( 1818 _("wrong database compression method passed"), 1819 ) 1820 ) 1821 1822 crippled_uri = EntropyTransceiver.get_uri_name(uri) 1823 database_path = self.Entropy.get_local_database_file(repo) 1824 database_dir_path = os.path.dirname( 1825 self.Entropy.get_local_database_file(repo)) 1826 download_data, critical, text_files = self.get_files_to_sync( 1827 cmethod, download = True, 1828 repo = repo, disabled_eapis = disabled_eapis) 1829 mytmpdir = self.entropyTools.get_random_temp_file() 1830 os.makedirs(mytmpdir) 1831 1832 self.Entropy.updateProgress( 1833 "[repo:%s|%s|%s] %s" % ( 1834 brown(repo), 1835 darkgreen(crippled_uri), 1836 red(_("download")), 1837 blue(_("preparing to download database from mirror")), 1838 ), 1839 importance = 1, 1840 type = "info", 1841 header = darkgreen(" * ") 1842 ) 1843 files_to_sync = sorted(download_data.keys()) 1844 for myfile in files_to_sync: 1845 self.Entropy.updateProgress( 1846 "%s: %s" % ( 1847 blue(_("download path")), 1848 brown(download_data[myfile]), 1849 ), 1850 importance = 0, 1851 type = "info", 1852 header = brown(" # ") 1853 ) 1854 1855 if lock_check: 1856 given_up = self.mirror_lock_check(uri, repo = repo) 1857 if given_up: 1858 download_errors = True 1859 broken_uris.add(uri) 1860 continue 1861 1862 # avoid having others messing while we're downloading 1863 self.lock_mirrors(True, [uri], repo = repo) 1864 1865 if not pretend: 1866 # download 1867 downloader = self.TransceiverServerHandler( 1868 self.Entropy, [uri], 1869 [download_data[x] for x in download_data], download = True, 1870 local_basedir = mytmpdir, critical_files = critical, 1871 repo = repo 1872 ) 1873 errors, m_fine_uris, m_broken_uris = downloader.go() 1874 if errors: 1875 my_broken_uris = sorted([ 1876 (EntropyTransceiver.get_uri_name(x[0]), 1877 x[1]) for x in m_broken_uris]) 1878 self.Entropy.updateProgress( 1879 "[repo:%s|%s|%s] %s" % ( 1880 brown(repo), 1881 darkgreen(crippled_uri), 1882 red(_("errors")), 1883 blue(_("failed to download from mirror")), 1884 ), 1885 importance = 0, 1886 type = "error", 1887 header = darkred(" !!! ") 1888 ) 1889 # get reason 1890 reason = my_broken_uris[0][1] 1891 self.Entropy.updateProgress( 1892 blue("%s: %s" % (_("reason"), reason,)), 1893 importance = 0, 1894 type = "error", 1895 header = blue(" # ") 1896 ) 1897 download_errors = True 1898 broken_uris |= m_broken_uris 1899 self.lock_mirrors(False, [uri], repo = repo) 1900 continue 1901 1902 # all fine then, we need to move data from mytmpdir 1903 # to database_dir_path 1904 1905 # EAPI 1 -- unpack database 1906 if 1 not in disabled_eapis: 1907 compressed_db_filename = os.path.basename( 1908 download_data['compressed_database_path']) 1909 uncompressed_db_filename = os.path.basename(database_path) 1910 compressed_file = os.path.join(mytmpdir, 1911 compressed_db_filename) 1912 uncompressed_file = os.path.join(mytmpdir, 1913 uncompressed_db_filename) 1914 self.entropyTools.uncompress_file(compressed_file, 1915 uncompressed_file, cmethod[0]) 1916 1917 # now move 1918 for myfile in os.listdir(mytmpdir): 1919 fromfile = os.path.join(mytmpdir, myfile) 1920 tofile = os.path.join(database_dir_path, myfile) 1921 shutil.move(fromfile, tofile) 1922 self.Entropy.ClientService.setup_default_file_perms(tofile) 1923 1924 if os.path.isdir(mytmpdir): 1925 shutil.rmtree(mytmpdir) 1926 if os.path.isdir(mytmpdir): 1927 os.rmdir(mytmpdir) 1928 1929 1930 fine_uris.add(uri) 1931 self.lock_mirrors(False, [uri], repo = repo) 1932 1933 return download_errors, fine_uris, broken_uris

1934

1935 - def calculate_database_sync_queues(self, repo = None):

1936 1937 if repo is None: 1938 repo = self.Entropy.default_repository 1939 1940 remote_status = self.get_remote_databases_status(repo) 1941 local_revision = self.Entropy.get_local_database_revision(repo) 1942 upload_queue = [] 1943 download_latest = () 1944 1945 # all mirrors are empty ? I rule 1946 if not [x for x in remote_status if x[1]]: 1947 upload_queue = remote_status[:] 1948 else: 1949 highest_remote_revision = max([x[1] for x in remote_status]) 1950 1951 if local_revision < highest_remote_revision: 1952 for remote_item in remote_status: 1953 if remote_item[1] == highest_remote_revision: 1954 download_latest = x 1955 break 1956 1957 if download_latest: 1958 upload_queue = [x for x in remote_status if \ 1959 (x[1] < highest_remote_revision)] 1960 else: 1961 upload_queue = [x for x in remote_status if \ 1962 (x[1] < local_revision)] 1963 1964 return download_latest, upload_queue

1965

1966 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 1967 repo = None):

1968 1969 if repo is None: 1970 repo = self.Entropy.default_repository 1971 1972 while True: 1973 1974 db_locked = False 1975 if self.is_local_database_locked(repo): 1976 db_locked = True 1977 1978 lock_data = self.get_mirrors_lock(repo) 1979 mirrors_locked = [x for x in lock_data if x[1]] 1980 1981 if not mirrors_locked and db_locked: 1982 # mirrors not locked remotely but only locally 1983 mylock_file = self.get_database_lockfile(repo) 1984 if os.access(mylock_file, os.W_OK) and \ 1985 os.path.isfile(mylock_file): 1986 1987 os.remove(mylock_file) 1988 continue 1989 1990 break 1991 1992 if mirrors_locked and not db_locked: 1993 mytxt = "%s, %s %s" % ( 1994 _("Mirrors are locked, someone is working on the repository"), 1995 _("try again later"), 1996 "...", 1997 ) 1998 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 1999 2000 download_latest, upload_queue = self.calculate_database_sync_queues( 2001 repo) 2002 2003 if not download_latest and not upload_queue: 2004 self.Entropy.updateProgress( 2005 "[repo:%s|%s] %s" % ( 2006 brown(repo), 2007 red(_("sync")), # something short please 2008 blue(_("database already in sync")), 2009 ), 2010 importance = 1, 2011 type = "info", 2012 header = blue(" @@ ") 2013 ) 2014 return 0, set(), set() 2015 2016 if download_latest: 2017 download_uri = download_latest[0] 2018 download_errors, fine_uris, broken_uris = self.download_database( 2019 [download_uri], repo = repo) 2020 if download_errors: 2021 self.Entropy.updateProgress( 2022 "[repo:%s|%s] %s: %s" % ( 2023 brown(repo), 2024 red(_("sync")), 2025 blue(_("database sync failed")), 2026 red(_("download issues")), 2027 ), 2028 importance = 1, 2029 type = "error", 2030 header = darkred(" !!! ") 2031 ) 2032 return 1, fine_uris, broken_uris 2033 2034 if upload_queue and not no_upload: 2035 2036 # XXX QA checks, 2037 # please group them into entropy.qa 2038 2039 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2040 base_repo = srv_set['base_repository_id'] 2041 if base_repo is None: 2042 base_repo = repo 2043 2044 base_deps_not_found = set() 2045 if base_repo != repo: 2046 base_deps_not_found = self.Entropy.dependencies_test( 2047 repo = base_repo) 2048 2049 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2050 if (deps_not_found or base_deps_not_found) \ 2051 and not self.Entropy.community_repo: 2052 2053 self.Entropy.updateProgress( 2054 "[repo:%s|%s] %s: %s" % ( 2055 brown(repo), 2056 red(_("sync")), 2057 blue(_("database sync forbidden")), 2058 red(_("dependencies_test() reported errors")), 2059 ), 2060 importance = 1, 2061 type = "error", 2062 header = darkred(" !!! ") 2063 ) 2064 return 3, set(), set() 2065 2066 problems = self.Entropy.check_config_file_updates() 2067 if problems: 2068 return 4, set(), set() 2069 2070 self.Entropy.updateProgress( 2071 "[repo:%s|%s] %s" % ( 2072 brown(repo), 2073 red(_("config files")), # something short please 2074 blue(_("no configuration files to commit. All fine.")), 2075 ), 2076 importance = 1, 2077 type = "info", 2078 header = blue(" @@ "), 2079 back = True 2080 ) 2081 2082 uris = [x[0] for x in upload_queue] 2083 errors, fine_uris, broken_uris = self.upload_database(uris, 2084 repo = repo) 2085 if errors: 2086 self.Entropy.updateProgress( 2087 "[repo:%s|%s] %s: %s" % ( 2088 brown(repo), 2089 red(_("sync")), 2090 blue(_("database sync failed")), 2091 red(_("upload issues")), 2092 ), 2093 importance = 1, 2094 type = "error", 2095 header = darkred(" !!! ") 2096 ) 2097 return 2, fine_uris, broken_uris 2098 2099 2100 self.Entropy.updateProgress( 2101 "[repo:%s|%s] %s" % ( 2102 brown(repo), 2103 red(_("sync")), 2104 blue(_("database sync completed successfully")), 2105 ), 2106 importance = 1, 2107 type = "info", 2108 header = darkgreen(" * ") 2109 ) 2110 2111 if unlock_mirrors: 2112 self.lock_mirrors(False, repo = repo) 2113 return 0, set(), set()

2114 2115

2116 - def calculate_local_upload_files(self, branch, repo = None):

2117 upload_files = 0 2118 upload_packages = set() 2119 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2120 branch) 2121 2122 # check if it exists 2123 if not os.path.isdir(upload_dir): 2124 return upload_files, upload_packages 2125 2126 pkg_ext = etpConst['packagesext'] 2127 pkg_md5_ext = etpConst['packagesmd5fileext'] 2128 for package in os.listdir(upload_dir): 2129 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2130 upload_packages.add(package) 2131 if package.endswith(pkg_ext): 2132 upload_files += 1 2133 2134 return upload_files, upload_packages

2135

2136 - def calculate_local_package_files(self, branch, repo = None):

2137 local_files = 0 2138 local_packages = set() 2139 packages_dir = os.path.join( 2140 self.Entropy.get_local_packages_directory(repo), branch) 2141 2142 if not os.path.isdir(packages_dir): 2143 os.makedirs(packages_dir) 2144 2145 pkg_ext = etpConst['packagesext'] 2146 pkg_md5_ext = etpConst['packagesmd5fileext'] 2147 for package in os.listdir(packages_dir): 2148 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2149 local_packages.add(package) 2150 if package.endswith(pkg_ext): 2151 local_files += 1 2152 2153 return local_files, local_packages

2154 2155

2156 - def _show_local_sync_stats(self, upload_files, local_files):

2157 self.Entropy.updateProgress( 2158 "%s:" % ( 2159 blue(_("Local statistics")), 2160 ), 2161 importance = 1, 2162 type = "info", 2163 header = red(" @@ ") 2164 ) 2165 self.Entropy.updateProgress( 2166 red("%s:\t\t%s %s" % ( 2167 blue(_("upload directory")), 2168 bold(str(upload_files)), 2169 red(_("files ready")), 2170 ) 2171 ), 2172 importance = 0, 2173 type = "info", 2174 header = red(" @@ ") 2175 ) 2176 self.Entropy.updateProgress( 2177 red("%s:\t\t%s %s" % ( 2178 blue(_("packages directory")), 2179 bold(str(local_files)), 2180 red(_("files ready")), 2181 ) 2182 ), 2183 importance = 0, 2184 type = "info", 2185 header = red(" @@ ") 2186 )

2187

2188 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2189 branch):

2190 2191 # show stats 2192 for package, size in upload: 2193 package = darkgreen(os.path.basename(package)) 2194 size = blue(self.entropyTools.bytes_into_human(size)) 2195 self.Entropy.updateProgress( 2196 "[branch:%s|%s] %s [%s]" % ( 2197 brown(branch), 2198 blue(_("upload")), 2199 darkgreen(package), 2200 size, 2201 ), 2202 importance = 0, 2203 type = "info", 2204 header = red(" # ") 2205 ) 2206 for package, size in download: 2207 package = darkred(os.path.basename(package)) 2208 size = blue(self.entropyTools.bytes_into_human(size)) 2209 self.Entropy.updateProgress( 2210 "[branch:%s|%s] %s [%s]" % ( 2211 brown(branch), 2212 darkred(_("download")), 2213 blue(package), 2214 size, 2215 ), 2216 importance = 0, 2217 type = "info", 2218 header = red(" # ") 2219 ) 2220 for package, size in copy: 2221 package = darkblue(os.path.basename(package)) 2222 size = blue(self.entropyTools.bytes_into_human(size)) 2223 self.Entropy.updateProgress( 2224 "[branch:%s|%s] %s [%s]" % ( 2225 brown(branch), 2226 darkgreen(_("copy")), 2227 brown(package), 2228 size, 2229 ), 2230 importance = 0, 2231 type = "info", 2232 header = red(" # ") 2233 ) 2234 for package, size in removal: 2235 package = brown(os.path.basename(package)) 2236 size = blue(self.entropyTools.bytes_into_human(size)) 2237 self.Entropy.updateProgress( 2238 "[branch:%s|%s] %s [%s]" % ( 2239 brown(branch), 2240 red(_("remove")), 2241 red(package), 2242 size, 2243 ), 2244 importance = 0, 2245 type = "info", 2246 header = red(" # ") 2247 ) 2248 2249 self.Entropy.updateProgress( 2250 "%s:\t\t\t%s" % ( 2251 blue(_("Packages to be removed")), 2252 darkred(str(len(removal))), 2253 ), 2254 importance = 0, 2255 type = "info", 2256 header = blue(" @@ ") 2257 ) 2258 self.Entropy.updateProgress( 2259 "%s:\t\t%s" % ( 2260 darkgreen(_("Packages to be moved locally")), 2261 darkgreen(str(len(copy))), 2262 ), 2263 importance = 0, 2264 type = "info", 2265 header = blue(" @@ ") 2266 ) 2267 self.Entropy.updateProgress( 2268 "%s:\t\t\t%s" % ( 2269 bold(_("Packages to be uploaded")), 2270 bold(str(len(upload))), 2271 ), 2272 importance = 0, 2273 type = "info", 2274 header = blue(" @@ ") 2275 ) 2276 2277 self.Entropy.updateProgress( 2278 "%s:\t\t\t%s" % ( 2279 darkred(_("Total removal size")), 2280 darkred( 2281 self.entropyTools.bytes_into_human(metainfo['removal']) 2282 ), 2283 ), 2284 importance = 0, 2285 type = "info", 2286 header = blue(" @@ ") 2287 ) 2288 2289 self.Entropy.updateProgress( 2290 "%s:\t\t\t%s" % ( 2291 blue(_("Total upload size")), 2292 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2293 ), 2294 importance = 0, 2295 type = "info", 2296 header = blue(" @@ ") 2297 ) 2298 self.Entropy.updateProgress( 2299 "%s:\t\t\t%s" % ( 2300 brown(_("Total download size")), 2301 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2302 ), 2303 importance = 0, 2304 type = "info", 2305 header = blue(" @@ ") 2306 )

2307

2308 - def _calculate_remote_package_files(self, uri, branch, txc_handler, 2309 repo = None):

2310 2311 remote_dir = os.path.join( 2312 self.Entropy.get_remote_packages_relative_path(repo), branch) 2313 2314 # create path to lock file if it doesn't exist 2315 if not txc_handler.is_dir(remote_dir): 2316 txc_handler.makedirs(remote_dir) 2317 2318 remote_packages_info = txc_handler.list_content_metadata(remote_dir) 2319 remote_packages = [x[0] for x in remote_packages_info] 2320 2321 remote_files = 0 2322 for pkg in remote_packages: 2323 if pkg.endswith(etpConst['packagesext']): 2324 remote_files += 1 2325 2326 remote_packages_data = {} 2327 for pkg in remote_packages_info: 2328 remote_packages_data[pkg[0]] = int(pkg[1]) 2329 2330 return remote_files, remote_packages, remote_packages_data

2331

2332 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2333 2334 if repo is None: 2335 repo = self.Entropy.default_repository 2336 2337 crippled_uri = EntropyTransceiver.get_uri_name(uri) 2338 upload_files, upload_packages = self.calculate_local_upload_files( 2339 branch, repo) 2340 local_files, local_packages = self.calculate_local_package_files(branch, 2341 repo) 2342 self._show_local_sync_stats(upload_files, local_files) 2343 2344 self.Entropy.updateProgress( 2345 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2346 importance = 1, 2347 type = "info", 2348 header = red(" @@ ") 2349 ) 2350 2351 txc = self.Entropy.Transceiver(uri) 2352 with txc as handler: 2353 remote_files, remote_packages, remote_packages_data = \ 2354 self._calculate_remote_package_files(uri, branch, handler, 2355 repo = repo) 2356 2357 self.Entropy.updateProgress( 2358 "%s:\t\t\t%s %s" % ( 2359 blue(_("remote packages")), 2360 bold(str(remote_files)), 2361 red(_("files stored")), 2362 ), 2363 importance = 0, 2364 type = "info", 2365 header = red(" @@ ") 2366 ) 2367 2368 mytxt = blue("%s ...") % ( 2369 _("Calculating queues"), 2370 ) 2371 self.Entropy.updateProgress( 2372 mytxt, 2373 importance = 1, 2374 type = "info", 2375 header = red(" @@ ") 2376 ) 2377 2378 upload_queue, download_queue, removal_queue, fine_queue = \ 2379 self.calculate_sync_queues(upload_packages, local_packages, 2380 remote_packages, remote_packages_data, branch, repo) 2381 return upload_queue, download_queue, removal_queue, fine_queue, \ 2382 remote_packages_data

2383

2384 - def calculate_sync_queues( 2385 self, 2386 upload_packages, 2387 local_packages, 2388 remote_packages, 2389 remote_packages_data, 2390 branch, 2391 repo = None 2392 ):

2393 2394 upload_queue = set() 2395 download_queue = set() 2396 removal_queue = set() 2397 fine_queue = set() 2398 2399 for local_package in upload_packages: 2400 if local_package in remote_packages: 2401 2402 local_filepath = os.path.join( 2403 self.Entropy.get_local_upload_directory(repo), branch, 2404 local_package) 2405 2406 local_size = self.entropyTools.get_file_size(local_filepath) 2407 remote_size = remote_packages_data.get(local_package) 2408 if remote_size is None: 2409 remote_size = 0 2410 if (local_size != remote_size): 2411 # size does not match, adding to the upload queue 2412 upload_queue.add(local_package) 2413 else: 2414 # just move from upload to packages 2415 fine_queue.add(local_package) 2416 else: 2417 # always force upload of packages in uploaddir 2418 upload_queue.add(local_package) 2419 2420 # if a package is in the packages directory but not online, 2421 # we have to upload it we have local_packages and remote_packages 2422 for local_package in local_packages: 2423 if local_package in remote_packages: 2424 local_filepath = os.path.join( 2425 self.Entropy.get_local_packages_directory(repo), branch, 2426 local_package) 2427 local_size = self.entropyTools.get_file_size(local_filepath) 2428 remote_size = remote_packages_data.get(local_package) 2429 if remote_size is None: 2430 remote_size = 0 2431 if (local_size != remote_size) and (local_size != 0): 2432 # size does not match, adding to the upload queue 2433 if local_package not in fine_queue: 2434 upload_queue.add(local_package) 2435 else: 2436 # this means that the local package does not exist 2437 # so, we need to download it 2438 upload_queue.add(local_package) 2439 2440 # Fill download_queue and removal_queue 2441 for remote_package in remote_packages: 2442 if remote_package in local_packages: 2443 local_filepath = os.path.join( 2444 self.Entropy.get_local_packages_directory(repo), branch, 2445 remote_package) 2446 local_size = self.entropyTools.get_file_size(local_filepath) 2447 remote_size = remote_packages_data.get(remote_package) 2448 if remote_size is None: 2449 remote_size = 0 2450 if (local_size != remote_size) and (local_size != 0): 2451 # size does not match, remove first 2452 # do it only if the package has not been 2453 # added to the upload_queue 2454 if remote_package not in upload_queue: 2455 # remotePackage == localPackage 2456 # just remove something that differs 2457 # from the content of the mirror 2458 removal_queue.add(remote_package) 2459 # then add to the download queue 2460 download_queue.add(remote_package) 2461 else: 2462 # this means that the local package does not exist 2463 # so, we need to download it 2464 # ignore .tmp files 2465 if not remote_package.endswith(".tmp"): 2466 download_queue.add(remote_package) 2467 2468 # Collect packages that don't exist anymore in the database 2469 # so we can filter them out from the download queue 2470 dbconn = self.Entropy.open_server_repository(just_reading = True, 2471 repo = repo) 2472 db_files = dbconn.listAllDownloads(do_sort = False, 2473 full_path = True) 2474 db_files = set([os.path.basename(x) for x in db_files if \ 2475 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2476 2477 exclude = set() 2478 for myfile in download_queue: 2479 if myfile.endswith(etpConst['packagesext']): 2480 if myfile not in db_files: 2481 exclude.add(myfile) 2482 download_queue -= exclude 2483 2484 exclude = set() 2485 for myfile in upload_queue: 2486 if myfile.endswith(etpConst['packagesext']): 2487 if myfile not in db_files: 2488 exclude.add(myfile) 2489 upload_queue -= exclude 2490 2491 exclude = set() 2492 for myfile in download_queue: 2493 if myfile in upload_queue: 2494 exclude.add(myfile) 2495 download_queue -= exclude 2496 2497 return upload_queue, download_queue, removal_queue, fine_queue

2498 2499

2500 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2501 remote_packages_data, branch, repo):

2502 2503 metainfo = { 2504 'removal': 0, 2505 'download': 0, 2506 'upload': 0, 2507 } 2508 removal = [] 2509 download = [] 2510 do_copy = [] 2511 upload = [] 2512 2513 for item in removal_queue: 2514 if not item.endswith(etpConst['packagesext']): 2515 continue 2516 local_filepath = os.path.join( 2517 self.Entropy.get_local_packages_directory(repo), branch, item) 2518 size = self.entropyTools.get_file_size(local_filepath) 2519 metainfo['removal'] += size 2520 removal.append((local_filepath, size)) 2521 2522 for item in download_queue: 2523 if not item.endswith(etpConst['packagesext']): 2524 continue 2525 local_filepath = os.path.join( 2526 self.Entropy.get_local_upload_directory(repo), branch, item) 2527 if not os.path.isfile(local_filepath): 2528 size = remote_packages_data.get(item) 2529 if size is None: 2530 size = 0 2531 size = int(size) 2532 metainfo['removal'] += size 2533 download.append((local_filepath, size)) 2534 else: 2535 size = self.entropyTools.get_file_size(local_filepath) 2536 do_copy.append((local_filepath, size)) 2537 2538 for item in upload_queue: 2539 if not item.endswith(etpConst['packagesext']): 2540 continue 2541 local_filepath = os.path.join( 2542 self.Entropy.get_local_upload_directory(repo), branch, item) 2543 local_filepath_pkgs = os.path.join( 2544 self.Entropy.get_local_packages_directory(repo), branch, item) 2545 if os.path.isfile(local_filepath): 2546 size = self.entropyTools.get_file_size(local_filepath) 2547 upload.append((local_filepath, size)) 2548 else: 2549 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2550 upload.append((local_filepath_pkgs, size)) 2551 metainfo['upload'] += size 2552 2553 return upload, download, removal, do_copy, metainfo

2554 2555

2556 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2557 2558 if repo is None: 2559 repo = self.Entropy.default_repository 2560 2561 for itemdata in removal_queue: 2562 2563 remove_filename = itemdata[0] 2564 remove_filepath = os.path.join( 2565 self.Entropy.get_local_packages_directory(repo), branch, 2566 remove_filename) 2567 remove_filepath_hash = remove_filepath + \ 2568 etpConst['packagesmd5fileext'] 2569 self.Entropy.updateProgress( 2570 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2571 brown(repo), 2572 red("sync"), 2573 brown(branch), 2574 blue(_("removing package+hash")), 2575 darkgreen(remove_filename), 2576 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2577 ), 2578 importance = 0, 2579 type = "info", 2580 header = darkred(" * ") 2581 ) 2582 2583 if os.path.isfile(remove_filepath): 2584 os.remove(remove_filepath) 2585 if os.path.isfile(remove_filepath_hash): 2586 os.remove(remove_filepath_hash) 2587 2588 self.Entropy.updateProgress( 2589 "[repo:%s|%s|%s] %s" % ( 2590 brown(repo), 2591 red(_("sync")), 2592 brown(branch), 2593 blue(_("removal complete")), 2594 ), 2595 importance = 0, 2596 type = "info", 2597 header = darkred(" * ") 2598 )

2599 2600

2601 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2602 2603 if repo is None: 2604 repo = self.Entropy.default_repository 2605 2606 for itemdata in copy_queue: 2607 2608 from_file = itemdata[0] 2609 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2610 to_file = os.path.join( 2611 self.Entropy.get_local_packages_directory(repo), branch, 2612 os.path.basename(from_file)) 2613 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2614 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2615 self.Entropy.updateProgress( 2616 "[repo:%s|%s|%s] %s: %s" % ( 2617 brown(repo), 2618 red("sync"), 2619 brown(branch), 2620 blue(_("copying file+hash to repository")), 2621 darkgreen(from_file), 2622 ), 2623 importance = 0, 2624 type = "info", 2625 header = darkred(" * ") 2626 ) 2627 2628 if not os.path.isdir(os.path.dirname(to_file)): 2629 os.makedirs(os.path.dirname(to_file)) 2630 2631 shutil.copy2(from_file, to_file) 2632 if not os.path.isfile(from_file_hash): 2633 self.create_file_checksum(from_file, from_file_hash) 2634 shutil.copy2(from_file_hash, to_file_hash) 2635 2636 # clear expiration file 2637 if os.path.isfile(expiration_file): 2638 os.remove(expiration_file)

2639 2640

2641 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2642 2643 if repo is None: 2644 repo = self.Entropy.default_repository 2645 2646 crippled_uri = EntropyTransceiver.get_uri_name(uri) 2647 myqueue = [] 2648 for itemdata in upload_queue: 2649 upload_item = itemdata[0] 2650 hash_file = upload_item + etpConst['packagesmd5fileext'] 2651 if not os.path.isfile(hash_file): 2652 self.entropyTools.create_md5_file(upload_item) 2653 myqueue.append(hash_file) 2654 myqueue.append(upload_item) 2655 2656 remote_dir = os.path.join( 2657 self.Entropy.get_remote_packages_relative_path(repo), branch) 2658 2659 uploader = self.TransceiverServerHandler(self.Entropy, [uri], 2660 myqueue, critical_files = myqueue, 2661 txc_basedir = remote_dir, 2662 handlers_data = {'branch': branch }, repo = repo) 2663 2664 errors, m_fine_uris, m_broken_uris = uploader.go() 2665 if errors: 2666 my_broken_uris = [ 2667 (EntropyTransceiver.get_uri_name(x[0]), x[1]) for \ 2668 x in m_broken_uris] 2669 reason = my_broken_uris[0][1] 2670 self.Entropy.updateProgress( 2671 "[branch:%s] %s: %s, %s: %s" % ( 2672 brown(branch), 2673 blue(_("upload errors")), 2674 red(crippled_uri), 2675 blue(_("reason")), 2676 darkgreen(repr(reason)), 2677 ), 2678 importance = 1, 2679 type = "error", 2680 header = darkred(" !!! ") 2681 ) 2682 return errors, m_fine_uris, m_broken_uris 2683 2684 self.Entropy.updateProgress( 2685 "[branch:%s] %s: %s" % ( 2686 brown(branch), 2687 blue(_("upload completed successfully")), 2688 red(crippled_uri), 2689 ), 2690 importance = 1, 2691 type = "info", 2692 header = blue(" @@ ") 2693 ) 2694 return errors, m_fine_uris, m_broken_uris

2695 2696

2697 - def _sync_run_download_queue(self, uri, download_queue, branch, 2698 repo = None):

2699 2700 if repo is None: 2701 repo = self.Entropy.default_repository 2702 2703 crippled_uri = EntropyTransceiver.get_uri_name(uri) 2704 myqueue = [] 2705 for package in download_queue: 2706 hash_file = package + etpConst['packagesmd5fileext'] 2707 myqueue.append(package) 2708 myqueue.append(hash_file) 2709 2710 remote_dir = os.path.join( 2711 self.Entropy.get_remote_packages_relative_path(repo), branch) 2712 local_basedir = os.path.join( 2713 self.Entropy.get_local_packages_directory(repo), branch) 2714 downloader = self.TransceiverServerHandler( 2715 self.Entropy, [uri], myqueue, 2716 critical_files = myqueue, 2717 txc_basedir = remote_dir, local_basedir = local_basedir, 2718 handlers_data = {'branch': branch }, download = True, repo = repo) 2719 2720 errors, m_fine_uris, m_broken_uris = downloader.go() 2721 if errors: 2722 my_broken_uris = [ 2723 (EntropyTransceiver.get_uri_name(x), y,) \ 2724 for x, y in m_broken_uris] 2725 reason = my_broken_uris[0][1] 2726 self.Entropy.updateProgress( 2727 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2728 brown(repo), 2729 red(_("sync")), 2730 brown(branch), 2731 blue(_("download errors")), 2732 darkgreen(crippled_uri), 2733 blue(_("reason")), 2734 reason, 2735 ), 2736 importance = 1, 2737 type = "error", 2738 header = darkred(" !!! ") 2739 ) 2740 return errors, m_fine_uris, m_broken_uris 2741 2742 self.Entropy.updateProgress( 2743 "[repo:%s|%s|%s] %s: %s" % ( 2744 brown(repo), 2745 red(_("sync")), 2746 brown(branch), 2747 blue(_("download completed successfully")), 2748 darkgreen(crippled_uri), 2749 ), 2750 importance = 1, 2751 type = "info", 2752 header = darkgreen(" * ") 2753 ) 2754 return errors, m_fine_uris, m_broken_uris

2755

2756 - def run_package_files_qa_checks(self, packages_list, repo = None):

2757 2758 if repo is None: 2759 repo = self.Entropy.default_repository 2760 2761 my_qa = self.Entropy.QA() 2762 2763 qa_total = len(packages_list) 2764 qa_count = 0 2765 qa_some_faulty = [] 2766 2767 for upload_package in packages_list: 2768 qa_count += 1 2769 2770 self.Entropy.updateProgress( 2771 "%s: %s" % ( 2772 purple(_("QA checking package file")), 2773 darkgreen(os.path.basename(upload_package)), 2774 ), 2775 importance = 0, 2776 type = "info", 2777 header = purple(" @@ "), 2778 back = True, 2779 count = (qa_count, qa_total,) 2780 ) 2781 2782 result = my_qa.entropy_package_checks(upload_package) 2783 if not result: 2784 # call wolfman-911 2785 qa_some_faulty.append(os.path.basename(upload_package)) 2786 2787 if qa_some_faulty: 2788 2789 for qa_faulty_pkg in qa_some_faulty: 2790 self.Entropy.updateProgress( 2791 "[repo:%s|branch:%s] %s: %s" % ( 2792 brown(repo), 2793 self.SystemSettings['repositories']['branch'], 2794 red(_("faulty package file, please fix")), 2795 blue(os.path.basename(qa_faulty_pkg)), 2796 ), 2797 importance = 1, 2798 type = "error", 2799 header = darkred(" @@ ") 2800 ) 2801 raise EntropyPackageException( 2802 'EntropyPackageException: cannot continue')

2803 2804

2805 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2806 repo = None):

2807 2808 if repo is None: 2809 repo = self.Entropy.default_repository 2810 2811 self.Entropy.updateProgress( 2812 "[repo:%s|%s] %s" % ( 2813 repo, 2814 red(_("sync")), 2815 darkgreen(_("starting packages sync")), 2816 ), 2817 importance = 1, 2818 type = "info", 2819 header = red(" @@ "), 2820 back = True 2821 ) 2822 2823 2824 successfull_mirrors = set() 2825 broken_mirrors = set() 2826 check_data = () 2827 upload_queue_qa_checked = set() 2828 mirrors_tainted = False 2829 mirror_errors = False 2830 mirrors_errors = False 2831 2832 for uri in self.Entropy.get_remote_mirrors(repo): 2833 2834 crippled_uri = EntropyTransceiver.get_uri_name(uri) 2835 mirror_errors = False 2836 2837 self.Entropy.updateProgress( 2838 "[repo:%s|%s|branch:%s] %s: %s" % ( 2839 repo, 2840 red(_("sync")), 2841 brown(self.SystemSettings['repositories']['branch']), 2842 blue(_("packages sync")), 2843 bold(crippled_uri), 2844 ), 2845 importance = 1, 2846 type = "info", 2847 header = red(" @@ ") 2848 ) 2849 2850 try: 2851 upload_queue, download_queue, removal_queue, fine_queue, \ 2852 remote_packages_data = self.calculate_packages_to_sync(uri, 2853 self.SystemSettings['repositories']['branch'], repo) 2854 except self.socket.error as err: 2855 self.Entropy.updateProgress( 2856 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2857 repo, 2858 red(_("sync")), 2859 self.SystemSettings['repositories']['branch'], 2860 darkred(_("socket error")), 2861 err, 2862 darkred(_("on")), 2863 crippled_uri, 2864 ), 2865 importance = 1, 2866 type = "error", 2867 header = darkgreen(" * ") 2868 ) 2869 continue 2870 2871 if (not upload_queue) and (not download_queue) and \ 2872 (not removal_queue): 2873 self.Entropy.updateProgress( 2874 "[repo:%s|%s|branch:%s] %s: %s" % ( 2875 repo, 2876 red(_("sync")), 2877 self.SystemSettings['repositories']['branch'], 2878 darkgreen(_("nothing to do on")), 2879 crippled_uri, 2880 ), 2881 importance = 1, 2882 type = "info", 2883 header = darkgreen(" * ") 2884 ) 2885 successfull_mirrors.add(uri) 2886 continue 2887 2888 self.Entropy.updateProgress( 2889 "%s:" % (blue(_("Expanding queues")),), 2890 importance = 1, 2891 type = "info", 2892 header = red(" ** ") 2893 ) 2894 2895 upload, download, removal, copy, metainfo = self.expand_queues( 2896 upload_queue, 2897 download_queue, 2898 removal_queue, 2899 remote_packages_data, 2900 self.SystemSettings['repositories']['branch'], 2901 repo 2902 ) 2903 del upload_queue, download_queue, removal_queue, \ 2904 remote_packages_data 2905 self._show_sync_queues(upload, download, removal, copy, metainfo, 2906 self.SystemSettings['repositories']['branch']) 2907 2908 if not len(upload)+len(download)+len(removal)+len(copy): 2909 2910 self.Entropy.updateProgress( 2911 "[repo:%s|%s|branch:%s] %s %s" % ( 2912 self.Entropy.default_repository, 2913 red(_("sync")), 2914 self.SystemSettings['repositories']['branch'], 2915 blue(_("nothing to sync for")), 2916 crippled_uri, 2917 ), 2918 importance = 1, 2919 type = "info", 2920 header = darkgreen(" @@ ") 2921 ) 2922 2923 successfull_mirrors.add(uri) 2924 continue 2925 2926 if pretend: 2927 successfull_mirrors.add(uri) 2928 continue 2929 2930 if ask: 2931 rc_sync = self.Entropy.askQuestion( 2932 _("Would you like to run the steps above ?")) 2933 if rc_sync == _("No"): 2934 continue 2935 2936 try: 2937 2938 # QA checks 2939 qa_package_files = [x[0] for x in upload if x[0] \ 2940 not in upload_queue_qa_checked] 2941 upload_queue_qa_checked |= set(qa_package_files) 2942 2943 self.run_package_files_qa_checks(qa_package_files, repo = repo) 2944 2945 if removal: 2946 self._sync_run_removal_queue(removal, 2947 self.SystemSettings['repositories']['branch'], repo) 2948 2949 if copy: 2950 self._sync_run_copy_queue(copy, 2951 self.SystemSettings['repositories']['branch'], repo) 2952 2953 if upload or download: 2954 mirrors_tainted = True 2955 2956 if upload: 2957 d_errors, m_fine_uris, \ 2958 m_broken_uris = self._sync_run_upload_queue( 2959 uri, upload, 2960 self.SystemSettings['repositories']['branch'], repo) 2961 2962 if d_errors: 2963 mirror_errors = True 2964 2965 if download: 2966 my_downlist = [x[0] for x in download] 2967 d_errors, m_fine_uris, \ 2968 m_broken_uris = self._sync_run_download_queue( 2969 uri, my_downlist, 2970 self.SystemSettings['repositories']['branch'], repo) 2971 2972 if d_errors: 2973 mirror_errors = True 2974 if not mirror_errors: 2975 successfull_mirrors.add(uri) 2976 else: 2977 mirrors_errors = True 2978 2979 except KeyboardInterrupt: 2980 self.Entropy.updateProgress( 2981 "[repo:%s|%s|branch:%s] %s" % ( 2982 repo, 2983 red(_("sync")), 2984 self.SystemSettings['repositories']['branch'], 2985 darkgreen(_("keyboard interrupt !")), 2986 ), 2987 importance = 1, 2988 type = "info", 2989 header = darkgreen(" * ") 2990 ) 2991 continue 2992 2993 except EntropyPackageException as err: 2994 2995 mirrors_errors = True 2996 broken_mirrors.add(uri) 2997 successfull_mirrors.clear() 2998 # so that people will realize this is a very bad thing 2999 self.Entropy.updateProgress( 3000 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3001 repo, 3002 red(_("sync")), 3003 self.SystemSettings['repositories']['branch'], 3004 darkred(_("you must package them again")), 3005 EntropyPackageException, 3006 _("error"), 3007 err, 3008 ), 3009 importance = 1, 3010 type = "error", 3011 header = darkred(" !!! ") 3012 ) 3013 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3014 broken_mirrors, check_data 3015 3016 except Exception as err: 3017 3018 self.entropyTools.print_traceback() 3019 mirrors_errors = True 3020 broken_mirrors.add(uri) 3021 self.Entropy.updateProgress( 3022 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3023 repo, 3024 red(_("sync")), 3025 self.SystemSettings['repositories']['branch'], 3026 darkred(_("exception caught")), 3027 Exception, 3028 _("error"), 3029 err, 3030 ), 3031 importance = 1, 3032 type = "error", 3033 header = darkred(" !!! ") 3034 ) 3035 3036 exc_txt = entropy.tools.print_exception( 3037 returndata = True) 3038 for line in exc_txt: 3039 self.Entropy.updateProgress( 3040 repr(line), 3041 importance = 1, 3042 type = "error", 3043 header = darkred(": ") 3044 ) 3045 3046 if len(successfull_mirrors) > 0: 3047 self.Entropy.updateProgress( 3048 "[repo:%s|%s|branch:%s] %s" % ( 3049 repo, 3050 red(_("sync")), 3051 self.SystemSettings['repositories']['branch'], 3052 darkred( 3053 _("at least one mirror synced properly!")), 3054 ), 3055 importance = 1, 3056 type = "error", 3057 header = darkred(" !!! ") 3058 ) 3059 continue 3060 3061 # if at least one server has been synced successfully, move files 3062 if (len(successfull_mirrors) > 0) and not pretend: 3063 self.remove_expiration_files( 3064 self.SystemSettings['repositories']['branch'], repo) 3065 3066 if packages_check: 3067 check_data = self.Entropy.verify_local_packages([], ask = ask, 3068 repo = repo) 3069 3070 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3071 broken_mirrors, check_data

3072

3073 - def remove_expiration_files(self, branch, repo = None):

3074 3075 if repo is None: 3076 repo = self.Entropy.default_repository 3077 3078 branch_dir = os.path.join( 3079 self.Entropy.get_local_upload_directory(repo), branch) 3080 3081 # check if it exists 3082 if not os.path.isdir(branch_dir): 3083 return None 3084 3085 branchcontent = os.listdir(branch_dir) 3086 for xfile in branchcontent: 3087 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3088 branch, xfile) 3089 destdir = os.path.join( 3090 self.Entropy.get_local_packages_directory(repo), branch) 3091 if not os.path.isdir(destdir): 3092 os.makedirs(destdir) 3093 dest = os.path.join(destdir, xfile) 3094 shutil.move(source, dest) 3095 # clear expiration file 3096 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3097 if os.path.isfile(dest_expiration): 3098 os.remove(dest_expiration)

3099 3100

3101 - def is_package_expired(self, package_file, branch, repo = None):

3102 3103 pkg_path = os.path.join( 3104 self.Entropy.get_local_packages_directory(repo), branch, 3105 package_file) 3106 pkg_path += etpConst['packagesexpirationfileext'] 3107 if not os.path.isfile(pkg_path): 3108 return False 3109 3110 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3111 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3112 days = srv_set['packages_expiration_days'] 3113 delta = int(days)*24*3600 3114 currmtime = time.time() 3115 file_delta = currmtime - mtime 3116 3117 if file_delta > delta: 3118 return True 3119 return False

3120

3121 - def create_expiration_file(self, package_file, branch, repo = None, 3122 gentle = False):

3123 3124 pkg_path = os.path.join( 3125 self.Entropy.get_local_packages_directory(repo), branch, 3126 package_file) 3127 pkg_path += etpConst['packagesexpirationfileext'] 3128 if gentle and os.path.isfile(pkg_path): 3129 return 3130 f_exp = open(pkg_path, "w") 3131 f_exp.flush() 3132 f_exp.close()

3133 3134

3135 - def collect_expiring_packages(self, branch, repo = None):

3136 3137 dbconn = self.Entropy.open_server_repository(just_reading = True, 3138 repo = repo) 3139 database_bins = dbconn.listAllDownloads(do_sort = False, 3140 full_path = True) 3141 bins_dir = os.path.join( 3142 self.Entropy.get_local_packages_directory(repo), branch) 3143 repo_bins = set() 3144 3145 if os.path.isdir(bins_dir): 3146 repo_bins = os.listdir(bins_dir) 3147 repo_bins = set([ 3148 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3149 in repo_bins if x.endswith(etpConst['packagesext'])]) 3150 repo_bins -= database_bins 3151 3152 return set([os.path.basename(x) for x in repo_bins])

3153 3154

3155 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3156 3157 if repo is None: 3158 repo = self.Entropy.default_repository 3159 3160 self.Entropy.updateProgress( 3161 "[repo:%s|%s|branch:%s] %s" % ( 3162 brown(repo), 3163 red(_("tidy")), 3164 blue(self.SystemSettings['repositories']['branch']), 3165 blue(_("collecting expired packages")), 3166 ), 3167 importance = 1, 3168 type = "info", 3169 header = red(" @@ ") 3170 ) 3171 3172 branch_data = {} 3173 errors = False 3174 branch_data['errors'] = False 3175 branch = self.SystemSettings['repositories']['branch'] 3176 3177 self.Entropy.updateProgress( 3178 "[branch:%s] %s" % ( 3179 brown(branch), 3180 blue(_("collecting expired packages in the selected branches")), 3181 ), 3182 importance = 1, 3183 type = "info", 3184 header = blue(" @@ ") 3185 ) 3186 3187 # collect removed packages 3188 expiring_packages = self.collect_expiring_packages(branch, repo) 3189 if expiring_packages: 3190 3191 # filter expired packages used by other branches 3192 # this is done for the sake of consistency 3193 # --- read packages.db.pkglist, make sure your repository 3194 # has been ported to latest Entropy 3195 3196 branch_pkglist_data = self.read_remote_file_in_branches( 3197 etpConst['etpdatabasepkglist'], repo = repo, 3198 excluded_branches = [branch]) 3199 # format data 3200 for key, val in list(branch_pkglist_data.items()): 3201 branch_pkglist_data[key] = val.split("\n") 3202 3203 3204 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3205 branch) 3206 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3207 expiring_packages]) 3208 3209 for other_branch in branch_pkglist_data: 3210 branch_pkglist = set(branch_pkglist_data[other_branch]) 3211 my_expiring_pkgs -= branch_pkglist 3212 3213 # fallback to normality, set new expiring packages var 3214 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3215 3216 removal = [] 3217 for package in expiring_packages: 3218 expired = self.is_package_expired(package, branch, repo) 3219 if expired: 3220 removal.append(package) 3221 else: 3222 self.create_expiration_file(package, branch, repo, 3223 gentle = True) 3224 3225 # fill returning data 3226 branch_data['removal'] = removal[:] 3227 3228 if not removal: 3229 self.Entropy.updateProgress( 3230 "[branch:%s] %s" % ( 3231 brown(branch), 3232 blue(_("nothing to remove on this branch")), 3233 ), 3234 importance = 1, 3235 type = "info", 3236 header = blue(" @@ ") 3237 ) 3238 return errors, branch_data 3239 else: 3240 self.Entropy.updateProgress( 3241 "[branch:%s] %s:" % ( 3242 brown(branch), 3243 blue(_("these are the expired packages")), 3244 ), 3245 importance = 1, 3246 type = "info", 3247 header = blue(" @@ ") 3248 ) 3249 for package in removal: 3250 self.Entropy.updateProgress( 3251 "[branch:%s] %s: %s" % ( 3252 brown(branch), 3253 blue(_("remove")), 3254 darkgreen(package), 3255 ), 3256 importance = 1, 3257 type = "info", 3258 header = brown(" # ") 3259 ) 3260 3261 if pretend: 3262 return errors, branch_data 3263 3264 if ask: 3265 rc_question = self.Entropy.askQuestion( 3266 _("Would you like to continue ?")) 3267 if rc_question == _("No"): 3268 return errors, branch_data 3269 3270 myqueue = [] 3271 for package in removal: 3272 myqueue.append(package+etpConst['packagesmd5fileext']) 3273 myqueue.append(package) 3274 3275 remote_dir = os.path.join( 3276 self.Entropy.get_remote_packages_relative_path(repo), branch) 3277 for uri in self.Entropy.get_remote_mirrors(repo): 3278 3279 self.Entropy.updateProgress( 3280 "[branch:%s] %s..." % ( 3281 brown(branch), 3282 blue(_("removing packages remotely")), 3283 ), 3284 importance = 1, 3285 type = "info", 3286 header = blue(" @@ ") 3287 ) 3288 3289 crippled_uri = EntropyTransceiver.get_uri_name(uri) 3290 destroyer = self.TransceiverServerHandler( 3291 self.Entropy, 3292 [uri], 3293 myqueue, 3294 critical_files = [], 3295 txc_basedir = remote_dir, 3296 remove = True, 3297 repo = repo 3298 ) 3299 errors, m_fine_uris, m_broken_uris = destroyer.go() 3300 if errors: 3301 my_broken_uris = [ 3302 (EntropyTransceiver.get_uri_name(x[0]), x[1]) \ 3303 for x in m_broken_uris] 3304 3305 reason = my_broken_uris[0][1] 3306 self.Entropy.updateProgress( 3307 "[branch:%s] %s: %s, %s: %s" % ( 3308 brown(branch), 3309 blue(_("remove errors")), 3310 red(crippled_uri), 3311 blue(_("reason")), 3312 reason, 3313 ), 3314 importance = 1, 3315 type = "warning", 3316 header = brown(" !!! ") 3317 ) 3318 branch_data['errors'] = True 3319 errors = True 3320 3321 self.Entropy.updateProgress( 3322 "[branch:%s] %s..." % ( 3323 brown(branch), 3324 blue(_("removing packages locally")), 3325 ), 3326 importance = 1, 3327 type = "info", 3328 header = blue(" @@ ") 3329 ) 3330 3331 branch_data['removed'] = set() 3332 for package in removal: 3333 package_path = os.path.join( 3334 self.Entropy.get_local_packages_directory(repo), 3335 branch, package) 3336 package_path_hash = package_path + \ 3337 etpConst['packagesmd5fileext'] 3338 package_path_expired = package_path + \ 3339 etpConst['packagesexpirationfileext'] 3340 3341 my_rm_list = (package_path_hash, package_path, 3342 package_path_expired) 3343 for myfile in my_rm_list: 3344 if os.path.isfile(myfile): 3345 self.Entropy.updateProgress( 3346 "[branch:%s] %s: %s" % ( 3347 brown(branch), 3348 blue(_("removing")), 3349 darkgreen(myfile), 3350 ), 3351 importance = 1, 3352 type = "info", 3353 header = brown(" @@ ") 3354 ) 3355 os.remove(myfile) 3356 branch_data['removed'].add(myfile) 3357 3358 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors