entropy.server.interfaces.mirrors

39 40 import socket 41 import entropy.dump as dumpTools 42 import entropy.tools as entropyTools

43 - def __init__(self, ServerInstance, repo = None):

44 45 from entropy.server.interfaces.main import Server as MainServer 46 if not isinstance(ServerInstance, MainServer): 47 mytxt = _("entropy.server.interfaces.main.Server needed") 48 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 49 50 self.Entropy = ServerInstance 51 from entropy.transceivers import FtpServerHandler 52 self.FtpServerHandler = FtpServerHandler 53 from entropy.cache import EntropyCacher 54 self.Cacher = EntropyCacher() 55 self.sys_settings_plugin_id = \ 56 etpConst['system_settings_plugins_ids']['server_plugin'] 57 self.SystemSettings = self.Entropy.SystemSettings 58 59 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 60 self.Entropy.updateProgress( 61 mytxt, 62 importance = 2, 63 type = "info", 64 header = red(" @@ ") 65 ) 66 mytxt = _("mirror") 67 for mirror in self.Entropy.get_remote_mirrors(repo): 68 mirror = self.entropyTools.hide_ftp_password(mirror) 69 self.Entropy.updateProgress( 70 blue("%s: %s") % (mytxt, darkgreen(mirror),), 71 importance = 0, 72 type = "info", 73 header = brown(" # ") 74 )

75

76 - def get_remote_branches(self, repo = None):

77 """ 78 Returns a list of remotely available branches for the provided 79 repository identifier. 80 81 @keyword repo: repository identifier 82 @type repo: string 83 @return: list of valid, available remote branches 84 @rtype: set 85 """ 86 87 if repo == None: 88 repo = self.Entropy.default_repository 89 90 remote_branches = set() 91 # this is used to validate Entropy repository path 92 ts_file = etpConst['etpdatabasetimestampfile'] 93 mirrors = self.Entropy.get_remote_mirrors(repo) 94 for uri in mirrors: 95 96 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 97 98 self.Entropy.updateProgress( 99 "[repo:%s] %s: %s" % ( 100 brown(repo), 101 blue(_("listing branches in mirror")), 102 darkgreen(crippled_uri), 103 ), 104 importance = 1, 105 type = "info", 106 header = brown(" @@ ") 107 ) 108 109 ftp = FtpInterface(uri, self.Entropy, verbose = False) 110 try: 111 112 branches_path = self.Entropy.get_remote_database_relative_path( 113 repo) 114 try: 115 ftp.set_cwd(branches_path) 116 except FtpError: 117 continue # no branches in this mirror 118 119 branches = ftp.list_dir() 120 for branch in branches: 121 mypath = os.path.join("/", branches_path, branch) 122 try: 123 ftp.set_cwd(mypath) 124 except FtpError: 125 # not a dir 126 continue 127 if ftp.is_file_available(ts_file): 128 remote_branches.add(branch) 129 130 finally: 131 ftp.close() 132 133 return remote_branches

134

135 - def read_remote_file_in_branches(self, filename, repo = None, 136 excluded_branches = None):

137 """ 138 Reads a file remotely located in all the available branches. 139 140 @param filename: name of the file that should be located inside 141 repository database directory 142 @type filename: string 143 @keyword repo: repository identifier 144 @type repo: string 145 @keyword excluded_branches: list of branch identifiers excluded or None 146 @type excluded_branches: list or None 147 @return: dictionary with branches as key and raw file content as value: 148 {'4': 'abcd\n', '5': 'defg\n'} 149 @rtype: dict 150 """ 151 if repo == None: 152 repo = self.Entropy.default_repository 153 if excluded_branches == None: 154 excluded_branches = [] 155 156 branch_data = {} 157 mirrors = self.Entropy.get_remote_mirrors(repo) 158 for uri in mirrors: 159 160 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 161 162 self.Entropy.updateProgress( 163 "[repo:%s] %s: %s => %s" % ( 164 brown(repo), 165 blue(_("looking for file in mirror")), 166 darkgreen(crippled_uri), 167 filename, 168 ), 169 importance = 1, 170 type = "info", 171 header = brown(" @@ ") 172 ) 173 174 ftp = FtpInterface(uri, self.Entropy, verbose = False) 175 try: 176 177 branches_path = self.Entropy.get_remote_database_relative_path( 178 repo) 179 try: 180 ftp.set_cwd(branches_path) 181 except FtpError: 182 continue # no branches in this mirror 183 184 branches = ftp.list_dir() 185 for branch in branches: 186 187 # is branch excluded ? 188 if branch in excluded_branches: 189 continue 190 191 if branch_data.get(branch) != None: 192 # already read 193 continue 194 195 mypath = os.path.join("/", branches_path, branch) 196 try: 197 ftp.set_cwd(mypath) 198 except FtpError: 199 # not a dir 200 continue 201 202 if not ftp.is_file_available(filename): 203 # nothing to do, no file here 204 continue 205 206 tmp_dir = tempfile.mkdtemp() 207 tries = 4 208 success = False 209 while tries: 210 downloaded = ftp.download_file(filename, tmp_dir) 211 if not downloaded: 212 tries -= 1 213 continue # argh! 214 success = True 215 break 216 217 down_path = os.path.join(tmp_dir, filename) 218 if success and os.access(down_path, os.F_OK): 219 down_f = open(down_path) 220 branch_data[branch] = down_f.read() 221 down_f.close() 222 shutil.rmtree(tmp_dir) 223 224 finally: 225 ftp.close() 226 227 return branch_data

228

229 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

230 231 if repo == None: 232 repo = self.Entropy.default_repository 233 234 if not mirrors: 235 mirrors = self.Entropy.get_remote_mirrors(repo) 236 237 issues = False 238 for uri in mirrors: 239 240 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 241 242 lock_text = _("unlocking") 243 if lock: 244 lock_text = _("locking") 245 self.Entropy.updateProgress( 246 "[repo:%s|%s] %s %s" % ( 247 brown(repo), 248 darkgreen(crippled_uri), 249 bold(lock_text), 250 blue("%s...") % (_("mirror"),), 251 ), 252 importance = 1, 253 type = "info", 254 header = brown(" * "), 255 back = True 256 ) 257 258 try: 259 ftp = FtpInterface(uri, self.Entropy) 260 except ConnectionError: 261 self.entropyTools.print_traceback() 262 return True # issues 263 my_path = os.path.join( 264 self.Entropy.get_remote_database_relative_path(repo), 265 self.SystemSettings['repositories']['branch']) 266 ftp.set_cwd(my_path, dodir = True) 267 268 lock_file = etpConst['etpdatabaselockfile'] 269 if lock and ftp.is_file_available(lock_file): 270 self.Entropy.updateProgress( 271 "[repo:%s|%s] %s" % ( 272 brown(repo), 273 darkgreen(crippled_uri), 274 blue(_("mirror already locked")), 275 ), 276 importance = 1, 277 type = "info", 278 header = darkgreen(" * ") 279 ) 280 ftp.close() 281 continue 282 elif not lock and not ftp.is_file_available(lock_file): 283 self.Entropy.updateProgress( 284 "[repo:%s|%s] %s" % ( 285 brown(repo), 286 darkgreen(crippled_uri), 287 blue(_("mirror already unlocked")), 288 ), 289 importance = 1, 290 type = "info", 291 header = darkgreen(" * ") 292 ) 293 ftp.close() 294 continue 295 296 if lock: 297 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 298 else: 299 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 300 ftp.close() 301 if not rc_lock: 302 issues = True 303 304 if not issues: 305 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 306 if os.path.isfile(db_taint_file): 307 os.remove(db_taint_file) 308 309 return issues

310 311

312 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 313 repo = None):

314 # this functions makes entropy clients to not download anything 315 # from the chosen mirrors. it is used to avoid clients to 316 # download databases while we're uploading a new one 317 318 if mirrors == None: 319 mirrors = [] 320 321 if repo == None: 322 repo = self.Entropy.default_repository 323 324 if not mirrors: 325 mirrors = self.Entropy.get_remote_mirrors(repo) 326 327 issues = False 328 for uri in mirrors: 329 330 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 331 332 lock_text = _("unlocking") 333 if lock: 334 lock_text = _("locking") 335 self.Entropy.updateProgress( 336 "[repo:%s|%s] %s %s..." % ( 337 blue(repo), 338 red(crippled_uri), 339 bold(lock_text), 340 blue(_("mirror for download")), 341 ), 342 importance = 1, 343 type = "info", 344 header = red(" @@ "), 345 back = True 346 ) 347 348 try: 349 ftp = FtpInterface(uri, self.Entropy) 350 except ConnectionError: 351 self.entropyTools.print_traceback() 352 return True # issues 353 my_path = os.path.join( 354 self.Entropy.get_remote_database_relative_path(repo), 355 self.SystemSettings['repositories']['branch']) 356 ftp.set_cwd(my_path, dodir = True) 357 358 lock_file = etpConst['etpdatabasedownloadlockfile'] 359 360 if lock and ftp.is_file_available(lock_file): 361 self.Entropy.updateProgress( 362 "[repo:%s|%s] %s" % ( 363 blue(repo), 364 red(crippled_uri), 365 blue(_("mirror already locked for download")), 366 ), 367 importance = 1, 368 type = "info", 369 header = red(" @@ ") 370 ) 371 ftp.close() 372 continue 373 374 elif not lock and not ftp.is_file_available(lock_file): 375 self.Entropy.updateProgress( 376 "[repo:%s|%s] %s" % ( 377 blue(repo), 378 red(crippled_uri), 379 blue(_("mirror already unlocked for download")), 380 ), 381 importance = 1, 382 type = "info", 383 header = red(" @@ ") 384 ) 385 ftp.close() 386 continue 387 388 if lock: 389 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 390 repo = repo) 391 else: 392 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 393 repo = repo) 394 ftp.close() 395 if not rc_lock: 396 issues = True 397 398 return issues

399

400 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 401 repo = None):

402 403 if repo == None: 404 repo = self.Entropy.default_repository 405 406 my_path = os.path.join( 407 self.Entropy.get_remote_database_relative_path(repo), 408 self.SystemSettings['repositories']['branch']) 409 if not ftp_connection: 410 try: 411 ftp_connection = FtpInterface(uri, self.Entropy) 412 except ConnectionError: 413 self.entropyTools.print_traceback() 414 return False # issues 415 ftp_connection.set_cwd(my_path, dodir = True) 416 else: 417 mycwd = ftp_connection.get_cwd() 418 if mycwd != my_path: 419 ftp_connection.set_basedir() 420 ftp_connection.set_cwd(my_path, dodir = True) 421 422 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 423 lock_string = u'' 424 425 if dblock: 426 self.create_local_database_lockfile(repo) 427 lock_file = self.get_database_lockfile(repo) 428 else: 429 # locking/unlocking mirror1 for download 430 lock_string = _('for download') 431 self.create_local_database_download_lockfile(repo) 432 lock_file = self.get_database_download_lockfile(repo) 433 434 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 435 if rc_upload: 436 self.Entropy.updateProgress( 437 "[repo:%s|%s] %s %s" % ( 438 blue(repo), 439 red(crippled_uri), 440 blue(_("mirror successfully locked")), 441 blue(lock_string), 442 ), 443 importance = 1, 444 type = "info", 445 header = red(" @@ ") 446 ) 447 else: 448 self.Entropy.updateProgress( 449 "[repo:%s|%s] %s: %s - %s %s" % ( 450 blue(repo), 451 red(crippled_uri), 452 blue("lock error"), 453 rc_upload, 454 blue(_("mirror not locked")), 455 blue(lock_string), 456 ), 457 importance = 1, 458 type = "error", 459 header = darkred(" * ") 460 ) 461 self.remove_local_database_lockfile(repo) 462 463 return rc_upload

464 465

466 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

467 468 if repo == None: 469 repo = self.Entropy.default_repository 470 471 my_path = os.path.join( 472 self.Entropy.get_remote_database_relative_path(repo), 473 self.SystemSettings['repositories']['branch']) 474 if not ftp_connection: 475 try: 476 ftp_connection = FtpInterface(uri, self.Entropy) 477 except ConnectionError: 478 self.entropyTools.print_traceback() 479 return False # issues 480 ftp_connection.set_cwd(my_path) 481 else: 482 mycwd = ftp_connection.get_cwd() 483 if mycwd != my_path: 484 ftp_connection.set_basedir() 485 ftp_connection.set_cwd(my_path) 486 487 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 488 489 if dblock: 490 dbfile = etpConst['etpdatabaselockfile'] 491 else: 492 dbfile = etpConst['etpdatabasedownloadlockfile'] 493 rc_delete = ftp_connection.delete_file(dbfile) 494 if rc_delete: 495 self.Entropy.updateProgress( 496 "[repo:%s|%s] %s" % ( 497 blue(repo), 498 red(crippled_uri), 499 blue(_("mirror successfully unlocked")), 500 ), 501 importance = 1, 502 type = "info", 503 header = darkgreen(" * ") 504 ) 505 if dblock: 506 self.remove_local_database_lockfile(repo) 507 else: 508 self.remove_local_database_download_lockfile(repo) 509 else: 510 self.Entropy.updateProgress( 511 "[repo:%s|%s] %s: %s - %s" % ( 512 blue(repo), 513 red(crippled_uri), 514 blue(_("unlock error")), 515 rc_delete, 516 blue(_("mirror not unlocked")), 517 ), 518 importance = 1, 519 type = "error", 520 header = darkred(" * ") 521 ) 522 523 return rc_delete

524

525 - def get_database_lockfile(self, repo = None):

526 if repo == None: 527 repo = self.Entropy.default_repository 528 return os.path.join(self.Entropy.get_local_database_dir(repo), 529 etpConst['etpdatabaselockfile'])

530

531 - def get_database_download_lockfile(self, repo = None):

532 if repo == None: 533 repo = self.Entropy.default_repository 534 return os.path.join(self.Entropy.get_local_database_dir(repo), 535 etpConst['etpdatabasedownloadlockfile'])

536

537 - def create_local_database_download_lockfile(self, repo = None):

538 if repo == None: 539 repo = self.Entropy.default_repository 540 lock_file = self.get_database_download_lockfile(repo) 541 f_lock = open(lock_file, "w") 542 f_lock.write("download locked") 543 f_lock.flush() 544 f_lock.close()

545

546 - def create_local_database_lockfile(self, repo = None):

547 if repo == None: 548 repo = self.Entropy.default_repository 549 lock_file = self.get_database_lockfile(repo) 550 f_lock = open(lock_file, "w") 551 f_lock.write("database locked") 552 f_lock.flush() 553 f_lock.close()

554

555 - def remove_local_database_lockfile(self, repo = None):

556 if repo == None: 557 repo = self.Entropy.default_repository 558 lock_file = self.get_database_lockfile(repo) 559 if os.path.isfile(lock_file): 560 os.remove(lock_file)

561

562 - def remove_local_database_download_lockfile(self, repo = None):

563 if repo == None: 564 repo = self.Entropy.default_repository 565 lock_file = self.get_database_download_lockfile(repo) 566 if os.path.isfile(lock_file): 567 os.remove(lock_file)

568

569 - def download_package(self, uri, pkg_relative_path, repo = None):

570 571 if repo == None: 572 repo = self.Entropy.default_repository 573 574 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 575 576 tries = 0 577 while tries < 5: 578 tries += 1 579 580 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 581 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 582 pkgfile = os.path.basename(pkg_relative_path) 583 584 self.Entropy.updateProgress( 585 "[repo:%s|%s|#%s] %s: %s" % ( 586 brown(repo), 587 darkgreen(crippled_uri), 588 brown(tries), 589 blue(_("connecting to download package")), 590 darkgreen(pkg_to_join_path), 591 ), 592 importance = 1, 593 type = "info", 594 header = darkgreen(" * "), 595 back = True 596 ) 597 598 try: 599 ftp = FtpInterface(uri, self.Entropy) 600 except ConnectionError: 601 self.entropyTools.print_traceback() 602 return False # issues 603 dirpath = os.path.join( 604 self.Entropy.get_remote_packages_relative_path(repo), 605 pkg_to_join_dirpath) 606 ftp.set_cwd(dirpath, dodir = True) 607 608 self.Entropy.updateProgress( 609 "[repo:%s|%s|#%s] %s: %s" % ( 610 brown(repo), 611 darkgreen(crippled_uri), 612 brown(tries), 613 blue(_("downloading package")), 614 darkgreen(pkg_to_join_path), 615 ), 616 importance = 1, 617 type = "info", 618 header = darkgreen(" * ") 619 ) 620 621 download_path = os.path.join( 622 self.Entropy.get_local_packages_directory(repo), 623 pkg_to_join_dirpath) 624 if (not os.path.isdir(download_path)) and \ 625 (not os.access(download_path, os.R_OK)): 626 os.makedirs(download_path) 627 rc_download = ftp.download_file(pkgfile, download_path) 628 if not rc_download: 629 self.Entropy.updateProgress( 630 "[repo:%s|%s|#%s] %s: %s %s" % ( 631 brown(repo), 632 darkgreen(crippled_uri), 633 brown(tries), 634 blue(_("package")), 635 darkgreen(pkg_to_join_path), 636 blue(_("does not exist")), 637 ), 638 importance = 1, 639 type = "error", 640 header = darkred(" !!! ") 641 ) 642 ftp.close() 643 return rc_download 644 645 dbconn = self.Entropy.open_server_repository(read_only = True, 646 no_upload = True, repo = repo) 647 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 648 if idpackage == -1: 649 self.Entropy.updateProgress( 650 "[repo:%s|%s|#%s] %s: %s %s" % ( 651 brown(repo), 652 darkgreen(crippled_uri), 653 brown(tries), 654 blue(_("package")), 655 darkgreen(pkgfile), 656 blue(_("is not listed in the repository !")), 657 ), 658 importance = 1, 659 type = "error", 660 header = darkred(" !!! ") 661 ) 662 ftp.close() 663 return False 664 665 storedmd5 = dbconn.retrieveDigest(idpackage) 666 self.Entropy.updateProgress( 667 "[repo:%s|%s|#%s] %s: %s" % ( 668 brown(repo), 669 darkgreen(crippled_uri), 670 brown(tries), 671 blue(_("verifying checksum of package")), 672 darkgreen(pkgfile), 673 ), 674 importance = 1, 675 type = "info", 676 header = darkgreen(" * "), 677 back = True 678 ) 679 680 pkg_path = os.path.join(download_path, pkgfile) 681 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 682 if md5check: 683 self.Entropy.updateProgress( 684 "[repo:%s|%s|#%s] %s: %s %s" % ( 685 brown(repo), 686 darkgreen(crippled_uri), 687 brown(tries), 688 blue(_("package")), 689 darkgreen(pkgfile), 690 blue(_("downloaded successfully")), 691 ), 692 importance = 1, 693 type = "info", 694 header = darkgreen(" * ") 695 ) 696 return True 697 else: 698 self.Entropy.updateProgress( 699 "[repo:%s|%s|#%s] %s: %s %s" % ( 700 brown(repo), 701 darkgreen(crippled_uri), 702 brown(tries), 703 blue(_("package")), 704 darkgreen(pkgfile), 705 blue(_("checksum does not match. re-downloading...")), 706 ), 707 importance = 1, 708 type = "warning", 709 header = darkred(" * ") 710 ) 711 if os.path.isfile(pkg_path): 712 os.remove(pkg_path) 713 714 continue 715 716 # if we get here it means the files hasn't been downloaded properly 717 self.Entropy.updateProgress( 718 "[repo:%s|%s|#%s] %s: %s %s" % ( 719 brown(repo), 720 darkgreen(crippled_uri), 721 brown(tries), 722 blue(_("package")), 723 darkgreen(pkgfile), 724 blue(_("seems broken. Consider to re-package it. Giving up!")), 725 ), 726 importance = 1, 727 type = "error", 728 header = darkred(" !!! ") 729 ) 730 return False

731 732

733 - def get_remote_databases_status(self, repo = None, mirrors = None):

734 735 if repo == None: 736 repo = self.Entropy.default_repository 737 738 if not mirrors: 739 mirrors = self.Entropy.get_remote_mirrors(repo) 740 741 data = [] 742 for uri in mirrors: 743 744 # let it raise an exception if connection is impossible 745 ftp = FtpInterface(uri, self.Entropy) 746 try: 747 my_path = os.path.join( 748 self.Entropy.get_remote_database_relative_path(repo), 749 self.SystemSettings['repositories']['branch']) 750 ftp.set_cwd(my_path, dodir = True) 751 except ftp.ftplib.error_perm: 752 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 753 self.Entropy.updateProgress( 754 "[repo:%s|%s] %s !" % ( 755 brown(repo), 756 darkgreen(crippled_uri), 757 blue(_("mirror has invalid directory structure")), 758 ), 759 importance = 1, 760 type = "warning", 761 header = darkred(" !!! ") 762 ) 763 ftp.close() 764 continue 765 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 766 db_format = sys_set['database_file_format'] 767 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 768 if cmethod == None: 769 raise InvalidDataType("InvalidDataType: %s." % ( 770 _("Wrong database compression method passed"), 771 ) 772 ) 773 compressedfile = etpConst[cmethod[2]] 774 775 revision = 0 776 rc1 = ftp.is_file_available(compressedfile) 777 revfilename = os.path.basename( 778 self.Entropy.get_local_database_revision_file(repo)) 779 rc2 = ftp.is_file_available(revfilename) 780 if rc1 and rc2: 781 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 782 revfilename) 783 dlcount = 5 784 while dlcount: 785 dled = ftp.download_file(revfilename, 786 etpConst['packagestmpdir'], True) 787 if dled: 788 break 789 dlcount -= 1 790 791 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 792 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 793 f_rev = open(revision_localtmppath,"r") 794 try: 795 revision = int(f_rev.readline().strip()) 796 except ValueError: 797 mytxt = _("mirror hasn't valid database revision file") 798 self.Entropy.updateProgress( 799 "[repo:%s|%s] %s: %s" % ( 800 brown(repo), 801 darkgreen(crippled_uri), 802 blue(mytxt), 803 bold(revision), 804 ), 805 importance = 1, 806 type = "error", 807 header = darkred(" !!! ") 808 ) 809 revision = 0 810 f_rev.close() 811 elif dlcount == 0: 812 self.Entropy.updateProgress( 813 "[repo:%s|%s] %s: %s" % ( 814 brown(repo), 815 darkgreen(crippled_uri), 816 blue(_("unable to download repository revision")), 817 bold(revision), 818 ), 819 importance = 1, 820 type = "error", 821 header = darkred(" !!! ") 822 ) 823 revision = 0 824 else: 825 self.Entropy.updateProgress( 826 "[repo:%s|%s] %s: %s" % ( 827 brown(repo), 828 darkgreen(crippled_uri), 829 blue(_("mirror doesn't have valid revision file")), 830 bold(revision), 831 ), 832 importance = 1, 833 type = "error", 834 header = darkred(" !!! ") 835 ) 836 revision = 0 837 if os.path.isfile(revision_localtmppath): 838 os.remove(revision_localtmppath) 839 840 info = [uri, revision] 841 data.append(info) 842 ftp.close() 843 844 return data

845

846 - def is_local_database_locked(self, repo = None):

847 local_repo = repo 848 if local_repo == None: 849 local_repo = self.Entropy.default_repository 850 lock_file = self.get_database_lockfile(local_repo) 851 return os.path.isfile(lock_file)

852

853 - def get_mirrors_lock(self, repo = None):

854 855 dbstatus = [] 856 for uri in self.Entropy.get_remote_mirrors(repo): 857 data = [uri, False, False] 858 ftp = FtpInterface(uri, self.Entropy) 859 try: 860 my_path = os.path.join( 861 self.Entropy.get_remote_database_relative_path(repo), 862 self.SystemSettings['repositories']['branch']) 863 ftp.set_cwd(my_path) 864 except ftp.ftplib.error_perm: 865 ftp.close() 866 continue 867 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 868 # upload locked 869 data[1] = True 870 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 871 # download locked 872 data[2] = True 873 ftp.close() 874 dbstatus.append(data) 875 return dbstatus

876

877 - def download_notice_board(self, repo = None):

878 879 if repo == None: 880 repo = self.Entropy.default_repository 881 mirrors = self.Entropy.get_remote_mirrors(repo) 882 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 883 mytmpdir = self.entropyTools.get_random_temp_file() 884 os.makedirs(mytmpdir) 885 886 self.Entropy.updateProgress( 887 "[repo:%s] %s %s" % ( 888 brown(repo), 889 blue(_("downloading notice board from mirrors to")), 890 red(rss_path), 891 ), 892 importance = 1, 893 type = "info", 894 header = blue(" @@ ") 895 ) 896 897 downloaded = False 898 for uri in mirrors: 899 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 900 downloader = self.FtpServerHandler( 901 FtpInterface, self.Entropy, [uri], 902 [rss_path], download = True, 903 local_basedir = mytmpdir, critical_files = [rss_path], 904 repo = repo 905 ) 906 errors, m_fine_uris, m_broken_uris = downloader.go() 907 if not errors: 908 self.Entropy.updateProgress( 909 "[repo:%s] %s: %s" % ( 910 brown(repo), 911 blue(_("notice board downloaded successfully from")), 912 red(crippled_uri), 913 ), 914 importance = 1, 915 type = "info", 916 header = blue(" @@ ") 917 ) 918 downloaded = True 919 break 920 921 if downloaded: 922 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 923 rss_path) 924 925 return downloaded

926

927 - def upload_notice_board(self, repo = None):

928 929 if repo == None: 930 repo = self.Entropy.default_repository 931 mirrors = self.Entropy.get_remote_mirrors(repo) 932 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 933 934 self.Entropy.updateProgress( 935 "[repo:%s] %s %s" % ( 936 brown(repo), 937 blue(_("uploading notice board from")), 938 red(rss_path), 939 ), 940 importance = 1, 941 type = "info", 942 header = blue(" @@ ") 943 ) 944 945 uploader = self.FtpServerHandler( 946 FtpInterface, 947 self.Entropy, 948 mirrors, 949 [rss_path], 950 critical_files = [rss_path], 951 repo = repo 952 ) 953 errors, m_fine_uris, m_broken_uris = uploader.go() 954 if errors: 955 m_broken_uris = sorted(m_broken_uris) 956 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 957 for x in m_broken_uris] 958 self.Entropy.updateProgress( 959 "[repo:%s] %s %s" % ( 960 brown(repo), 961 blue(_("notice board upload failed on")), 962 red(', '.join(m_broken_uris)), 963 ), 964 importance = 1, 965 type = "info", 966 header = blue(" @@ ") 967 ) 968 return False 969 self.Entropy.updateProgress( 970 "[repo:%s] %s" % ( 971 brown(repo), 972 blue(_("notice board upload success")), 973 ), 974 importance = 1, 975 type = "info", 976 header = blue(" @@ ") 977 ) 978 return True

979 980

981 - def update_notice_board(self, title, notice_text, link = None, repo = None):

982 983 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 984 rss_description = "Inform about important distribution activities." 985 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 986 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 987 if not link: 988 link = srv_set['rss']['website_url'] 989 990 self.download_notice_board(repo) 991 rss_main = RSS(rss_path, rss_title, rss_description, 992 maxentries = 20) 993 rss_main.add_item(title, link, description = notice_text) 994 rss_main.write_changes() 995 status = self.upload_notice_board(repo) 996 return status

997

998 - def read_notice_board(self, do_download = True, repo = None):

999 1000 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1001 if do_download: 1002 self.download_notice_board(repo) 1003 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1004 return None 1005 rss_main = RSS(rss_path, '', '') 1006 return rss_main.get_entries()

1007

1008 - def remove_from_notice_board(self, identifier, repo = None):

1009 1010 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1011 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 1012 rss_description = "Inform about important distribution activities." 1013 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1014 return 0 1015 rss_main = RSS(rss_path, rss_title, rss_description) 1016 data = rss_main.remove_entry(identifier) 1017 rss_main.write_changes() 1018 return data

1019

1020 - def update_rss_feed(self, repo = None):

1021 1022 product = self.SystemSettings['repositories']['product'] 1023 #db_dir = self.Entropy.get_local_database_dir(repo) 1024 rss_path = self.Entropy.get_local_database_rss_file(repo) 1025 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 1026 rss_dump_name = etpConst['rss-dump-name'] 1027 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 1028 1029 rss_title = u"%s Online Repository Status" % ( 1030 self.SystemSettings['system']['name'],) 1031 rss_description = \ 1032 u"Keep you updated on what's going on in the %s Repository." % ( 1033 self.SystemSettings['system']['name'],) 1034 1035 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1036 1037 rss_main = RSS(rss_path, rss_title, rss_description, 1038 maxentries = srv_set['rss']['max_entries']) 1039 # load dump 1040 db_actions = self.Cacher.pop(rss_dump_name) 1041 if db_actions: 1042 try: 1043 f_rev = open(db_revision_path) 1044 revision = f_rev.readline().strip() 1045 f_rev.close() 1046 except (IOError, OSError): 1047 revision = u"N/A" 1048 commitmessage = u'' 1049 if self.Entropy.rssMessages['commitmessage']: 1050 commitmessage = u' :: ' + \ 1051 self.Entropy.rssMessages['commitmessage'] 1052 1053 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 1054 product[0].upper() + product[1:] + " " + \ 1055 self.SystemSettings['repositories']['branch'] + \ 1056 " :: Revision: " + revision + commitmessage 1057 1058 link = srv_set['rss']['base_url'] 1059 # create description 1060 added_items = db_actions.get("added") 1061 1062 if added_items: 1063 for atom in sorted(added_items): 1064 mylink = link + "?search=" + atom.split("~")[0] + \ 1065 "&arch=" + etpConst['currentarch'] + "&product="+product 1066 description = atom + ": " + added_items[atom]['description'] 1067 rss_main.add_item(title = "Added/Updated" + title, 1068 link = mylink, description = description) 1069 removed_items = db_actions.get("removed") 1070 1071 if removed_items: 1072 for atom in sorted(removed_items): 1073 description = atom + ": " + \ 1074 removed_items[atom]['description'] 1075 rss_main.add_item(title = "Removed" + title, link = link, 1076 description = description) 1077 1078 light_items = db_actions.get('light') 1079 if light_items: 1080 rss_light = RSS(rss_light_path, rss_title, rss_description, 1081 maxentries = srv_set['rss']['light_max_entries']) 1082 for atom in sorted(light_items): 1083 mylink = link + "?search=" + atom.split("~")[0] + \ 1084 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1085 product 1086 description = light_items[atom]['description'] 1087 rss_light.add_item(title = "[" + revision + "] " + atom, 1088 link = mylink, description = description) 1089 rss_light.write_changes() 1090 1091 rss_main.write_changes() 1092 self.Entropy.rssMessages.clear() 1093 self.dumpTools.removeobj(rss_dump_name)

1094 1095

1096 - def dump_database_to_file(self, db_path, destination_path, opener, 1097 exclude_tables = None, repo = None):

1098 1099 if not exclude_tables: 1100 exclude_tables = [] 1101 1102 f_out = opener(destination_path, "wb") 1103 dbconn = self.Entropy.open_server_repository(db_path, 1104 just_reading = True, repo = repo, do_treeupdates = False) 1105 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1106 self.Entropy.close_server_database(dbconn) 1107 f_out.close()

1108

1109 - def create_file_checksum(self, file_path, checksum_path):

1110 mydigest = self.entropyTools.md5sum(file_path) 1111 f_ck = open(checksum_path, "w") 1112 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1113 f_ck.write(mystring) 1114 f_ck.flush() 1115 f_ck.close()

1116

1117 - def compress_file(self, file_path, destination_path, opener):

1118 f_out = opener(destination_path, "wb") 1119 f_in = open(file_path,"rb") 1120 data = f_in.read(8192) 1121 while data: 1122 f_out.write(data) 1123 data = f_in.read(8192) 1124 f_in.close() 1125 if hasattr(f_out,'flush'): 1126 f_out.flush() 1127 f_out.close()

1128

1129 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1130 disabled_eapis = None):

1131 1132 if disabled_eapis == None: 1133 disabled_eapis = [] 1134 1135 critical = [] 1136 extra_text_files = [] 1137 data = {} 1138 data['database_revision_file'] = \ 1139 self.Entropy.get_local_database_revision_file(repo) 1140 extra_text_files.append(data['database_revision_file']) 1141 critical.append(data['database_revision_file']) 1142 1143 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1144 if os.path.isfile(database_ts_file) or download: 1145 data['database_timestamp_file'] = database_ts_file 1146 if not download: 1147 critical.append(database_ts_file) 1148 1149 database_package_mask_file = \ 1150 self.Entropy.get_local_database_mask_file(repo) 1151 extra_text_files.append(database_package_mask_file) 1152 if os.path.isfile(database_package_mask_file) or download: 1153 data['database_package_mask_file'] = database_package_mask_file 1154 if not download: 1155 critical.append(data['database_package_mask_file']) 1156 1157 database_package_system_mask_file = \ 1158 self.Entropy.get_local_database_system_mask_file(repo) 1159 extra_text_files.append(database_package_system_mask_file) 1160 if os.path.isfile(database_package_system_mask_file) or download: 1161 data['database_package_system_mask_file'] = \ 1162 database_package_system_mask_file 1163 if not download: 1164 critical.append(data['database_package_system_mask_file']) 1165 1166 database_package_confl_tagged_file = \ 1167 self.Entropy.get_local_database_confl_tagged_file(repo) 1168 extra_text_files.append(database_package_confl_tagged_file) 1169 if os.path.isfile(database_package_confl_tagged_file) or download: 1170 data['database_package_confl_tagged_file'] = \ 1171 database_package_confl_tagged_file 1172 if not download: 1173 critical.append(data['database_package_confl_tagged_file']) 1174 1175 database_license_whitelist_file = \ 1176 self.Entropy.get_local_database_licensewhitelist_file(repo) 1177 extra_text_files.append(database_license_whitelist_file) 1178 if os.path.isfile(database_license_whitelist_file) or download: 1179 data['database_license_whitelist_file'] = \ 1180 database_license_whitelist_file 1181 if not download: 1182 critical.append(data['database_license_whitelist_file']) 1183 1184 exp_based_pkgs_removal_file = \ 1185 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1186 extra_text_files.append(exp_based_pkgs_removal_file) 1187 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1188 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1189 if not download: 1190 critical.append(data['exp_based_pkgs_removal_file']) 1191 1192 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1193 if os.path.isfile(database_rss_file) or download: 1194 data['database_rss_file'] = database_rss_file 1195 if not download: 1196 critical.append(data['database_rss_file']) 1197 database_rss_light_file = \ 1198 self.Entropy.get_local_database_rsslight_file(repo) 1199 1200 extra_text_files.append(database_rss_light_file) 1201 if os.path.isfile(database_rss_light_file) or download: 1202 data['database_rss_light_file'] = database_rss_light_file 1203 if not download: 1204 critical.append(data['database_rss_light_file']) 1205 1206 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1207 data['pkglist_file'] = pkglist_file 1208 if not download: 1209 critical.append(data['pkglist_file']) 1210 1211 # EAPI 2,3 1212 if not download: # we don't need to get the dump 1213 1214 if 3 not in disabled_eapis: 1215 1216 data['metafiles_path'] = \ 1217 self.Entropy.get_local_database_compressed_metafiles_file( 1218 repo) 1219 critical.append(data['metafiles_path']) 1220 1221 if 2 not in disabled_eapis: 1222 1223 data['dump_path_light'] = os.path.join( 1224 self.Entropy.get_local_database_dir(repo), 1225 etpConst[cmethod[5]]) 1226 critical.append(data['dump_path_light']) 1227 1228 data['dump_path_digest_light'] = os.path.join( 1229 self.Entropy.get_local_database_dir(repo), 1230 etpConst[cmethod[6]]) 1231 critical.append(data['dump_path_digest_light']) 1232 1233 # EAPI 1 1234 if 1 not in disabled_eapis: 1235 1236 data['compressed_database_path'] = os.path.join( 1237 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1238 critical.append(data['compressed_database_path']) 1239 data['compressed_database_path_light'] = os.path.join( 1240 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1241 1242 data['database_path_digest'] = os.path.join( 1243 self.Entropy.get_local_database_dir(repo), 1244 etpConst['etpdatabasehashfile'] 1245 ) 1246 critical.append(data['database_path_digest']) 1247 1248 data['compressed_database_path_digest'] = os.path.join( 1249 self.Entropy.get_local_database_dir(repo), 1250 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1251 ) 1252 critical.append(data['compressed_database_path_digest']) 1253 1254 data['compressed_database_path_digest_light'] = os.path.join( 1255 self.Entropy.get_local_database_dir(repo), 1256 etpConst[cmethod[8]] 1257 ) 1258 critical.append(data['compressed_database_path_digest_light']) 1259 1260 1261 # SSL cert file, just for reference 1262 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1263 extra_text_files.append(ssl_ca_cert) 1264 if os.path.isfile(ssl_ca_cert): 1265 data['ssl_ca_cert_file'] = ssl_ca_cert 1266 if not download: 1267 critical.append(ssl_ca_cert) 1268 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1269 extra_text_files.append(ssl_server_cert) 1270 if os.path.isfile(ssl_server_cert): 1271 data['ssl_server_cert_file'] = ssl_server_cert 1272 if not download: 1273 critical.append(ssl_server_cert) 1274 1275 # Some information regarding how packages are built 1276 spm_files = [ 1277 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1278 (etpConst['spm']['global_package_keywords'], 1279 "global_package_keywords"), 1280 (etpConst['spm']['global_package_use'], "global_package_use"), 1281 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1282 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1283 ] 1284 for myfile, myname in spm_files: 1285 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1286 data[myname] = myfile 1287 extra_text_files.append(myfile) 1288 1289 make_profile = etpConst['spm']['global_make_profile'] 1290 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1291 mytmpdir = os.path.dirname(rnd_tmp_file) 1292 mytmpfile = os.path.join(mytmpdir, 1293 etpConst['spm']['global_make_profile_link_name']) 1294 extra_text_files.append(mytmpfile) 1295 1296 if os.path.islink(make_profile): 1297 mylink = os.readlink(make_profile) 1298 f_mkp = open(mytmpfile,"w") 1299 f_mkp.write(mylink) 1300 f_mkp.flush() 1301 f_mkp.close() 1302 data['global_make_profile'] = mytmpfile 1303 1304 return data, critical, extra_text_files

1305

1306 - def _show_package_sets_messages(self, repo):

1307 1308 self.Entropy.updateProgress( 1309 "[repo:%s] %s:" % ( 1310 brown(repo), 1311 blue(_("configured package sets")), 1312 ), 1313 importance = 0, 1314 type = "info", 1315 header = darkgreen(" * ") 1316 ) 1317 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1318 if not sets_data: 1319 self.Entropy.updateProgress( 1320 "%s" % (_("None configured"),), 1321 importance = 0, 1322 type = "info", 1323 header = brown(" # ") 1324 ) 1325 return 1326 for s_repo, s_name, s_sets in sets_data: 1327 self.Entropy.updateProgress( 1328 blue("%s" % (s_name,)), 1329 importance = 0, 1330 type = "info", 1331 header = brown(" # ") 1332 )

1333

1334 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1335 1336 self.Entropy.updateProgress( 1337 "[repo:%s|%s|%s:%s] %s" % ( 1338 brown(repo), 1339 darkgreen(crippled_uri), 1340 red("EAPI"), 1341 bold("3"), 1342 blue(_("preparing uncompressed database for the upload")), 1343 ), 1344 importance = 0, 1345 type = "info", 1346 header = darkgreen(" * ") 1347 ) 1348 self.Entropy.updateProgress( 1349 "%s: %s" % (_("database path"), blue(database_path),), 1350 importance = 0, 1351 type = "info", 1352 header = brown(" # ") 1353 )

1354

1355 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1356 upload_data, cmethod, repo):

1357 1358 if repo == None: 1359 repo = self.Entropy.default_repository 1360 1361 self.Entropy.updateProgress( 1362 "[repo:%s|%s|%s:%s] %s" % ( 1363 brown(repo), 1364 darkgreen(crippled_uri), 1365 red("EAPI"), 1366 bold("2"), 1367 blue(_("creating compressed database dump + checksum")), 1368 ), 1369 importance = 0, 1370 type = "info", 1371 header = darkgreen(" * ") 1372 ) 1373 self.Entropy.updateProgress( 1374 "%s: %s" % (_("database path"), blue(database_path),), 1375 importance = 0, 1376 type = "info", 1377 header = brown(" # ") 1378 ) 1379 self.Entropy.updateProgress( 1380 "%s: %s" % ( 1381 _("dump light"), 1382 blue(upload_data['dump_path_light']), 1383 ), 1384 importance = 0, 1385 type = "info", 1386 header = brown(" # ") 1387 ) 1388 self.Entropy.updateProgress( 1389 "%s: %s" % ( 1390 _("dump light checksum"), 1391 blue(upload_data['dump_path_digest_light']), 1392 ), 1393 importance = 0, 1394 type = "info", 1395 header = brown(" # ") 1396 ) 1397 1398 self.Entropy.updateProgress( 1399 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1400 importance = 0, 1401 type = "info", 1402 header = brown(" # ") 1403 )

1404

1405 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1406 upload_data, cmethod, repo):

1407 1408 self.Entropy.updateProgress( 1409 "[repo:%s|%s|%s:%s] %s" % ( 1410 brown(repo), 1411 darkgreen(crippled_uri), 1412 red("EAPI"), 1413 bold("1"), 1414 blue(_("compressing database + checksum")), 1415 ), 1416 importance = 0, 1417 type = "info", 1418 header = darkgreen(" * "), 1419 back = True 1420 ) 1421 self.Entropy.updateProgress( 1422 "%s: %s" % (_("database path"), blue(database_path),), 1423 importance = 0, 1424 type = "info", 1425 header = brown(" # ") 1426 ) 1427 self.Entropy.updateProgress( 1428 "%s: %s" % ( 1429 _("compressed database path"), 1430 blue(upload_data['compressed_database_path']), 1431 ), 1432 importance = 0, 1433 type = "info", 1434 header = brown(" # ") 1435 ) 1436 self.Entropy.updateProgress( 1437 "%s: %s" % ( 1438 _("database checksum"), 1439 blue(upload_data['database_path_digest']), 1440 ), 1441 importance = 0, 1442 type = "info", 1443 header = brown(" # ") 1444 ) 1445 self.Entropy.updateProgress( 1446 "%s: %s" % ( 1447 _("compressed checksum"), 1448 blue(upload_data['compressed_database_path_digest']), 1449 ), 1450 importance = 0, 1451 type = "info", 1452 header = brown(" # ") 1453 ) 1454 self.Entropy.updateProgress( 1455 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1456 importance = 0, 1457 type = "info", 1458 header = brown(" # ") 1459 )

1460

1461 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1462 1463 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1464 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1465 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1466 file_list if x not in found_file_list] 1467 metafile_not_found_file = \ 1468 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1469 1470 f_meta = open(metafile_not_found_file,"w") 1471 f_meta.writelines(not_found_file_list) 1472 f_meta.flush() 1473 f_meta.close() 1474 found_file_list.append(metafile_not_found_file) 1475 if os.path.isfile(compressed_dest_path): 1476 os.remove(compressed_dest_path) 1477 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1478

1479 - def create_mirror_directories(self, ftp_connection, path_to_create):

1480 bdir = "" 1481 for mydir in path_to_create.split("/"): 1482 bdir += "/"+mydir 1483 if not ftp_connection.is_file_available(bdir): 1484 try: 1485 ftp_connection.mkdir(bdir) 1486 except Exception, err: 1487 error = unicode(err) 1488 if (error.find("550") == -1) and \ 1489 (error.find("File exist") == -1): 1490 1491 mytxt = "%s %s, %s: %s" % ( 1492 _("cannot create mirror directory"), 1493 bdir, 1494 _("error"), 1495 err, 1496 ) 1497 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1498 mytxt,)) 1499 raise

1500

1501 - def mirror_lock_check(self, uri, repo = None):

1502 1503 if repo == None: 1504 repo = self.Entropy.default_repository 1505 1506 gave_up = False 1507 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1508 try: 1509 ftp = FtpInterface(uri, self.Entropy) 1510 except ConnectionError: 1511 self.entropyTools.print_traceback() 1512 return True # gave up 1513 my_path = os.path.join( 1514 self.Entropy.get_remote_database_relative_path(repo), 1515 self.SystemSettings['repositories']['branch']) 1516 ftp.set_cwd(my_path, dodir = True) 1517 1518 lock_file = self.get_database_lockfile(repo) 1519 lock_filename = os.path.basename(lock_file) 1520 if not os.path.isfile(lock_file) and \ 1521 ftp.is_file_available(lock_filename): 1522 1523 self.Entropy.updateProgress( 1524 "[repo:%s|%s|%s] %s, %s" % ( 1525 brown(str(repo)), 1526 darkgreen(crippled_uri), 1527 red(_("locking")), 1528 darkblue(_("mirror already locked")), 1529 blue(_("waiting up to 2 minutes before giving up")), 1530 ), 1531 importance = 1, 1532 type = "warning", 1533 header = brown(" * "), 1534 back = True 1535 ) 1536 unlocked = False 1537 count = 0 1538 while count < 120: 1539 count += 1 1540 time.sleep(1) 1541 if not ftp.is_file_available(os.path.basename(lock_file)): 1542 self.Entropy.updateProgress( 1543 red("[repo:%s|%s|%s] %s !" % ( 1544 repo, 1545 crippled_uri, 1546 _("locking"), 1547 _("mirror unlocked"), 1548 ) 1549 ), 1550 importance = 1, 1551 type = "info", 1552 header = darkgreen(" * ") 1553 ) 1554 unlocked = True 1555 break 1556 if not unlocked: 1557 gave_up = True 1558 1559 ftp.close() 1560 return gave_up

1561

1562 - def shrink_database_and_close(self, repo = None):

1563 dbconn = self.Entropy.open_server_repository(read_only = False, 1564 no_upload = True, repo = repo, indexing = False, 1565 do_treeupdates = False) 1566 dbconn.dropAllIndexes() 1567 dbconn.vacuum() 1568 dbconn.vacuum() 1569 dbconn.commitChanges() 1570 self.Entropy.close_server_database(dbconn)

1571

1572 - def update_repository_timestamp(self, repo = None):

1573 if repo == None: 1574 repo = self.Entropy.default_repository 1575 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1576 current_ts = self.Entropy.get_current_timestamp() 1577 ts_f = open(ts_file,"w") 1578 ts_f.write(current_ts) 1579 ts_f.flush() 1580 ts_f.close()

1581

1582 - def sync_database_treeupdates(self, repo = None):

1583 1584 if repo == None: 1585 repo = self.Entropy.default_repository 1586 dbconn = self.Entropy.open_server_repository(read_only = False, 1587 no_upload = True, repo = repo, do_treeupdates = False) 1588 # grab treeupdates from other databases and inject 1589 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1590 server_repos = srv_set['repositories'].keys() 1591 all_actions = set() 1592 for myrepo in server_repos: 1593 1594 # avoid __default__ 1595 if myrepo == etpConst['clientserverrepoid']: 1596 continue 1597 1598 mydbc = self.Entropy.open_server_repository(just_reading = True, 1599 repo = myrepo) 1600 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1601 for data in actions: 1602 all_actions.add(data) 1603 if not actions: 1604 continue 1605 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1606 try: 1607 # clear first 1608 dbconn.removeTreeUpdatesActions(repo) 1609 dbconn.insertTreeUpdatesActions(all_actions, repo) 1610 except dbapi2.Error, err: 1611 self.entropyTools.print_traceback() 1612 mytxt = "%s, %s: %s. %s" % ( 1613 _("Troubles with treeupdates"), 1614 _("error"), 1615 err, 1616 _("Bumping old data back"), 1617 ) 1618 self.Entropy.updateProgress( 1619 mytxt, 1620 importance = 1, 1621 type = "warning" 1622 ) 1623 # restore previous data 1624 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1625 1626 dbconn.commitChanges() 1627 self.Entropy.close_server_database(dbconn)

1628

1629 - def upload_database(self, uris, lock_check = False, pretend = False, 1630 repo = None):

1631 1632 if repo == None: 1633 repo = self.Entropy.default_repository 1634 1635 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1636 if srv_set['rss']['enabled']: 1637 self.update_rss_feed(repo = repo) 1638 1639 upload_errors = False 1640 broken_uris = set() 1641 fine_uris = set() 1642 1643 disabled_eapis = sorted(srv_set['disabled_eapis']) 1644 1645 for uri in uris: 1646 1647 db_format = srv_set['database_file_format'] 1648 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1649 if cmethod == None: 1650 raise InvalidDataType("InvalidDataType: %s." % ( 1651 _("wrong database compression method passed"), 1652 ) 1653 ) 1654 1655 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1656 database_path = self.Entropy.get_local_database_file(repo) 1657 1658 if disabled_eapis: 1659 self.Entropy.updateProgress( 1660 "[repo:%s|%s|%s] %s: %s" % ( 1661 blue(repo), 1662 red(crippled_uri), 1663 darkgreen(_("upload")), 1664 darkred(_("disabled EAPI")), 1665 bold(', '.join(disabled_eapis)), 1666 ), 1667 importance = 1, 1668 type = "warning", 1669 header = darkgreen(" * ") 1670 ) 1671 1672 # create/update timestamp file 1673 self.update_repository_timestamp(repo) 1674 # create pkglist service file 1675 self.Entropy.create_repository_pkglist(repo) 1676 1677 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1678 repo = repo, disabled_eapis = disabled_eapis) 1679 1680 if lock_check: 1681 given_up = self.mirror_lock_check(uri, repo = repo) 1682 if given_up: 1683 upload_errors = True 1684 broken_uris.add(uri) 1685 continue 1686 1687 self.lock_mirrors_for_download(True, [uri], repo = repo) 1688 1689 self.Entropy.updateProgress( 1690 "[repo:%s|%s|%s] %s" % ( 1691 blue(repo), 1692 red(crippled_uri), 1693 darkgreen(_("upload")), 1694 darkgreen(_("preparing to upload database to mirror")), 1695 ), 1696 importance = 1, 1697 type = "info", 1698 header = darkgreen(" * ") 1699 ) 1700 1701 # Package Sets info 1702 self._show_package_sets_messages(repo) 1703 self.sync_database_treeupdates(repo) 1704 self.Entropy.update_database_package_sets(repo) 1705 self.Entropy.close_server_databases() 1706 1707 # backup current database to avoid re-indexing 1708 old_dbpath = self.Entropy.get_local_database_file(repo) 1709 backup_dbpath = old_dbpath + ".up_backup" 1710 copy_back = False 1711 if not pretend: 1712 try: 1713 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1714 os.remove(backup_dbpath) 1715 shutil.copy2(old_dbpath, backup_dbpath) 1716 copy_back = True 1717 except shutil.Error: 1718 copy_back = False 1719 1720 self.shrink_database_and_close(repo) 1721 1722 # EAPI 3 1723 if 3 not in disabled_eapis: 1724 self._create_metafiles_file(upload_data['metafiles_path'], 1725 text_files, repo) 1726 self._show_eapi3_upload_messages(crippled_uri, database_path, 1727 repo) 1728 1729 # EAPI 2 1730 if 2 not in disabled_eapis: 1731 self._show_eapi2_upload_messages(crippled_uri, database_path, 1732 upload_data, cmethod, repo) 1733 1734 # create compressed dump + checksum 1735 self.dump_database_to_file(database_path, 1736 upload_data['dump_path_light'], cmethod[0], 1737 exclude_tables = ["content"], repo = repo) 1738 self.create_file_checksum(upload_data['dump_path_light'], 1739 upload_data['dump_path_digest_light']) 1740 1741 1742 # EAPI 1 1743 if 1 not in disabled_eapis: 1744 self._show_eapi1_upload_messages(crippled_uri, database_path, 1745 upload_data, cmethod, repo) 1746 1747 # compress the database and create uncompressed 1748 # database checksum -- DEPRECATED 1749 self.compress_file(database_path, 1750 upload_data['compressed_database_path'], cmethod[0]) 1751 self.create_file_checksum(database_path, 1752 upload_data['database_path_digest']) 1753 1754 # create compressed database checksum 1755 self.create_file_checksum( 1756 upload_data['compressed_database_path'], 1757 upload_data['compressed_database_path_digest']) 1758 1759 # create light version of the compressed db 1760 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1761 temp_eapi1_dbfile = eapi1_dbfile+".light" 1762 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1763 # open and remove content table 1764 eapi1_tmp_dbconn = \ 1765 self.Entropy.ClientService.open_generic_database( 1766 temp_eapi1_dbfile, indexing_override = False, 1767 xcache = False) 1768 eapi1_tmp_dbconn.dropContent() 1769 eapi1_tmp_dbconn.commitChanges() 1770 eapi1_tmp_dbconn.vacuum() 1771 eapi1_tmp_dbconn.closeDB() 1772 1773 # compress 1774 self.compress_file(temp_eapi1_dbfile, 1775 upload_data['compressed_database_path_light'], cmethod[0]) 1776 # go away, we don't need you anymore 1777 os.remove(temp_eapi1_dbfile) 1778 # create compressed light database checksum 1779 self.create_file_checksum( 1780 upload_data['compressed_database_path_light'], 1781 upload_data['compressed_database_path_digest_light']) 1782 1783 if not pretend: 1784 # upload 1785 uploader = self.FtpServerHandler( 1786 FtpInterface, 1787 self.Entropy, 1788 [uri], 1789 [upload_data[x] for x in upload_data], 1790 critical_files = critical, 1791 repo = repo 1792 ) 1793 errors, m_fine_uris, m_broken_uris = uploader.go() 1794 if errors: 1795 my_broken_uris = sorted([ 1796 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1797 x[1]) for x in m_broken_uris]) 1798 self.Entropy.updateProgress( 1799 "[repo:%s|%s|%s] %s" % ( 1800 repo, 1801 crippled_uri, 1802 _("errors"), 1803 _("upload failed, not unlocking and continuing"), 1804 ), 1805 importance = 0, 1806 type = "error", 1807 header = darkred(" !!! ") 1808 ) 1809 # get reason 1810 reason = my_broken_uris[0][1] 1811 self.Entropy.updateProgress( 1812 blue("%s: %s" % (_("reason"), reason,)), 1813 importance = 0, 1814 type = "error", 1815 header = blue(" # ") 1816 ) 1817 upload_errors = True 1818 broken_uris |= m_broken_uris 1819 continue 1820 1821 # copy db back 1822 if copy_back and os.path.isfile(backup_dbpath): 1823 self.Entropy.close_server_databases() 1824 further_backup_dbpath = old_dbpath+".security_backup" 1825 if os.path.isfile(further_backup_dbpath): 1826 os.remove(further_backup_dbpath) 1827 shutil.copy2(old_dbpath, further_backup_dbpath) 1828 shutil.move(backup_dbpath, old_dbpath) 1829 1830 # unlock 1831 self.lock_mirrors_for_download(False, [uri], repo = repo) 1832 fine_uris |= m_fine_uris 1833 1834 if not fine_uris: 1835 upload_errors = True 1836 return upload_errors, broken_uris, fine_uris

1837 1838

1839 - def download_database(self, uris, lock_check = False, pretend = False, 1840 repo = None):

1841 1842 if repo == None: 1843 repo = self.Entropy.default_repository 1844 1845 download_errors = False 1846 broken_uris = set() 1847 fine_uris = set() 1848 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1849 disabled_eapis = sorted(srv_set['disabled_eapis']) 1850 1851 for uri in uris: 1852 1853 db_format = srv_set['database_file_format'] 1854 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1855 if cmethod == None: 1856 raise InvalidDataType("InvalidDataType: %s." % ( 1857 _("wrong database compression method passed"), 1858 ) 1859 ) 1860 1861 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1862 database_path = self.Entropy.get_local_database_file(repo) 1863 database_dir_path = os.path.dirname( 1864 self.Entropy.get_local_database_file(repo)) 1865 download_data, critical, text_files = self.get_files_to_sync( 1866 cmethod, download = True, 1867 repo = repo, disabled_eapis = disabled_eapis) 1868 mytmpdir = self.entropyTools.get_random_temp_file() 1869 os.makedirs(mytmpdir) 1870 1871 self.Entropy.updateProgress( 1872 "[repo:%s|%s|%s] %s" % ( 1873 brown(repo), 1874 darkgreen(crippled_uri), 1875 red(_("download")), 1876 blue(_("preparing to download database from mirror")), 1877 ), 1878 importance = 1, 1879 type = "info", 1880 header = darkgreen(" * ") 1881 ) 1882 files_to_sync = sorted(download_data.keys()) 1883 for myfile in files_to_sync: 1884 self.Entropy.updateProgress( 1885 "%s: %s" % ( 1886 blue(_("download path")), 1887 brown(unicode(download_data[myfile])), 1888 ), 1889 importance = 0, 1890 type = "info", 1891 header = brown(" # ") 1892 ) 1893 1894 if lock_check: 1895 given_up = self.mirror_lock_check(uri, repo = repo) 1896 if given_up: 1897 download_errors = True 1898 broken_uris.add(uri) 1899 continue 1900 1901 # avoid having others messing while we're downloading 1902 self.lock_mirrors(True, [uri], repo = repo) 1903 1904 if not pretend: 1905 # download 1906 downloader = self.FtpServerHandler( 1907 FtpInterface, self.Entropy, [uri], 1908 [download_data[x] for x in download_data], download = True, 1909 local_basedir = mytmpdir, critical_files = critical, 1910 repo = repo 1911 ) 1912 errors, m_fine_uris, m_broken_uris = downloader.go() 1913 if errors: 1914 my_broken_uris = sorted([ 1915 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1916 x[1]) for x in m_broken_uris]) 1917 self.Entropy.updateProgress( 1918 "[repo:%s|%s|%s] %s" % ( 1919 brown(repo), 1920 darkgreen(crippled_uri), 1921 red(_("errors")), 1922 blue(_("failed to download from mirror")), 1923 ), 1924 importance = 0, 1925 type = "error", 1926 header = darkred(" !!! ") 1927 ) 1928 # get reason 1929 reason = my_broken_uris[0][1] 1930 self.Entropy.updateProgress( 1931 blue("%s: %s" % (_("reason"), reason,)), 1932 importance = 0, 1933 type = "error", 1934 header = blue(" # ") 1935 ) 1936 download_errors = True 1937 broken_uris |= m_broken_uris 1938 self.lock_mirrors(False, [uri], repo = repo) 1939 continue 1940 1941 # all fine then, we need to move data from mytmpdir 1942 # to database_dir_path 1943 1944 # EAPI 1 -- unpack database 1945 if 1 not in disabled_eapis: 1946 compressed_db_filename = os.path.basename( 1947 download_data['compressed_database_path']) 1948 uncompressed_db_filename = os.path.basename(database_path) 1949 compressed_file = os.path.join(mytmpdir, 1950 compressed_db_filename) 1951 uncompressed_file = os.path.join(mytmpdir, 1952 uncompressed_db_filename) 1953 self.entropyTools.uncompress_file(compressed_file, 1954 uncompressed_file, cmethod[0]) 1955 1956 # now move 1957 for myfile in os.listdir(mytmpdir): 1958 fromfile = os.path.join(mytmpdir, myfile) 1959 tofile = os.path.join(database_dir_path, myfile) 1960 shutil.move(fromfile, tofile) 1961 self.Entropy.ClientService.setup_default_file_perms(tofile) 1962 1963 if os.path.isdir(mytmpdir): 1964 shutil.rmtree(mytmpdir) 1965 if os.path.isdir(mytmpdir): 1966 os.rmdir(mytmpdir) 1967 1968 1969 fine_uris.add(uri) 1970 self.lock_mirrors(False, [uri], repo = repo) 1971 1972 return download_errors, fine_uris, broken_uris

1973

1974 - def calculate_database_sync_queues(self, repo = None):

1975 1976 if repo == None: 1977 repo = self.Entropy.default_repository 1978 1979 remote_status = self.get_remote_databases_status(repo) 1980 local_revision = self.Entropy.get_local_database_revision(repo) 1981 upload_queue = [] 1982 download_latest = () 1983 1984 # all mirrors are empty ? I rule 1985 if not [x for x in remote_status if x[1]]: 1986 upload_queue = remote_status[:] 1987 else: 1988 highest_remote_revision = max([x[1] for x in remote_status]) 1989 1990 if local_revision < highest_remote_revision: 1991 for remote_item in remote_status: 1992 if remote_item[1] == highest_remote_revision: 1993 download_latest = x 1994 break 1995 1996 if download_latest: 1997 upload_queue = [x for x in remote_status if \ 1998 (x[1] < highest_remote_revision)] 1999 else: 2000 upload_queue = [x for x in remote_status if \ 2001 (x[1] < local_revision)] 2002 2003 return download_latest, upload_queue

2004

2005 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 2006 repo = None):

2007 2008 if repo == None: 2009 repo = self.Entropy.default_repository 2010 2011 while 1: 2012 2013 db_locked = False 2014 if self.is_local_database_locked(repo): 2015 db_locked = True 2016 2017 lock_data = self.get_mirrors_lock(repo) 2018 mirrors_locked = [x for x in lock_data if x[1]] 2019 2020 if not mirrors_locked and db_locked: 2021 # mirrors not locked remotely but only locally 2022 mylock_file = self.get_database_lockfile(repo) 2023 if os.access(mylock_file, os.W_OK | os.F_OK): 2024 os.remove(mylock_file) 2025 continue 2026 2027 break 2028 2029 if mirrors_locked and not db_locked: 2030 mytxt = "%s, %s %s" % ( 2031 _("Mirrors are locked, someone is working on the repository"), 2032 _("try again later"), 2033 "...", 2034 ) 2035 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 2036 2037 download_latest, upload_queue = self.calculate_database_sync_queues( 2038 repo) 2039 2040 if not download_latest and not upload_queue: 2041 self.Entropy.updateProgress( 2042 "[repo:%s|%s] %s" % ( 2043 brown(repo), 2044 red(_("sync")), # something short please 2045 blue(_("database already in sync")), 2046 ), 2047 importance = 1, 2048 type = "info", 2049 header = blue(" @@ ") 2050 ) 2051 return 0, set(), set() 2052 2053 if download_latest: 2054 download_uri = download_latest[0] 2055 download_errors, fine_uris, broken_uris = self.download_database( 2056 [download_uri], repo = repo) 2057 if download_errors: 2058 self.Entropy.updateProgress( 2059 "[repo:%s|%s] %s: %s" % ( 2060 brown(repo), 2061 red(_("sync")), 2062 blue(_("database sync failed")), 2063 red(_("download issues")), 2064 ), 2065 importance = 1, 2066 type = "error", 2067 header = darkred(" !!! ") 2068 ) 2069 return 1, fine_uris, broken_uris 2070 2071 if upload_queue and not no_upload: 2072 2073 # XXX QA checks, 2074 # please group them into entropy.qa 2075 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2076 if deps_not_found and not self.Entropy.community_repo: 2077 self.Entropy.updateProgress( 2078 "[repo:%s|%s] %s: %s" % ( 2079 brown(repo), 2080 red(_("sync")), 2081 blue(_("database sync forbidden")), 2082 red(_("dependencies_test() reported errors")), 2083 ), 2084 importance = 1, 2085 type = "error", 2086 header = darkred(" !!! ") 2087 ) 2088 return 3, set(), set() 2089 2090 problems = self.Entropy.check_config_file_updates() 2091 if problems: 2092 return 4, set(), set() 2093 2094 self.Entropy.updateProgress( 2095 "[repo:%s|%s] %s" % ( 2096 brown(repo), 2097 red(_("config files")), # something short please 2098 blue(_("no configuration files to commit. All fine.")), 2099 ), 2100 importance = 1, 2101 type = "info", 2102 header = blue(" @@ "), 2103 back = True 2104 ) 2105 2106 uris = [x[0] for x in upload_queue] 2107 errors, fine_uris, broken_uris = self.upload_database(uris, 2108 repo = repo) 2109 if errors: 2110 self.Entropy.updateProgress( 2111 "[repo:%s|%s] %s: %s" % ( 2112 brown(repo), 2113 red(_("sync")), 2114 blue(_("database sync failed")), 2115 red(_("upload issues")), 2116 ), 2117 importance = 1, 2118 type = "error", 2119 header = darkred(" !!! ") 2120 ) 2121 return 2, fine_uris, broken_uris 2122 2123 2124 self.Entropy.updateProgress( 2125 "[repo:%s|%s] %s" % ( 2126 brown(repo), 2127 red(_("sync")), 2128 blue(_("database sync completed successfully")), 2129 ), 2130 importance = 1, 2131 type = "info", 2132 header = darkgreen(" * ") 2133 ) 2134 2135 if unlock_mirrors: 2136 self.lock_mirrors(False, repo = repo) 2137 return 0, set(), set()

2138 2139

2140 - def calculate_local_upload_files(self, branch, repo = None):

2141 upload_files = 0 2142 upload_packages = set() 2143 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2144 branch) 2145 2146 # check if it exists 2147 if not os.path.isdir(upload_dir): 2148 return upload_files, upload_packages 2149 2150 pkg_ext = etpConst['packagesext'] 2151 pkg_md5_ext = etpConst['packagesmd5fileext'] 2152 for package in os.listdir(upload_dir): 2153 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2154 upload_packages.add(package) 2155 if package.endswith(pkg_ext): 2156 upload_files += 1 2157 2158 return upload_files, upload_packages

2159

2160 - def calculate_local_package_files(self, branch, repo = None):

2161 local_files = 0 2162 local_packages = set() 2163 packages_dir = os.path.join( 2164 self.Entropy.get_local_packages_directory(repo), branch) 2165 2166 if not os.path.isdir(packages_dir): 2167 os.makedirs(packages_dir) 2168 2169 pkg_ext = etpConst['packagesext'] 2170 pkg_md5_ext = etpConst['packagesmd5fileext'] 2171 for package in os.listdir(packages_dir): 2172 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2173 local_packages.add(package) 2174 if package.endswith(pkg_ext): 2175 local_files += 1 2176 2177 return local_files, local_packages

2178 2179

2180 - def _show_local_sync_stats(self, upload_files, local_files):

2181 self.Entropy.updateProgress( 2182 "%s:" % ( 2183 blue(_("Local statistics")), 2184 ), 2185 importance = 1, 2186 type = "info", 2187 header = red(" @@ ") 2188 ) 2189 self.Entropy.updateProgress( 2190 red("%s:\t\t%s %s" % ( 2191 blue(_("upload directory")), 2192 bold(str(upload_files)), 2193 red(_("files ready")), 2194 ) 2195 ), 2196 importance = 0, 2197 type = "info", 2198 header = red(" @@ ") 2199 ) 2200 self.Entropy.updateProgress( 2201 red("%s:\t\t%s %s" % ( 2202 blue(_("packages directory")), 2203 bold(str(local_files)), 2204 red(_("files ready")), 2205 ) 2206 ), 2207 importance = 0, 2208 type = "info", 2209 header = red(" @@ ") 2210 )

2211

2212 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2213 branch):

2214 2215 # show stats 2216 for package, size in upload: 2217 package = darkgreen(os.path.basename(package)) 2218 size = blue(self.entropyTools.bytes_into_human(size)) 2219 self.Entropy.updateProgress( 2220 "[branch:%s|%s] %s [%s]" % ( 2221 brown(branch), 2222 blue(_("upload")), 2223 darkgreen(package), 2224 size, 2225 ), 2226 importance = 0, 2227 type = "info", 2228 header = red(" # ") 2229 ) 2230 for package, size in download: 2231 package = darkred(os.path.basename(package)) 2232 size = blue(self.entropyTools.bytes_into_human(size)) 2233 self.Entropy.updateProgress( 2234 "[branch:%s|%s] %s [%s]" % ( 2235 brown(branch), 2236 darkred(_("download")), 2237 blue(package), 2238 size, 2239 ), 2240 importance = 0, 2241 type = "info", 2242 header = red(" # ") 2243 ) 2244 for package, size in copy: 2245 package = darkblue(os.path.basename(package)) 2246 size = blue(self.entropyTools.bytes_into_human(size)) 2247 self.Entropy.updateProgress( 2248 "[branch:%s|%s] %s [%s]" % ( 2249 brown(branch), 2250 darkgreen(_("copy")), 2251 brown(package), 2252 size, 2253 ), 2254 importance = 0, 2255 type = "info", 2256 header = red(" # ") 2257 ) 2258 for package, size in removal: 2259 package = brown(os.path.basename(package)) 2260 size = blue(self.entropyTools.bytes_into_human(size)) 2261 self.Entropy.updateProgress( 2262 "[branch:%s|%s] %s [%s]" % ( 2263 brown(branch), 2264 red(_("remove")), 2265 red(package), 2266 size, 2267 ), 2268 importance = 0, 2269 type = "info", 2270 header = red(" # ") 2271 ) 2272 2273 self.Entropy.updateProgress( 2274 "%s:\t\t\t%s" % ( 2275 blue(_("Packages to be removed")), 2276 darkred(str(len(removal))), 2277 ), 2278 importance = 0, 2279 type = "info", 2280 header = blue(" @@ ") 2281 ) 2282 self.Entropy.updateProgress( 2283 "%s:\t\t%s" % ( 2284 darkgreen(_("Packages to be moved locally")), 2285 darkgreen(str(len(copy))), 2286 ), 2287 importance = 0, 2288 type = "info", 2289 header = blue(" @@ ") 2290 ) 2291 self.Entropy.updateProgress( 2292 "%s:\t\t\t%s" % ( 2293 bold(_("Packages to be uploaded")), 2294 bold(str(len(upload))), 2295 ), 2296 importance = 0, 2297 type = "info", 2298 header = blue(" @@ ") 2299 ) 2300 2301 self.Entropy.updateProgress( 2302 "%s:\t\t\t%s" % ( 2303 darkred(_("Total removal size")), 2304 darkred( 2305 self.entropyTools.bytes_into_human(metainfo['removal']) 2306 ), 2307 ), 2308 importance = 0, 2309 type = "info", 2310 header = blue(" @@ ") 2311 ) 2312 2313 self.Entropy.updateProgress( 2314 "%s:\t\t\t%s" % ( 2315 blue(_("Total upload size")), 2316 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2317 ), 2318 importance = 0, 2319 type = "info", 2320 header = blue(" @@ ") 2321 ) 2322 self.Entropy.updateProgress( 2323 "%s:\t\t\t%s" % ( 2324 brown(_("Total download size")), 2325 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2326 ), 2327 importance = 0, 2328 type = "info", 2329 header = blue(" @@ ") 2330 )

2331 2332

2333 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2334 repo = None):

2335 2336 remote_files = 0 2337 close_conn = False 2338 remote_packages_data = {} 2339 2340 my_path = os.path.join( 2341 self.Entropy.get_remote_packages_relative_path(repo), branch) 2342 if ftp_connection == None: 2343 close_conn = True 2344 # let it raise an exception if things go bad 2345 ftp_connection = FtpInterface(uri, self.Entropy) 2346 ftp_connection.set_cwd(my_path, dodir = True) 2347 2348 remote_packages = ftp_connection.list_dir() 2349 remote_packages_info = ftp_connection.get_raw_list() 2350 if close_conn: 2351 ftp_connection.close() 2352 2353 for tbz2 in remote_packages: 2354 if tbz2.endswith(etpConst['packagesext']): 2355 remote_files += 1 2356 2357 for remote_package in remote_packages_info: 2358 remote_packages_data[remote_package.split()[8]] = \ 2359 int(remote_package.split()[4]) 2360 2361 return remote_files, remote_packages, remote_packages_data

2362

2363 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2364 2365 if repo == None: 2366 repo = self.Entropy.default_repository 2367 2368 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2369 upload_files, upload_packages = self.calculate_local_upload_files( 2370 branch, repo) 2371 local_files, local_packages = self.calculate_local_package_files(branch, 2372 repo) 2373 self._show_local_sync_stats(upload_files, local_files) 2374 2375 self.Entropy.updateProgress( 2376 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2377 importance = 1, 2378 type = "info", 2379 header = red(" @@ ") 2380 ) 2381 remote_files, remote_packages, remote_packages_data = \ 2382 self.calculate_remote_package_files(uri, branch, repo = repo) 2383 self.Entropy.updateProgress( 2384 "%s:\t\t\t%s %s" % ( 2385 blue(_("remote packages")), 2386 bold(str(remote_files)), 2387 red(_("files stored")), 2388 ), 2389 importance = 0, 2390 type = "info", 2391 header = red(" @@ ") 2392 ) 2393 2394 mytxt = blue("%s ...") % ( 2395 _("Calculating queues"), 2396 ) 2397 self.Entropy.updateProgress( 2398 mytxt, 2399 importance = 1, 2400 type = "info", 2401 header = red(" @@ ") 2402 ) 2403 2404 upload_queue, download_queue, removal_queue, fine_queue = \ 2405 self.calculate_sync_queues(upload_packages, local_packages, 2406 remote_packages, remote_packages_data, branch, repo) 2407 return upload_queue, download_queue, removal_queue, fine_queue, \ 2408 remote_packages_data

2409

2410 - def calculate_sync_queues( 2411 self, 2412 upload_packages, 2413 local_packages, 2414 remote_packages, 2415 remote_packages_data, 2416 branch, 2417 repo = None 2418 ):

2419 2420 upload_queue = set() 2421 download_queue = set() 2422 removal_queue = set() 2423 fine_queue = set() 2424 2425 for local_package in upload_packages: 2426 if local_package in remote_packages: 2427 2428 local_filepath = os.path.join( 2429 self.Entropy.get_local_upload_directory(repo), branch, 2430 local_package) 2431 2432 local_size = self.entropyTools.get_file_size(local_filepath) 2433 remote_size = remote_packages_data.get(local_package) 2434 if remote_size == None: 2435 remote_size = 0 2436 if (local_size != remote_size): 2437 # size does not match, adding to the upload queue 2438 upload_queue.add(local_package) 2439 else: 2440 # just move from upload to packages 2441 fine_queue.add(local_package) 2442 else: 2443 # always force upload of packages in uploaddir 2444 upload_queue.add(local_package) 2445 2446 # if a package is in the packages directory but not online, 2447 # we have to upload it we have local_packages and remote_packages 2448 for local_package in local_packages: 2449 if local_package in remote_packages: 2450 local_filepath = os.path.join( 2451 self.Entropy.get_local_packages_directory(repo), branch, 2452 local_package) 2453 local_size = self.entropyTools.get_file_size(local_filepath) 2454 remote_size = remote_packages_data.get(local_package) 2455 if remote_size == None: 2456 remote_size = 0 2457 if (local_size != remote_size) and (local_size != 0): 2458 # size does not match, adding to the upload queue 2459 if local_package not in fine_queue: 2460 upload_queue.add(local_package) 2461 else: 2462 # this means that the local package does not exist 2463 # so, we need to download it 2464 upload_queue.add(local_package) 2465 2466 # Fill download_queue and removal_queue 2467 for remote_package in remote_packages: 2468 if remote_package in local_packages: 2469 local_filepath = os.path.join( 2470 self.Entropy.get_local_packages_directory(repo), branch, 2471 remote_package) 2472 local_size = self.entropyTools.get_file_size(local_filepath) 2473 remote_size = remote_packages_data.get(remote_package) 2474 if remote_size == None: 2475 remote_size = 0 2476 if (local_size != remote_size) and (local_size != 0): 2477 # size does not match, remove first 2478 # do it only if the package has not been 2479 # added to the upload_queue 2480 if remote_package not in upload_queue: 2481 # remotePackage == localPackage 2482 # just remove something that differs 2483 # from the content of the mirror 2484 removal_queue.add(remote_package) 2485 # then add to the download queue 2486 download_queue.add(remote_package) 2487 else: 2488 # this means that the local package does not exist 2489 # so, we need to download it 2490 # ignore .tmp files 2491 if not remote_package.endswith(".tmp"): 2492 download_queue.add(remote_package) 2493 2494 # Collect packages that don't exist anymore in the database 2495 # so we can filter them out from the download queue 2496 dbconn = self.Entropy.open_server_repository(just_reading = True, 2497 repo = repo) 2498 db_files = dbconn.listAllDownloads(do_sort = False, 2499 full_path = True) 2500 db_files = set([os.path.basename(x) for x in db_files if \ 2501 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2502 2503 exclude = set() 2504 for myfile in download_queue: 2505 if myfile.endswith(etpConst['packagesext']): 2506 if myfile not in db_files: 2507 exclude.add(myfile) 2508 download_queue -= exclude 2509 2510 exclude = set() 2511 for myfile in upload_queue: 2512 if myfile.endswith(etpConst['packagesext']): 2513 if myfile not in db_files: 2514 exclude.add(myfile) 2515 upload_queue -= exclude 2516 2517 exclude = set() 2518 for myfile in download_queue: 2519 if myfile in upload_queue: 2520 exclude.add(myfile) 2521 download_queue -= exclude 2522 2523 return upload_queue, download_queue, removal_queue, fine_queue

2524 2525

2526 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2527 remote_packages_data, branch, repo):

2528 2529 metainfo = { 2530 'removal': 0, 2531 'download': 0, 2532 'upload': 0, 2533 } 2534 removal = [] 2535 download = [] 2536 do_copy = [] 2537 upload = [] 2538 2539 for item in removal_queue: 2540 if not item.endswith(etpConst['packagesext']): 2541 continue 2542 local_filepath = os.path.join( 2543 self.Entropy.get_local_packages_directory(repo), branch, item) 2544 size = self.entropyTools.get_file_size(local_filepath) 2545 metainfo['removal'] += size 2546 removal.append((local_filepath, size)) 2547 2548 for item in download_queue: 2549 if not item.endswith(etpConst['packagesext']): 2550 continue 2551 local_filepath = os.path.join( 2552 self.Entropy.get_local_upload_directory(repo), branch, item) 2553 if not os.path.isfile(local_filepath): 2554 size = remote_packages_data.get(item) 2555 if size == None: 2556 size = 0 2557 size = int(size) 2558 metainfo['removal'] += size 2559 download.append((local_filepath, size)) 2560 else: 2561 size = self.entropyTools.get_file_size(local_filepath) 2562 do_copy.append((local_filepath, size)) 2563 2564 for item in upload_queue: 2565 if not item.endswith(etpConst['packagesext']): 2566 continue 2567 local_filepath = os.path.join( 2568 self.Entropy.get_local_upload_directory(repo), branch, item) 2569 local_filepath_pkgs = os.path.join( 2570 self.Entropy.get_local_packages_directory(repo), branch, item) 2571 if os.path.isfile(local_filepath): 2572 size = self.entropyTools.get_file_size(local_filepath) 2573 upload.append((local_filepath, size)) 2574 else: 2575 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2576 upload.append((local_filepath_pkgs, size)) 2577 metainfo['upload'] += size 2578 2579 return upload, download, removal, do_copy, metainfo

2580 2581

2582 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2583 2584 if repo == None: 2585 repo = self.Entropy.default_repository 2586 2587 for itemdata in removal_queue: 2588 2589 remove_filename = itemdata[0] 2590 remove_filepath = os.path.join( 2591 self.Entropy.get_local_packages_directory(repo), branch, 2592 remove_filename) 2593 remove_filepath_hash = remove_filepath + \ 2594 etpConst['packagesmd5fileext'] 2595 self.Entropy.updateProgress( 2596 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2597 brown(repo), 2598 red("sync"), 2599 brown(branch), 2600 blue(_("removing package+hash")), 2601 darkgreen(remove_filename), 2602 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2603 ), 2604 importance = 0, 2605 type = "info", 2606 header = darkred(" * ") 2607 ) 2608 2609 if os.path.isfile(remove_filepath): 2610 os.remove(remove_filepath) 2611 if os.path.isfile(remove_filepath_hash): 2612 os.remove(remove_filepath_hash) 2613 2614 self.Entropy.updateProgress( 2615 "[repo:%s|%s|%s] %s" % ( 2616 brown(repo), 2617 red(_("sync")), 2618 brown(branch), 2619 blue(_("removal complete")), 2620 ), 2621 importance = 0, 2622 type = "info", 2623 header = darkred(" * ") 2624 )

2625 2626

2627 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2628 2629 if repo == None: 2630 repo = self.Entropy.default_repository 2631 2632 for itemdata in copy_queue: 2633 2634 from_file = itemdata[0] 2635 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2636 to_file = os.path.join( 2637 self.Entropy.get_local_packages_directory(repo), branch, 2638 os.path.basename(from_file)) 2639 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2640 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2641 self.Entropy.updateProgress( 2642 "[repo:%s|%s|%s] %s: %s" % ( 2643 brown(repo), 2644 red("sync"), 2645 brown(branch), 2646 blue(_("copying file+hash to repository")), 2647 darkgreen(from_file), 2648 ), 2649 importance = 0, 2650 type = "info", 2651 header = darkred(" * ") 2652 ) 2653 2654 if not os.path.isdir(os.path.dirname(to_file)): 2655 os.makedirs(os.path.dirname(to_file)) 2656 2657 shutil.copy2(from_file, to_file) 2658 if not os.path.isfile(from_file_hash): 2659 self.create_file_checksum(from_file, from_file_hash) 2660 shutil.copy2(from_file_hash, to_file_hash) 2661 2662 # clear expiration file 2663 if os.path.isfile(expiration_file): 2664 os.remove(expiration_file)

2665 2666

2667 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2668 2669 if repo == None: 2670 repo = self.Entropy.default_repository 2671 2672 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2673 myqueue = [] 2674 for itemdata in upload_queue: 2675 upload_item = itemdata[0] 2676 hash_file = upload_item + etpConst['packagesmd5fileext'] 2677 if not os.path.isfile(hash_file): 2678 self.entropyTools.create_md5_file(upload_item) 2679 myqueue.append(hash_file) 2680 myqueue.append(upload_item) 2681 2682 ftp_basedir = os.path.join( 2683 self.Entropy.get_remote_packages_relative_path(repo), branch) 2684 uploader = self.FtpServerHandler(FtpInterface, 2685 self.Entropy, [uri], myqueue, critical_files = myqueue, 2686 use_handlers = True, ftp_basedir = ftp_basedir, 2687 handlers_data = {'branch': branch }, repo = repo) 2688 errors, m_fine_uris, m_broken_uris = uploader.go() 2689 if errors: 2690 my_broken_uris = [ 2691 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2692 x in m_broken_uris] 2693 reason = my_broken_uris[0][1] 2694 self.Entropy.updateProgress( 2695 "[branch:%s] %s: %s, %s: %s" % ( 2696 brown(branch), 2697 blue(_("upload errors")), 2698 red(crippled_uri), 2699 blue(_("reason")), 2700 darkgreen(unicode(reason)), 2701 ), 2702 importance = 1, 2703 type = "error", 2704 header = darkred(" !!! ") 2705 ) 2706 return errors, m_fine_uris, m_broken_uris 2707 2708 self.Entropy.updateProgress( 2709 "[branch:%s] %s: %s" % ( 2710 brown(branch), 2711 blue(_("upload completed successfully")), 2712 red(crippled_uri), 2713 ), 2714 importance = 1, 2715 type = "info", 2716 header = blue(" @@ ") 2717 ) 2718 return errors, m_fine_uris, m_broken_uris

2719 2720

2721 - def _sync_run_download_queue(self, uri, download_queue, branch, 2722 repo = None):

2723 2724 if repo == None: 2725 repo = self.Entropy.default_repository 2726 2727 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2728 myqueue = [] 2729 for package in download_queue: 2730 hash_file = package + etpConst['packagesmd5fileext'] 2731 myqueue.append(package) 2732 myqueue.append(hash_file) 2733 2734 ftp_basedir = os.path.join( 2735 self.Entropy.get_remote_packages_relative_path(repo), branch) 2736 local_basedir = os.path.join( 2737 self.Entropy.get_local_packages_directory(repo), branch) 2738 downloader = self.FtpServerHandler( 2739 FtpInterface, self.Entropy, [uri], myqueue, 2740 critical_files = myqueue, use_handlers = True, 2741 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2742 handlers_data = {'branch': branch }, download = True, repo = repo) 2743 2744 errors, m_fine_uris, m_broken_uris = downloader.go() 2745 if errors: 2746 my_broken_uris = [ 2747 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2748 for x, y in m_broken_uris] 2749 reason = my_broken_uris[0][1] 2750 self.Entropy.updateProgress( 2751 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2752 brown(repo), 2753 red(_("sync")), 2754 brown(branch), 2755 blue(_("download errors")), 2756 darkgreen(crippled_uri), 2757 blue(_("reason")), 2758 reason, 2759 ), 2760 importance = 1, 2761 type = "error", 2762 header = darkred(" !!! ") 2763 ) 2764 return errors, m_fine_uris, m_broken_uris 2765 2766 self.Entropy.updateProgress( 2767 "[repo:%s|%s|%s] %s: %s" % ( 2768 brown(repo), 2769 red(_("sync")), 2770 brown(branch), 2771 blue(_("download completed successfully")), 2772 darkgreen(crippled_uri), 2773 ), 2774 importance = 1, 2775 type = "info", 2776 header = darkgreen(" * ") 2777 ) 2778 return errors, m_fine_uris, m_broken_uris

2779

2780 - def run_package_files_qa_checks(self, packages_list, repo = None):

2781 2782 if repo == None: 2783 repo = self.Entropy.default_repository 2784 2785 my_qa = self.Entropy.QA() 2786 2787 qa_total = len(packages_list) 2788 qa_count = 0 2789 qa_some_faulty = [] 2790 2791 for upload_package in packages_list: 2792 qa_count += 1 2793 2794 self.Entropy.updateProgress( 2795 "%s: %s" % ( 2796 purple(_("QA checking package file")), 2797 darkgreen(os.path.basename(upload_package)), 2798 ), 2799 importance = 0, 2800 type = "info", 2801 header = purple(" @@ "), 2802 back = True, 2803 count = (qa_count, qa_total,) 2804 ) 2805 2806 result = my_qa.entropy_package_checks(upload_package) 2807 if not result: 2808 # call wolfman-911 2809 qa_some_faulty.append(os.path.basename(upload_package)) 2810 2811 if qa_some_faulty: 2812 2813 for qa_faulty_pkg in qa_some_faulty: 2814 self.Entropy.updateProgress( 2815 "[repo:%s|branch:%s] %s: %s" % ( 2816 brown(repo), 2817 self.SystemSettings['repositories']['branch'], 2818 red(_("faulty package file, please fix")), 2819 blue(os.path.basename(qa_faulty_pkg)), 2820 ), 2821 importance = 1, 2822 type = "error", 2823 header = darkred(" @@ ") 2824 ) 2825 raise EntropyPackageException( 2826 'EntropyPackageException: cannot continue')

2827 2828

2829 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2830 repo = None):

2831 2832 if repo == None: 2833 repo = self.Entropy.default_repository 2834 2835 self.Entropy.updateProgress( 2836 "[repo:%s|%s] %s" % ( 2837 repo, 2838 red(_("sync")), 2839 darkgreen(_("starting packages sync")), 2840 ), 2841 importance = 1, 2842 type = "info", 2843 header = red(" @@ "), 2844 back = True 2845 ) 2846 2847 2848 successfull_mirrors = set() 2849 broken_mirrors = set() 2850 check_data = () 2851 upload_queue_qa_checked = set() 2852 mirrors_tainted = False 2853 mirror_errors = False 2854 mirrors_errors = False 2855 2856 for uri in self.Entropy.get_remote_mirrors(repo): 2857 2858 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2859 mirror_errors = False 2860 2861 self.Entropy.updateProgress( 2862 "[repo:%s|%s|branch:%s] %s: %s" % ( 2863 repo, 2864 red(_("sync")), 2865 brown(self.SystemSettings['repositories']['branch']), 2866 blue(_("packages sync")), 2867 bold(crippled_uri), 2868 ), 2869 importance = 1, 2870 type = "info", 2871 header = red(" @@ ") 2872 ) 2873 2874 try: 2875 upload_queue, download_queue, removal_queue, fine_queue, \ 2876 remote_packages_data = self.calculate_packages_to_sync(uri, 2877 self.SystemSettings['repositories']['branch'], repo) 2878 except self.socket.error, err: 2879 self.Entropy.updateProgress( 2880 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2881 repo, 2882 red(_("sync")), 2883 self.SystemSettings['repositories']['branch'], 2884 darkred(_("socket error")), 2885 err, 2886 darkred(_("on")), 2887 crippled_uri, 2888 ), 2889 importance = 1, 2890 type = "error", 2891 header = darkgreen(" * ") 2892 ) 2893 continue 2894 2895 if (not upload_queue) and (not download_queue) and \ 2896 (not removal_queue): 2897 self.Entropy.updateProgress( 2898 "[repo:%s|%s|branch:%s] %s: %s" % ( 2899 repo, 2900 red(_("sync")), 2901 self.SystemSettings['repositories']['branch'], 2902 darkgreen(_("nothing to do on")), 2903 crippled_uri, 2904 ), 2905 importance = 1, 2906 type = "info", 2907 header = darkgreen(" * ") 2908 ) 2909 successfull_mirrors.add(uri) 2910 continue 2911 2912 self.Entropy.updateProgress( 2913 "%s:" % (blue(_("Expanding queues")),), 2914 importance = 1, 2915 type = "info", 2916 header = red(" ** ") 2917 ) 2918 2919 upload, download, removal, copy, metainfo = self.expand_queues( 2920 upload_queue, 2921 download_queue, 2922 removal_queue, 2923 remote_packages_data, 2924 self.SystemSettings['repositories']['branch'], 2925 repo 2926 ) 2927 del upload_queue, download_queue, removal_queue, \ 2928 remote_packages_data 2929 self._show_sync_queues(upload, download, removal, copy, metainfo, 2930 self.SystemSettings['repositories']['branch']) 2931 2932 if not len(upload)+len(download)+len(removal)+len(copy): 2933 2934 self.Entropy.updateProgress( 2935 "[repo:%s|%s|branch:%s] %s %s" % ( 2936 self.Entropy.default_repository, 2937 red(_("sync")), 2938 self.SystemSettings['repositories']['branch'], 2939 blue(_("nothing to sync for")), 2940 crippled_uri, 2941 ), 2942 importance = 1, 2943 type = "info", 2944 header = darkgreen(" @@ ") 2945 ) 2946 2947 successfull_mirrors.add(uri) 2948 continue 2949 2950 if pretend: 2951 successfull_mirrors.add(uri) 2952 continue 2953 2954 if ask: 2955 rc_sync = self.Entropy.askQuestion( 2956 _("Would you like to run the steps above ?")) 2957 if rc_sync == "No": 2958 continue 2959 2960 try: 2961 2962 # QA checks 2963 qa_package_files = [x[0] for x in upload if x[0] \ 2964 not in upload_queue_qa_checked] 2965 upload_queue_qa_checked |= set(qa_package_files) 2966 2967 self.run_package_files_qa_checks(qa_package_files, repo = repo) 2968 2969 if removal: 2970 self._sync_run_removal_queue(removal, 2971 self.SystemSettings['repositories']['branch'], repo) 2972 2973 if copy: 2974 self._sync_run_copy_queue(copy, 2975 self.SystemSettings['repositories']['branch'], repo) 2976 2977 if upload or download: 2978 mirrors_tainted = True 2979 2980 if upload: 2981 d_errors, m_fine_uris, \ 2982 m_broken_uris = self._sync_run_upload_queue( 2983 uri, upload, 2984 self.SystemSettings['repositories']['branch'], repo) 2985 2986 if d_errors: 2987 mirror_errors = True 2988 2989 if download: 2990 my_downlist = [x[0] for x in download] 2991 d_errors, m_fine_uris, \ 2992 m_broken_uris = self._sync_run_download_queue( 2993 uri, my_downlist, 2994 self.SystemSettings['repositories']['branch'], repo) 2995 2996 if d_errors: 2997 mirror_errors = True 2998 if not mirror_errors: 2999 successfull_mirrors.add(uri) 3000 else: 3001 mirrors_errors = True 3002 3003 except KeyboardInterrupt: 3004 self.Entropy.updateProgress( 3005 "[repo:%s|%s|branch:%s] %s" % ( 3006 repo, 3007 red(_("sync")), 3008 self.SystemSettings['repositories']['branch'], 3009 darkgreen(_("keyboard interrupt !")), 3010 ), 3011 importance = 1, 3012 type = "info", 3013 header = darkgreen(" * ") 3014 ) 3015 continue 3016 3017 except EntropyPackageException: 3018 3019 mirrors_errors = True 3020 broken_mirrors.add(uri) 3021 successfull_mirrors.clear() 3022 # so that people will realize this is a very bad thing 3023 self.Entropy.updateProgress( 3024 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3025 repo, 3026 red(_("sync")), 3027 self.SystemSettings['repositories']['branch'], 3028 darkred(_("you must package them again")), 3029 Exception, 3030 _("error"), 3031 err, 3032 ), 3033 importance = 1, 3034 type = "error", 3035 header = darkred(" !!! ") 3036 ) 3037 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3038 broken_mirrors, check_data 3039 3040 except Exception, err: 3041 3042 self.entropyTools.print_traceback() 3043 mirrors_errors = True 3044 broken_mirrors.add(uri) 3045 self.Entropy.updateProgress( 3046 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3047 repo, 3048 red(_("sync")), 3049 self.SystemSettings['repositories']['branch'], 3050 darkred(_("exception caught")), 3051 Exception, 3052 _("error"), 3053 err, 3054 ), 3055 importance = 1, 3056 type = "error", 3057 header = darkred(" !!! ") 3058 ) 3059 3060 exc_txt = self.Entropy.entropyTools.print_exception( 3061 returndata = True) 3062 for line in exc_txt: 3063 self.Entropy.updateProgress( 3064 unicode(line), 3065 importance = 1, 3066 type = "error", 3067 header = darkred(": ") 3068 ) 3069 3070 if len(successfull_mirrors) > 0: 3071 self.Entropy.updateProgress( 3072 "[repo:%s|%s|branch:%s] %s" % ( 3073 repo, 3074 red(_("sync")), 3075 self.SystemSettings['repositories']['branch'], 3076 darkred( 3077 _("at least one mirror synced properly!")), 3078 ), 3079 importance = 1, 3080 type = "error", 3081 header = darkred(" !!! ") 3082 ) 3083 continue 3084 3085 # if at least one server has been synced successfully, move files 3086 if (len(successfull_mirrors) > 0) and not pretend: 3087 self.remove_expiration_files( 3088 self.SystemSettings['repositories']['branch'], repo) 3089 3090 if packages_check: 3091 check_data = self.Entropy.verify_local_packages([], ask = ask, 3092 repo = repo) 3093 3094 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3095 broken_mirrors, check_data

3096

3097 - def remove_expiration_files(self, branch, repo = None):

3098 3099 if repo == None: 3100 repo = self.Entropy.default_repository 3101 3102 branch_dir = os.path.join( 3103 self.Entropy.get_local_upload_directory(repo), branch) 3104 3105 # check if it exists 3106 if not os.path.isdir(branch_dir): 3107 return None 3108 3109 branchcontent = os.listdir(branch_dir) 3110 for xfile in branchcontent: 3111 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3112 branch, xfile) 3113 destdir = os.path.join( 3114 self.Entropy.get_local_packages_directory(repo), branch) 3115 if not os.path.isdir(destdir): 3116 os.makedirs(destdir) 3117 dest = os.path.join(destdir, xfile) 3118 shutil.move(source, dest) 3119 # clear expiration file 3120 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3121 if os.path.isfile(dest_expiration): 3122 os.remove(dest_expiration)

3123 3124

3125 - def is_package_expired(self, package_file, branch, repo = None):

3126 3127 pkg_path = os.path.join( 3128 self.Entropy.get_local_packages_directory(repo), branch, 3129 package_file) 3130 pkg_path += etpConst['packagesexpirationfileext'] 3131 if not os.path.isfile(pkg_path): 3132 return False 3133 3134 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3135 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3136 days = srv_set['packages_expiration_days'] 3137 delta = int(days)*24*3600 3138 currmtime = time.time() 3139 file_delta = currmtime - mtime 3140 3141 if file_delta > delta: 3142 return True 3143 return False

3144

3145 - def create_expiration_file(self, package_file, branch, repo = None, 3146 gentle = False):

3147 3148 pkg_path = os.path.join( 3149 self.Entropy.get_local_packages_directory(repo), branch, 3150 package_file) 3151 pkg_path += etpConst['packagesexpirationfileext'] 3152 if gentle and os.path.isfile(pkg_path): 3153 return 3154 f_exp = open(pkg_path,"w") 3155 f_exp.flush() 3156 f_exp.close()

3157 3158

3159 - def collect_expiring_packages(self, branch, repo = None):

3160 3161 dbconn = self.Entropy.open_server_repository(just_reading = True, 3162 repo = repo) 3163 database_bins = dbconn.listAllDownloads(do_sort = False, 3164 full_path = True) 3165 bins_dir = os.path.join( 3166 self.Entropy.get_local_packages_directory(repo), branch) 3167 repo_bins = set() 3168 3169 if os.path.isdir(bins_dir): 3170 repo_bins = os.listdir(bins_dir) 3171 repo_bins = set([ 3172 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3173 in repo_bins if x.endswith(etpConst['packagesext'])]) 3174 repo_bins -= database_bins 3175 3176 return set([os.path.basename(x) for x in repo_bins])

3177 3178

3179 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3180 3181 if repo == None: 3182 repo = self.Entropy.default_repository 3183 3184 self.Entropy.updateProgress( 3185 "[repo:%s|%s|branch:%s] %s" % ( 3186 brown(repo), 3187 red(_("tidy")), 3188 blue(self.SystemSettings['repositories']['branch']), 3189 blue(_("collecting expired packages")), 3190 ), 3191 importance = 1, 3192 type = "info", 3193 header = red(" @@ ") 3194 ) 3195 3196 branch_data = {} 3197 errors = False 3198 branch_data['errors'] = False 3199 branch = self.SystemSettings['repositories']['branch'] 3200 3201 self.Entropy.updateProgress( 3202 "[branch:%s] %s" % ( 3203 brown(branch), 3204 blue(_("collecting expired packages in the selected branches")), 3205 ), 3206 importance = 1, 3207 type = "info", 3208 header = blue(" @@ ") 3209 ) 3210 3211 # collect removed packages 3212 expiring_packages = self.collect_expiring_packages(branch, repo) 3213 if expiring_packages: 3214 3215 # filter expired packages used by other branches 3216 # this is done for the sake of consistency 3217 # --- read packages.db.pkglist, make sure your repository 3218 # has been ported to latest Entropy 3219 3220 branch_pkglist_data = self.read_remote_file_in_branches( 3221 etpConst['etpdatabasepkglist'], repo = repo, 3222 excluded_branches = [branch]) 3223 # format data 3224 for key, val in branch_pkglist_data.items(): 3225 branch_pkglist_data[key] = val.split("\n") 3226 3227 3228 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3229 branch) 3230 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3231 expiring_packages]) 3232 3233 for other_branch in branch_pkglist_data: 3234 branch_pkglist = set(branch_pkglist_data[other_branch]) 3235 my_expiring_pkgs -= branch_pkglist 3236 3237 # fallback to normality, set new expiring packages var 3238 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3239 3240 removal = [] 3241 for package in expiring_packages: 3242 expired = self.is_package_expired(package, branch, repo) 3243 if expired: 3244 removal.append(package) 3245 else: 3246 self.create_expiration_file(package, branch, repo, 3247 gentle = True) 3248 3249 # fill returning data 3250 branch_data['removal'] = removal[:] 3251 3252 if not removal: 3253 self.Entropy.updateProgress( 3254 "[branch:%s] %s" % ( 3255 brown(branch), 3256 blue(_("nothing to remove on this branch")), 3257 ), 3258 importance = 1, 3259 type = "info", 3260 header = blue(" @@ ") 3261 ) 3262 return errors, branch_data 3263 else: 3264 self.Entropy.updateProgress( 3265 "[branch:%s] %s:" % ( 3266 brown(branch), 3267 blue(_("these are the expired packages")), 3268 ), 3269 importance = 1, 3270 type = "info", 3271 header = blue(" @@ ") 3272 ) 3273 for package in removal: 3274 self.Entropy.updateProgress( 3275 "[branch:%s] %s: %s" % ( 3276 brown(branch), 3277 blue(_("remove")), 3278 darkgreen(package), 3279 ), 3280 importance = 1, 3281 type = "info", 3282 header = brown(" # ") 3283 ) 3284 3285 if pretend: 3286 return errors, branch_data 3287 3288 if ask: 3289 rc_question = self.Entropy.askQuestion( 3290 _("Would you like to continue ?")) 3291 if rc_question == "No": 3292 return errors, branch_data 3293 3294 myqueue = [] 3295 for package in removal: 3296 myqueue.append(package+etpConst['packagesmd5fileext']) 3297 myqueue.append(package) 3298 3299 ftp_basedir = os.path.join( 3300 self.Entropy.get_remote_packages_relative_path(repo), branch) 3301 for uri in self.Entropy.get_remote_mirrors(repo): 3302 3303 self.Entropy.updateProgress( 3304 "[branch:%s] %s..." % ( 3305 brown(branch), 3306 blue(_("removing packages remotely")), 3307 ), 3308 importance = 1, 3309 type = "info", 3310 header = blue(" @@ ") 3311 ) 3312 3313 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3314 destroyer = self.FtpServerHandler( 3315 FtpInterface, 3316 self.Entropy, 3317 [uri], 3318 myqueue, 3319 critical_files = [], 3320 ftp_basedir = ftp_basedir, 3321 remove = True, 3322 repo = repo 3323 ) 3324 errors, m_fine_uris, m_broken_uris = destroyer.go() 3325 if errors: 3326 my_broken_uris = [ 3327 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3328 for x in m_broken_uris] 3329 3330 reason = my_broken_uris[0][1] 3331 self.Entropy.updateProgress( 3332 "[branch:%s] %s: %s, %s: %s" % ( 3333 brown(branch), 3334 blue(_("remove errors")), 3335 red(crippled_uri), 3336 blue(_("reason")), 3337 reason, 3338 ), 3339 importance = 1, 3340 type = "warning", 3341 header = brown(" !!! ") 3342 ) 3343 branch_data['errors'] = True 3344 errors = True 3345 3346 self.Entropy.updateProgress( 3347 "[branch:%s] %s..." % ( 3348 brown(branch), 3349 blue(_("removing packages locally")), 3350 ), 3351 importance = 1, 3352 type = "info", 3353 header = blue(" @@ ") 3354 ) 3355 3356 branch_data['removed'] = set() 3357 for package in removal: 3358 package_path = os.path.join( 3359 self.Entropy.get_local_packages_directory(repo), 3360 branch, package) 3361 package_path_hash = package_path + \ 3362 etpConst['packagesmd5fileext'] 3363 package_path_expired = package_path + \ 3364 etpConst['packagesexpirationfileext'] 3365 3366 my_rm_list = (package_path_hash, package_path, 3367 package_path_expired) 3368 for myfile in my_rm_list: 3369 if os.path.isfile(myfile): 3370 self.Entropy.updateProgress( 3371 "[branch:%s] %s: %s" % ( 3372 brown(branch), 3373 blue(_("removing")), 3374 darkgreen(myfile), 3375 ), 3376 importance = 1, 3377 type = "info", 3378 header = brown(" @@ ") 3379 ) 3380 os.remove(myfile) 3381 branch_data['removed'].add(myfile) 3382 3383 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors