entropy.server.interfaces.mirrors

39 40 import socket 41 import entropy.dump as dumpTools 42 import entropy.tools as entropyTools

43 - def __init__(self, ServerInstance, repo = None):

44 45 from entropy.server.interfaces.main import Server as MainServer 46 if not isinstance(ServerInstance, MainServer): 47 mytxt = _("entropy.server.interfaces.main.Server needed") 48 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 49 50 self.Entropy = ServerInstance 51 from entropy.transceivers import FtpServerHandler 52 self.FtpServerHandler = FtpServerHandler 53 from entropy.cache import EntropyCacher 54 self.Cacher = EntropyCacher() 55 self.sys_settings_plugin_id = \ 56 etpConst['system_settings_plugins_ids']['server_plugin'] 57 self.SystemSettings = self.Entropy.SystemSettings 58 59 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 60 self.Entropy.updateProgress( 61 mytxt, 62 importance = 2, 63 type = "info", 64 header = red(" @@ ") 65 ) 66 mytxt = _("mirror") 67 for mirror in self.Entropy.get_remote_mirrors(repo): 68 mirror = self.entropyTools.hide_ftp_password(mirror) 69 self.Entropy.updateProgress( 70 blue("%s: %s") % (mytxt, darkgreen(mirror),), 71 importance = 0, 72 type = "info", 73 header = brown(" # ") 74 )

75

76 - def get_remote_branches(self, repo = None):

77 """ 78 Returns a list of remotely available branches for the provided 79 repository identifier. 80 81 @keyword repo: repository identifier 82 @type repo: string 83 @return: list of valid, available remote branches 84 @rtype: set 85 """ 86 87 if repo == None: 88 repo = self.Entropy.default_repository 89 90 remote_branches = set() 91 # this is used to validate Entropy repository path 92 ts_file = etpConst['etpdatabasetimestampfile'] 93 mirrors = self.Entropy.get_remote_mirrors(repo) 94 for uri in mirrors: 95 96 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 97 98 self.Entropy.updateProgress( 99 "[repo:%s] %s: %s" % ( 100 brown(repo), 101 blue(_("listing branches in mirror")), 102 darkgreen(crippled_uri), 103 ), 104 importance = 1, 105 type = "info", 106 header = brown(" @@ ") 107 ) 108 109 ftp = FtpInterface(uri, self.Entropy, verbose = False) 110 try: 111 112 branches_path = self.Entropy.get_remote_database_relative_path( 113 repo) 114 try: 115 ftp.set_cwd(branches_path) 116 except FtpError: 117 continue # no branches in this mirror 118 119 branches = ftp.list_dir() 120 for branch in branches: 121 mypath = os.path.join("/", branches_path, branch) 122 try: 123 ftp.set_cwd(mypath) 124 except FtpError: 125 # not a dir 126 continue 127 if ftp.is_file_available(ts_file): 128 remote_branches.add(branch) 129 130 finally: 131 ftp.close() 132 133 return remote_branches

134

135 - def read_remote_file_in_branches(self, filename, repo = None, 136 excluded_branches = None):

137 """ 138 Reads a file remotely located in all the available branches. 139 140 @param filename: name of the file that should be located inside 141 repository database directory 142 @type filename: string 143 @keyword repo: repository identifier 144 @type repo: string 145 @keyword excluded_branches: list of branch identifiers excluded or None 146 @type excluded_branches: list or None 147 @return: dictionary with branches as key and raw file content as value: 148 {'4': 'abcd\n', '5': 'defg\n'} 149 @rtype: dict 150 """ 151 if repo == None: 152 repo = self.Entropy.default_repository 153 if excluded_branches == None: 154 excluded_branches = [] 155 156 branch_data = {} 157 mirrors = self.Entropy.get_remote_mirrors(repo) 158 for uri in mirrors: 159 160 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 161 162 self.Entropy.updateProgress( 163 "[repo:%s] %s: %s => %s" % ( 164 brown(repo), 165 blue(_("looking for file in mirror")), 166 darkgreen(crippled_uri), 167 filename, 168 ), 169 importance = 1, 170 type = "info", 171 header = brown(" @@ ") 172 ) 173 174 ftp = FtpInterface(uri, self.Entropy, verbose = False) 175 try: 176 177 branches_path = self.Entropy.get_remote_database_relative_path( 178 repo) 179 try: 180 ftp.set_cwd(branches_path) 181 except FtpError: 182 continue # no branches in this mirror 183 184 branches = ftp.list_dir() 185 for branch in branches: 186 187 # is branch excluded ? 188 if branch in excluded_branches: 189 continue 190 191 if branch_data.get(branch) != None: 192 # already read 193 continue 194 195 mypath = os.path.join("/", branches_path, branch) 196 try: 197 ftp.set_cwd(mypath) 198 except FtpError: 199 # not a dir 200 continue 201 202 if not ftp.is_file_available(filename): 203 # nothing to do, no file here 204 continue 205 206 tmp_dir = tempfile.mkdtemp() 207 tries = 4 208 success = False 209 while tries: 210 downloaded = ftp.download_file(filename, tmp_dir) 211 if not downloaded: 212 tries -= 1 213 continue # argh! 214 success = True 215 break 216 217 down_path = os.path.join(tmp_dir, filename) 218 if success and os.access(down_path, os.F_OK): 219 down_f = open(down_path) 220 branch_data[branch] = down_f.read() 221 down_f.close() 222 shutil.rmtree(tmp_dir) 223 224 finally: 225 ftp.close() 226 227 return branch_data

228

229 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

230 231 if repo == None: 232 repo = self.Entropy.default_repository 233 234 if not mirrors: 235 mirrors = self.Entropy.get_remote_mirrors(repo) 236 237 issues = False 238 for uri in mirrors: 239 240 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 241 242 lock_text = _("unlocking") 243 if lock: 244 lock_text = _("locking") 245 self.Entropy.updateProgress( 246 "[repo:%s|%s] %s %s" % ( 247 brown(repo), 248 darkgreen(crippled_uri), 249 bold(lock_text), 250 blue("%s...") % (_("mirror"),), 251 ), 252 importance = 1, 253 type = "info", 254 header = brown(" * "), 255 back = True 256 ) 257 258 try: 259 ftp = FtpInterface(uri, self.Entropy) 260 except ConnectionError: 261 self.entropyTools.print_traceback() 262 return True # issues 263 my_path = os.path.join( 264 self.Entropy.get_remote_database_relative_path(repo), 265 self.SystemSettings['repositories']['branch']) 266 ftp.set_cwd(my_path, dodir = True) 267 268 lock_file = etpConst['etpdatabaselockfile'] 269 if lock and ftp.is_file_available(lock_file): 270 self.Entropy.updateProgress( 271 "[repo:%s|%s] %s" % ( 272 brown(repo), 273 darkgreen(crippled_uri), 274 blue(_("mirror already locked")), 275 ), 276 importance = 1, 277 type = "info", 278 header = darkgreen(" * ") 279 ) 280 ftp.close() 281 continue 282 elif not lock and not ftp.is_file_available(lock_file): 283 self.Entropy.updateProgress( 284 "[repo:%s|%s] %s" % ( 285 brown(repo), 286 darkgreen(crippled_uri), 287 blue(_("mirror already unlocked")), 288 ), 289 importance = 1, 290 type = "info", 291 header = darkgreen(" * ") 292 ) 293 ftp.close() 294 continue 295 296 if lock: 297 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 298 else: 299 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 300 ftp.close() 301 if not rc_lock: 302 issues = True 303 304 if not issues: 305 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 306 if os.path.isfile(db_taint_file): 307 os.remove(db_taint_file) 308 309 return issues

310 311

312 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 313 repo = None):

314 # this functions makes entropy clients to not download anything 315 # from the chosen mirrors. it is used to avoid clients to 316 # download databases while we're uploading a new one 317 318 if mirrors == None: 319 mirrors = [] 320 321 if repo == None: 322 repo = self.Entropy.default_repository 323 324 if not mirrors: 325 mirrors = self.Entropy.get_remote_mirrors(repo) 326 327 issues = False 328 for uri in mirrors: 329 330 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 331 332 lock_text = _("unlocking") 333 if lock: 334 lock_text = _("locking") 335 self.Entropy.updateProgress( 336 "[repo:%s|%s] %s %s..." % ( 337 blue(repo), 338 red(crippled_uri), 339 bold(lock_text), 340 blue(_("mirror for download")), 341 ), 342 importance = 1, 343 type = "info", 344 header = red(" @@ "), 345 back = True 346 ) 347 348 try: 349 ftp = FtpInterface(uri, self.Entropy) 350 except ConnectionError: 351 self.entropyTools.print_traceback() 352 return True # issues 353 my_path = os.path.join( 354 self.Entropy.get_remote_database_relative_path(repo), 355 self.SystemSettings['repositories']['branch']) 356 ftp.set_cwd(my_path, dodir = True) 357 358 lock_file = etpConst['etpdatabasedownloadlockfile'] 359 360 if lock and ftp.is_file_available(lock_file): 361 self.Entropy.updateProgress( 362 "[repo:%s|%s] %s" % ( 363 blue(repo), 364 red(crippled_uri), 365 blue(_("mirror already locked for download")), 366 ), 367 importance = 1, 368 type = "info", 369 header = red(" @@ ") 370 ) 371 ftp.close() 372 continue 373 374 elif not lock and not ftp.is_file_available(lock_file): 375 self.Entropy.updateProgress( 376 "[repo:%s|%s] %s" % ( 377 blue(repo), 378 red(crippled_uri), 379 blue(_("mirror already unlocked for download")), 380 ), 381 importance = 1, 382 type = "info", 383 header = red(" @@ ") 384 ) 385 ftp.close() 386 continue 387 388 if lock: 389 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 390 repo = repo) 391 else: 392 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 393 repo = repo) 394 ftp.close() 395 if not rc_lock: 396 issues = True 397 398 return issues

399

400 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 401 repo = None):

402 403 if repo == None: 404 repo = self.Entropy.default_repository 405 406 my_path = os.path.join( 407 self.Entropy.get_remote_database_relative_path(repo), 408 self.SystemSettings['repositories']['branch']) 409 if not ftp_connection: 410 try: 411 ftp_connection = FtpInterface(uri, self.Entropy) 412 except ConnectionError: 413 self.entropyTools.print_traceback() 414 return False # issues 415 ftp_connection.set_cwd(my_path, dodir = True) 416 else: 417 mycwd = ftp_connection.get_cwd() 418 if mycwd != my_path: 419 ftp_connection.set_basedir() 420 ftp_connection.set_cwd(my_path, dodir = True) 421 422 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 423 lock_string = u'' 424 425 if dblock: 426 self.create_local_database_lockfile(repo) 427 lock_file = self.get_database_lockfile(repo) 428 else: 429 # locking/unlocking mirror1 for download 430 lock_string = _('for download') 431 self.create_local_database_download_lockfile(repo) 432 lock_file = self.get_database_download_lockfile(repo) 433 434 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 435 if rc_upload: 436 self.Entropy.updateProgress( 437 "[repo:%s|%s] %s %s" % ( 438 blue(repo), 439 red(crippled_uri), 440 blue(_("mirror successfully locked")), 441 blue(lock_string), 442 ), 443 importance = 1, 444 type = "info", 445 header = red(" @@ ") 446 ) 447 else: 448 self.Entropy.updateProgress( 449 "[repo:%s|%s] %s: %s - %s %s" % ( 450 blue(repo), 451 red(crippled_uri), 452 blue("lock error"), 453 rc_upload, 454 blue(_("mirror not locked")), 455 blue(lock_string), 456 ), 457 importance = 1, 458 type = "error", 459 header = darkred(" * ") 460 ) 461 self.remove_local_database_lockfile(repo) 462 463 return rc_upload

464 465

466 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

467 468 if repo == None: 469 repo = self.Entropy.default_repository 470 471 my_path = os.path.join( 472 self.Entropy.get_remote_database_relative_path(repo), 473 self.SystemSettings['repositories']['branch']) 474 if not ftp_connection: 475 try: 476 ftp_connection = FtpInterface(uri, self.Entropy) 477 except ConnectionError: 478 self.entropyTools.print_traceback() 479 return False # issues 480 ftp_connection.set_cwd(my_path) 481 else: 482 mycwd = ftp_connection.get_cwd() 483 if mycwd != my_path: 484 ftp_connection.set_basedir() 485 ftp_connection.set_cwd(my_path) 486 487 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 488 489 if dblock: 490 dbfile = etpConst['etpdatabaselockfile'] 491 else: 492 dbfile = etpConst['etpdatabasedownloadlockfile'] 493 rc_delete = ftp_connection.delete_file(dbfile) 494 if rc_delete: 495 self.Entropy.updateProgress( 496 "[repo:%s|%s] %s" % ( 497 blue(repo), 498 red(crippled_uri), 499 blue(_("mirror successfully unlocked")), 500 ), 501 importance = 1, 502 type = "info", 503 header = darkgreen(" * ") 504 ) 505 if dblock: 506 self.remove_local_database_lockfile(repo) 507 else: 508 self.remove_local_database_download_lockfile(repo) 509 else: 510 self.Entropy.updateProgress( 511 "[repo:%s|%s] %s: %s - %s" % ( 512 blue(repo), 513 red(crippled_uri), 514 blue(_("unlock error")), 515 rc_delete, 516 blue(_("mirror not unlocked")), 517 ), 518 importance = 1, 519 type = "error", 520 header = darkred(" * ") 521 ) 522 523 return rc_delete

524

525 - def get_database_lockfile(self, repo = None):

526 if repo == None: 527 repo = self.Entropy.default_repository 528 return os.path.join(self.Entropy.get_local_database_dir(repo), 529 etpConst['etpdatabaselockfile'])

530

531 - def get_database_download_lockfile(self, repo = None):

532 if repo == None: 533 repo = self.Entropy.default_repository 534 return os.path.join(self.Entropy.get_local_database_dir(repo), 535 etpConst['etpdatabasedownloadlockfile'])

536

537 - def create_local_database_download_lockfile(self, repo = None):

538 if repo == None: 539 repo = self.Entropy.default_repository 540 lock_file = self.get_database_download_lockfile(repo) 541 f_lock = open(lock_file, "w") 542 f_lock.write("download locked") 543 f_lock.flush() 544 f_lock.close()

545

546 - def create_local_database_lockfile(self, repo = None):

547 if repo == None: 548 repo = self.Entropy.default_repository 549 lock_file = self.get_database_lockfile(repo) 550 f_lock = open(lock_file, "w") 551 f_lock.write("database locked") 552 f_lock.flush() 553 f_lock.close()

554

555 - def remove_local_database_lockfile(self, repo = None):

556 if repo == None: 557 repo = self.Entropy.default_repository 558 lock_file = self.get_database_lockfile(repo) 559 if os.path.isfile(lock_file): 560 os.remove(lock_file)

561

562 - def remove_local_database_download_lockfile(self, repo = None):

563 if repo == None: 564 repo = self.Entropy.default_repository 565 lock_file = self.get_database_download_lockfile(repo) 566 if os.path.isfile(lock_file): 567 os.remove(lock_file)

568

569 - def download_package(self, uri, pkg_relative_path, repo = None):

570 571 if repo == None: 572 repo = self.Entropy.default_repository 573 574 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 575 576 tries = 0 577 while tries < 5: 578 tries += 1 579 580 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 581 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 582 pkgfile = os.path.basename(pkg_relative_path) 583 584 self.Entropy.updateProgress( 585 "[repo:%s|%s|#%s] %s: %s" % ( 586 brown(repo), 587 darkgreen(crippled_uri), 588 brown(tries), 589 blue(_("connecting to download package")), 590 darkgreen(pkg_to_join_path), 591 ), 592 importance = 1, 593 type = "info", 594 header = darkgreen(" * "), 595 back = True 596 ) 597 598 try: 599 ftp = FtpInterface(uri, self.Entropy) 600 except ConnectionError: 601 self.entropyTools.print_traceback() 602 return False # issues 603 dirpath = os.path.join( 604 self.Entropy.get_remote_packages_relative_path(repo), 605 pkg_to_join_dirpath) 606 ftp.set_cwd(dirpath, dodir = True) 607 608 self.Entropy.updateProgress( 609 "[repo:%s|%s|#%s] %s: %s" % ( 610 brown(repo), 611 darkgreen(crippled_uri), 612 brown(tries), 613 blue(_("downloading package")), 614 darkgreen(pkg_to_join_path), 615 ), 616 importance = 1, 617 type = "info", 618 header = darkgreen(" * ") 619 ) 620 621 download_path = os.path.join( 622 self.Entropy.get_local_packages_directory(repo), 623 pkg_to_join_dirpath) 624 if (not os.path.isdir(download_path)) and \ 625 (not os.access(download_path, os.R_OK)): 626 os.makedirs(download_path) 627 rc_download = ftp.download_file(pkgfile, download_path) 628 if not rc_download: 629 self.Entropy.updateProgress( 630 "[repo:%s|%s|#%s] %s: %s %s" % ( 631 brown(repo), 632 darkgreen(crippled_uri), 633 brown(tries), 634 blue(_("package")), 635 darkgreen(pkg_to_join_path), 636 blue(_("does not exist")), 637 ), 638 importance = 1, 639 type = "error", 640 header = darkred(" !!! ") 641 ) 642 ftp.close() 643 return rc_download 644 645 dbconn = self.Entropy.open_server_repository(read_only = True, 646 no_upload = True, repo = repo) 647 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 648 if idpackage == -1: 649 self.Entropy.updateProgress( 650 "[repo:%s|%s|#%s] %s: %s %s" % ( 651 brown(repo), 652 darkgreen(crippled_uri), 653 brown(tries), 654 blue(_("package")), 655 darkgreen(pkgfile), 656 blue(_("is not listed in the repository !")), 657 ), 658 importance = 1, 659 type = "error", 660 header = darkred(" !!! ") 661 ) 662 ftp.close() 663 return False 664 665 storedmd5 = dbconn.retrieveDigest(idpackage) 666 self.Entropy.updateProgress( 667 "[repo:%s|%s|#%s] %s: %s" % ( 668 brown(repo), 669 darkgreen(crippled_uri), 670 brown(tries), 671 blue(_("verifying checksum of package")), 672 darkgreen(pkgfile), 673 ), 674 importance = 1, 675 type = "info", 676 header = darkgreen(" * "), 677 back = True 678 ) 679 680 pkg_path = os.path.join(download_path, pkgfile) 681 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 682 if md5check: 683 self.Entropy.updateProgress( 684 "[repo:%s|%s|#%s] %s: %s %s" % ( 685 brown(repo), 686 darkgreen(crippled_uri), 687 brown(tries), 688 blue(_("package")), 689 darkgreen(pkgfile), 690 blue(_("downloaded successfully")), 691 ), 692 importance = 1, 693 type = "info", 694 header = darkgreen(" * ") 695 ) 696 return True 697 else: 698 self.Entropy.updateProgress( 699 "[repo:%s|%s|#%s] %s: %s %s" % ( 700 brown(repo), 701 darkgreen(crippled_uri), 702 brown(tries), 703 blue(_("package")), 704 darkgreen(pkgfile), 705 blue(_("checksum does not match. re-downloading...")), 706 ), 707 importance = 1, 708 type = "warning", 709 header = darkred(" * ") 710 ) 711 if os.path.isfile(pkg_path): 712 os.remove(pkg_path) 713 714 continue 715 716 # if we get here it means the files hasn't been downloaded properly 717 self.Entropy.updateProgress( 718 "[repo:%s|%s|#%s] %s: %s %s" % ( 719 brown(repo), 720 darkgreen(crippled_uri), 721 brown(tries), 722 blue(_("package")), 723 darkgreen(pkgfile), 724 blue(_("seems broken. Consider to re-package it. Giving up!")), 725 ), 726 importance = 1, 727 type = "error", 728 header = darkred(" !!! ") 729 ) 730 return False

731 732

733 - def get_remote_databases_status(self, repo = None, mirrors = None):

734 735 if repo == None: 736 repo = self.Entropy.default_repository 737 738 if not mirrors: 739 mirrors = self.Entropy.get_remote_mirrors(repo) 740 741 data = [] 742 for uri in mirrors: 743 744 # let it raise an exception if connection is impossible 745 ftp = FtpInterface(uri, self.Entropy) 746 try: 747 my_path = os.path.join( 748 self.Entropy.get_remote_database_relative_path(repo), 749 self.SystemSettings['repositories']['branch']) 750 ftp.set_cwd(my_path, dodir = True) 751 except ftp.ftplib.error_perm: 752 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 753 self.Entropy.updateProgress( 754 "[repo:%s|%s] %s !" % ( 755 brown(repo), 756 darkgreen(crippled_uri), 757 blue(_("mirror has invalid directory structure")), 758 ), 759 importance = 1, 760 type = "warning", 761 header = darkred(" !!! ") 762 ) 763 ftp.close() 764 continue 765 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 766 db_format = sys_set['database_file_format'] 767 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 768 if cmethod == None: 769 raise InvalidDataType("InvalidDataType: %s." % ( 770 _("Wrong database compression method passed"), 771 ) 772 ) 773 compressedfile = etpConst[cmethod[2]] 774 775 revision = 0 776 rc1 = ftp.is_file_available(compressedfile) 777 revfilename = os.path.basename( 778 self.Entropy.get_local_database_revision_file(repo)) 779 rc2 = ftp.is_file_available(revfilename) 780 if rc1 and rc2: 781 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 782 revfilename) 783 dlcount = 5 784 while dlcount: 785 dled = ftp.download_file(revfilename, 786 etpConst['packagestmpdir'], True) 787 if dled: 788 break 789 dlcount -= 1 790 791 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 792 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 793 f_rev = open(revision_localtmppath,"r") 794 try: 795 revision = int(f_rev.readline().strip()) 796 except ValueError: 797 mytxt = _("mirror hasn't valid database revision file") 798 self.Entropy.updateProgress( 799 "[repo:%s|%s] %s: %s" % ( 800 brown(repo), 801 darkgreen(crippled_uri), 802 blue(mytxt), 803 bold(revision), 804 ), 805 importance = 1, 806 type = "error", 807 header = darkred(" !!! ") 808 ) 809 revision = 0 810 f_rev.close() 811 elif dlcount == 0: 812 self.Entropy.updateProgress( 813 "[repo:%s|%s] %s: %s" % ( 814 brown(repo), 815 darkgreen(crippled_uri), 816 blue(_("unable to download repository revision")), 817 bold(revision), 818 ), 819 importance = 1, 820 type = "error", 821 header = darkred(" !!! ") 822 ) 823 revision = 0 824 else: 825 self.Entropy.updateProgress( 826 "[repo:%s|%s] %s: %s" % ( 827 brown(repo), 828 darkgreen(crippled_uri), 829 blue(_("mirror doesn't have valid revision file")), 830 bold(revision), 831 ), 832 importance = 1, 833 type = "error", 834 header = darkred(" !!! ") 835 ) 836 revision = 0 837 if os.path.isfile(revision_localtmppath): 838 os.remove(revision_localtmppath) 839 840 info = [uri, revision] 841 data.append(info) 842 ftp.close() 843 844 return data

845

846 - def is_local_database_locked(self, repo = None):

847 local_repo = repo 848 if local_repo == None: 849 local_repo = self.Entropy.default_repository 850 lock_file = self.get_database_lockfile(local_repo) 851 return os.path.isfile(lock_file)

852

853 - def get_mirrors_lock(self, repo = None):

854 855 dbstatus = [] 856 for uri in self.Entropy.get_remote_mirrors(repo): 857 data = [uri, False, False] 858 ftp = FtpInterface(uri, self.Entropy) 859 try: 860 my_path = os.path.join( 861 self.Entropy.get_remote_database_relative_path(repo), 862 self.SystemSettings['repositories']['branch']) 863 ftp.set_cwd(my_path) 864 except ftp.ftplib.error_perm: 865 ftp.close() 866 continue 867 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 868 # upload locked 869 data[1] = True 870 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 871 # download locked 872 data[2] = True 873 ftp.close() 874 dbstatus.append(data) 875 return dbstatus

876

877 - def download_notice_board(self, repo = None):

878 879 if repo == None: 880 repo = self.Entropy.default_repository 881 mirrors = self.Entropy.get_remote_mirrors(repo) 882 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 883 mytmpdir = self.entropyTools.get_random_temp_file() 884 os.makedirs(mytmpdir) 885 886 self.Entropy.updateProgress( 887 "[repo:%s] %s %s" % ( 888 brown(repo), 889 blue(_("downloading notice board from mirrors to")), 890 red(rss_path), 891 ), 892 importance = 1, 893 type = "info", 894 header = blue(" @@ ") 895 ) 896 897 downloaded = False 898 for uri in mirrors: 899 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 900 downloader = self.FtpServerHandler( 901 FtpInterface, self.Entropy, [uri], 902 [rss_path], download = True, 903 local_basedir = mytmpdir, critical_files = [rss_path], 904 repo = repo 905 ) 906 errors, m_fine_uris, m_broken_uris = downloader.go() 907 if not errors: 908 self.Entropy.updateProgress( 909 "[repo:%s] %s: %s" % ( 910 brown(repo), 911 blue(_("notice board downloaded successfully from")), 912 red(crippled_uri), 913 ), 914 importance = 1, 915 type = "info", 916 header = blue(" @@ ") 917 ) 918 downloaded = True 919 break 920 921 if downloaded: 922 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 923 rss_path) 924 925 return downloaded

926

927 - def upload_notice_board(self, repo = None):

928 929 if repo == None: 930 repo = self.Entropy.default_repository 931 mirrors = self.Entropy.get_remote_mirrors(repo) 932 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 933 934 self.Entropy.updateProgress( 935 "[repo:%s] %s %s" % ( 936 brown(repo), 937 blue(_("uploading notice board from")), 938 red(rss_path), 939 ), 940 importance = 1, 941 type = "info", 942 header = blue(" @@ ") 943 ) 944 945 uploader = self.FtpServerHandler( 946 FtpInterface, 947 self.Entropy, 948 mirrors, 949 [rss_path], 950 critical_files = [rss_path], 951 repo = repo 952 ) 953 errors, m_fine_uris, m_broken_uris = uploader.go() 954 if errors: 955 m_broken_uris = sorted(m_broken_uris) 956 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 957 for x in m_broken_uris] 958 self.Entropy.updateProgress( 959 "[repo:%s] %s %s" % ( 960 brown(repo), 961 blue(_("notice board upload failed on")), 962 red(', '.join(m_broken_uris)), 963 ), 964 importance = 1, 965 type = "info", 966 header = blue(" @@ ") 967 ) 968 return False 969 self.Entropy.updateProgress( 970 "[repo:%s] %s" % ( 971 brown(repo), 972 blue(_("notice board upload success")), 973 ), 974 importance = 1, 975 type = "info", 976 header = blue(" @@ ") 977 ) 978 return True

979 980

981 - def update_notice_board(self, title, notice_text, link = None, repo = None):

982 983 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 984 rss_description = "Inform about important distribution activities." 985 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 986 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 987 if not link: 988 link = srv_set['rss']['website_url'] 989 990 self.download_notice_board(repo) 991 rss_main = RSS(rss_path, rss_title, rss_description, 992 maxentries = 20) 993 rss_main.add_item(title, link, description = notice_text) 994 rss_main.write_changes() 995 status = self.upload_notice_board(repo) 996 return status

997

998 - def read_notice_board(self, do_download = True, repo = None):

999 1000 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1001 if do_download: 1002 self.download_notice_board(repo) 1003 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1004 return None 1005 rss_main = RSS(rss_path, '', '') 1006 return rss_main.get_entries()

1007

1008 - def remove_from_notice_board(self, identifier, repo = None):

1009 1010 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1011 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 1012 rss_description = "Inform about important distribution activities." 1013 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1014 return 0 1015 rss_main = RSS(rss_path, rss_title, rss_description) 1016 data = rss_main.remove_entry(identifier) 1017 rss_main.write_changes() 1018 return data

1019

1020 - def update_rss_feed(self, repo = None):

1021 1022 product = self.SystemSettings['repositories']['product'] 1023 #db_dir = self.Entropy.get_local_database_dir(repo) 1024 rss_path = self.Entropy.get_local_database_rss_file(repo) 1025 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 1026 rss_dump_name = etpConst['rss-dump-name'] 1027 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 1028 1029 rss_title = u"%s Online Repository Status" % ( 1030 self.SystemSettings['system']['name'],) 1031 rss_description = \ 1032 u"Keep you updated on what's going on in the %s Repository." % ( 1033 self.SystemSettings['system']['name'],) 1034 1035 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1036 1037 rss_main = RSS(rss_path, rss_title, rss_description, 1038 maxentries = srv_set['rss']['max_entries']) 1039 # load dump 1040 db_actions = self.Cacher.pop(rss_dump_name) 1041 if db_actions: 1042 try: 1043 f_rev = open(db_revision_path) 1044 revision = f_rev.readline().strip() 1045 f_rev.close() 1046 except (IOError, OSError): 1047 revision = u"N/A" 1048 commitmessage = u'' 1049 if self.Entropy.rssMessages['commitmessage']: 1050 commitmessage = u' :: ' + \ 1051 self.Entropy.rssMessages['commitmessage'] 1052 1053 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 1054 product[0].upper() + product[1:] + " " + \ 1055 self.SystemSettings['repositories']['branch'] + \ 1056 " :: Revision: " + revision + commitmessage 1057 1058 link = srv_set['rss']['base_url'] 1059 # create description 1060 added_items = db_actions.get("added") 1061 1062 if added_items: 1063 for atom in sorted(added_items): 1064 mylink = link + "?search=" + atom.split("~")[0] + \ 1065 "&arch=" + etpConst['currentarch'] + "&product="+product 1066 description = atom + ": " + added_items[atom]['description'] 1067 rss_main.add_item(title = "Added/Updated" + title, 1068 link = mylink, description = description) 1069 removed_items = db_actions.get("removed") 1070 1071 if removed_items: 1072 for atom in sorted(removed_items): 1073 description = atom + ": " + \ 1074 removed_items[atom]['description'] 1075 rss_main.add_item(title = "Removed" + title, link = link, 1076 description = description) 1077 1078 light_items = db_actions.get('light') 1079 if light_items: 1080 rss_light = RSS(rss_light_path, rss_title, rss_description, 1081 maxentries = srv_set['rss']['light_max_entries']) 1082 for atom in sorted(light_items): 1083 mylink = link + "?search=" + atom.split("~")[0] + \ 1084 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1085 product 1086 description = light_items[atom]['description'] 1087 rss_light.add_item(title = "[" + revision + "] " + atom, 1088 link = mylink, description = description) 1089 rss_light.write_changes() 1090 1091 rss_main.write_changes() 1092 self.Entropy.rssMessages.clear() 1093 self.dumpTools.removeobj(rss_dump_name)

1094 1095

1096 - def dump_database_to_file(self, db_path, destination_path, opener, 1097 exclude_tables = None, repo = None):

1098 1099 if not exclude_tables: 1100 exclude_tables = [] 1101 1102 f_out = opener(destination_path, "wb") 1103 dbconn = self.Entropy.open_server_repository(db_path, 1104 just_reading = True, repo = repo, do_treeupdates = False) 1105 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1106 self.Entropy.close_server_database(dbconn) 1107 f_out.close()

1108

1109 - def create_file_checksum(self, file_path, checksum_path):

1110 mydigest = self.entropyTools.md5sum(file_path) 1111 f_ck = open(checksum_path, "w") 1112 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1113 f_ck.write(mystring) 1114 f_ck.flush() 1115 f_ck.close()

1116

1117 - def compress_file(self, file_path, destination_path, opener):

1118 f_out = opener(destination_path, "wb") 1119 f_in = open(file_path,"rb") 1120 data = f_in.read(8192) 1121 while data: 1122 f_out.write(data) 1123 data = f_in.read(8192) 1124 f_in.close() 1125 if hasattr(f_out,'flush'): 1126 f_out.flush() 1127 f_out.close()

1128

1129 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1130 disabled_eapis = None):

1131 1132 if disabled_eapis == None: 1133 disabled_eapis = [] 1134 1135 critical = [] 1136 extra_text_files = [] 1137 data = {} 1138 data['database_revision_file'] = \ 1139 self.Entropy.get_local_database_revision_file(repo) 1140 extra_text_files.append(data['database_revision_file']) 1141 critical.append(data['database_revision_file']) 1142 1143 1144 # branch migration support scripts 1145 data['database_post_branch_hop_script'] = \ 1146 self.Entropy.get_local_post_branch_mig_script(repo) 1147 extra_text_files.append(data['database_post_branch_hop_script']) 1148 1149 data['database_post_branch_upgrade_script'] = \ 1150 self.Entropy.get_local_post_branch_upg_script(repo) 1151 extra_text_files.append(data['database_post_branch_upgrade_script']) 1152 1153 1154 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1155 if os.path.isfile(database_ts_file) or download: 1156 data['database_timestamp_file'] = database_ts_file 1157 if not download: 1158 critical.append(database_ts_file) 1159 1160 database_package_mask_file = \ 1161 self.Entropy.get_local_database_mask_file(repo) 1162 extra_text_files.append(database_package_mask_file) 1163 if os.path.isfile(database_package_mask_file) or download: 1164 data['database_package_mask_file'] = database_package_mask_file 1165 if not download: 1166 critical.append(data['database_package_mask_file']) 1167 1168 database_package_system_mask_file = \ 1169 self.Entropy.get_local_database_system_mask_file(repo) 1170 extra_text_files.append(database_package_system_mask_file) 1171 if os.path.isfile(database_package_system_mask_file) or download: 1172 data['database_package_system_mask_file'] = \ 1173 database_package_system_mask_file 1174 if not download: 1175 critical.append(data['database_package_system_mask_file']) 1176 1177 database_package_confl_tagged_file = \ 1178 self.Entropy.get_local_database_confl_tagged_file(repo) 1179 extra_text_files.append(database_package_confl_tagged_file) 1180 if os.path.isfile(database_package_confl_tagged_file) or download: 1181 data['database_package_confl_tagged_file'] = \ 1182 database_package_confl_tagged_file 1183 if not download: 1184 critical.append(data['database_package_confl_tagged_file']) 1185 1186 database_license_whitelist_file = \ 1187 self.Entropy.get_local_database_licensewhitelist_file(repo) 1188 extra_text_files.append(database_license_whitelist_file) 1189 if os.path.isfile(database_license_whitelist_file) or download: 1190 data['database_license_whitelist_file'] = \ 1191 database_license_whitelist_file 1192 if not download: 1193 critical.append(data['database_license_whitelist_file']) 1194 1195 exp_based_pkgs_removal_file = \ 1196 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1197 extra_text_files.append(exp_based_pkgs_removal_file) 1198 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1199 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1200 if not download: 1201 critical.append(data['exp_based_pkgs_removal_file']) 1202 1203 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1204 if os.path.isfile(database_rss_file) or download: 1205 data['database_rss_file'] = database_rss_file 1206 if not download: 1207 critical.append(data['database_rss_file']) 1208 database_rss_light_file = \ 1209 self.Entropy.get_local_database_rsslight_file(repo) 1210 1211 extra_text_files.append(database_rss_light_file) 1212 if os.path.isfile(database_rss_light_file) or download: 1213 data['database_rss_light_file'] = database_rss_light_file 1214 if not download: 1215 critical.append(data['database_rss_light_file']) 1216 1217 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1218 data['pkglist_file'] = pkglist_file 1219 if not download: 1220 critical.append(data['pkglist_file']) 1221 1222 # EAPI 2,3 1223 if not download: # we don't need to get the dump 1224 1225 if 3 not in disabled_eapis: 1226 1227 data['metafiles_path'] = \ 1228 self.Entropy.get_local_database_compressed_metafiles_file( 1229 repo) 1230 critical.append(data['metafiles_path']) 1231 1232 if 2 not in disabled_eapis: 1233 1234 data['dump_path_light'] = os.path.join( 1235 self.Entropy.get_local_database_dir(repo), 1236 etpConst[cmethod[5]]) 1237 critical.append(data['dump_path_light']) 1238 1239 data['dump_path_digest_light'] = os.path.join( 1240 self.Entropy.get_local_database_dir(repo), 1241 etpConst[cmethod[6]]) 1242 critical.append(data['dump_path_digest_light']) 1243 1244 # EAPI 1 1245 if 1 not in disabled_eapis: 1246 1247 data['compressed_database_path'] = os.path.join( 1248 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1249 critical.append(data['compressed_database_path']) 1250 data['compressed_database_path_light'] = os.path.join( 1251 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1252 1253 data['database_path_digest'] = os.path.join( 1254 self.Entropy.get_local_database_dir(repo), 1255 etpConst['etpdatabasehashfile'] 1256 ) 1257 critical.append(data['database_path_digest']) 1258 1259 data['compressed_database_path_digest'] = os.path.join( 1260 self.Entropy.get_local_database_dir(repo), 1261 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1262 ) 1263 critical.append(data['compressed_database_path_digest']) 1264 1265 data['compressed_database_path_digest_light'] = os.path.join( 1266 self.Entropy.get_local_database_dir(repo), 1267 etpConst[cmethod[8]] 1268 ) 1269 critical.append(data['compressed_database_path_digest_light']) 1270 1271 1272 # SSL cert file, just for reference 1273 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1274 extra_text_files.append(ssl_ca_cert) 1275 if os.path.isfile(ssl_ca_cert): 1276 data['ssl_ca_cert_file'] = ssl_ca_cert 1277 if not download: 1278 critical.append(ssl_ca_cert) 1279 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1280 extra_text_files.append(ssl_server_cert) 1281 if os.path.isfile(ssl_server_cert): 1282 data['ssl_server_cert_file'] = ssl_server_cert 1283 if not download: 1284 critical.append(ssl_server_cert) 1285 1286 # Some information regarding how packages are built 1287 spm_files = [ 1288 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1289 (etpConst['spm']['global_package_keywords'], 1290 "global_package_keywords"), 1291 (etpConst['spm']['global_package_use'], "global_package_use"), 1292 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1293 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1294 ] 1295 for myfile, myname in spm_files: 1296 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1297 data[myname] = myfile 1298 extra_text_files.append(myfile) 1299 1300 make_profile = etpConst['spm']['global_make_profile'] 1301 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1302 mytmpdir = os.path.dirname(rnd_tmp_file) 1303 mytmpfile = os.path.join(mytmpdir, 1304 etpConst['spm']['global_make_profile_link_name']) 1305 extra_text_files.append(mytmpfile) 1306 1307 if os.path.islink(make_profile): 1308 mylink = os.readlink(make_profile) 1309 f_mkp = open(mytmpfile,"w") 1310 f_mkp.write(mylink) 1311 f_mkp.flush() 1312 f_mkp.close() 1313 data['global_make_profile'] = mytmpfile 1314 1315 return data, critical, extra_text_files

1316

1317 - def _show_package_sets_messages(self, repo):

1318 1319 self.Entropy.updateProgress( 1320 "[repo:%s] %s:" % ( 1321 brown(repo), 1322 blue(_("configured package sets")), 1323 ), 1324 importance = 0, 1325 type = "info", 1326 header = darkgreen(" * ") 1327 ) 1328 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1329 if not sets_data: 1330 self.Entropy.updateProgress( 1331 "%s" % (_("None configured"),), 1332 importance = 0, 1333 type = "info", 1334 header = brown(" # ") 1335 ) 1336 return 1337 for s_repo, s_name, s_sets in sets_data: 1338 self.Entropy.updateProgress( 1339 blue("%s" % (s_name,)), 1340 importance = 0, 1341 type = "info", 1342 header = brown(" # ") 1343 )

1344

1345 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1346 1347 self.Entropy.updateProgress( 1348 "[repo:%s|%s|%s:%s] %s" % ( 1349 brown(repo), 1350 darkgreen(crippled_uri), 1351 red("EAPI"), 1352 bold("3"), 1353 blue(_("preparing uncompressed database for the upload")), 1354 ), 1355 importance = 0, 1356 type = "info", 1357 header = darkgreen(" * ") 1358 ) 1359 self.Entropy.updateProgress( 1360 "%s: %s" % (_("database path"), blue(database_path),), 1361 importance = 0, 1362 type = "info", 1363 header = brown(" # ") 1364 )

1365

1366 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1367 upload_data, cmethod, repo):

1368 1369 if repo == None: 1370 repo = self.Entropy.default_repository 1371 1372 self.Entropy.updateProgress( 1373 "[repo:%s|%s|%s:%s] %s" % ( 1374 brown(repo), 1375 darkgreen(crippled_uri), 1376 red("EAPI"), 1377 bold("2"), 1378 blue(_("creating compressed database dump + checksum")), 1379 ), 1380 importance = 0, 1381 type = "info", 1382 header = darkgreen(" * ") 1383 ) 1384 self.Entropy.updateProgress( 1385 "%s: %s" % (_("database path"), blue(database_path),), 1386 importance = 0, 1387 type = "info", 1388 header = brown(" # ") 1389 ) 1390 self.Entropy.updateProgress( 1391 "%s: %s" % ( 1392 _("dump light"), 1393 blue(upload_data['dump_path_light']), 1394 ), 1395 importance = 0, 1396 type = "info", 1397 header = brown(" # ") 1398 ) 1399 self.Entropy.updateProgress( 1400 "%s: %s" % ( 1401 _("dump light checksum"), 1402 blue(upload_data['dump_path_digest_light']), 1403 ), 1404 importance = 0, 1405 type = "info", 1406 header = brown(" # ") 1407 ) 1408 1409 self.Entropy.updateProgress( 1410 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1411 importance = 0, 1412 type = "info", 1413 header = brown(" # ") 1414 )

1415

1416 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1417 upload_data, cmethod, repo):

1418 1419 self.Entropy.updateProgress( 1420 "[repo:%s|%s|%s:%s] %s" % ( 1421 brown(repo), 1422 darkgreen(crippled_uri), 1423 red("EAPI"), 1424 bold("1"), 1425 blue(_("compressing database + checksum")), 1426 ), 1427 importance = 0, 1428 type = "info", 1429 header = darkgreen(" * "), 1430 back = True 1431 ) 1432 self.Entropy.updateProgress( 1433 "%s: %s" % (_("database path"), blue(database_path),), 1434 importance = 0, 1435 type = "info", 1436 header = brown(" # ") 1437 ) 1438 self.Entropy.updateProgress( 1439 "%s: %s" % ( 1440 _("compressed database path"), 1441 blue(upload_data['compressed_database_path']), 1442 ), 1443 importance = 0, 1444 type = "info", 1445 header = brown(" # ") 1446 ) 1447 self.Entropy.updateProgress( 1448 "%s: %s" % ( 1449 _("database checksum"), 1450 blue(upload_data['database_path_digest']), 1451 ), 1452 importance = 0, 1453 type = "info", 1454 header = brown(" # ") 1455 ) 1456 self.Entropy.updateProgress( 1457 "%s: %s" % ( 1458 _("compressed checksum"), 1459 blue(upload_data['compressed_database_path_digest']), 1460 ), 1461 importance = 0, 1462 type = "info", 1463 header = brown(" # ") 1464 ) 1465 self.Entropy.updateProgress( 1466 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1467 importance = 0, 1468 type = "info", 1469 header = brown(" # ") 1470 )

1471

1472 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1473 1474 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1475 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1476 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1477 file_list if x not in found_file_list] 1478 metafile_not_found_file = \ 1479 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1480 1481 f_meta = open(metafile_not_found_file,"w") 1482 f_meta.writelines(not_found_file_list) 1483 f_meta.flush() 1484 f_meta.close() 1485 found_file_list.append(metafile_not_found_file) 1486 if os.path.isfile(compressed_dest_path): 1487 os.remove(compressed_dest_path) 1488 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1489

1490 - def create_mirror_directories(self, ftp_connection, path_to_create):

1491 bdir = "" 1492 for mydir in path_to_create.split("/"): 1493 bdir += "/"+mydir 1494 if not ftp_connection.is_file_available(bdir): 1495 try: 1496 ftp_connection.mkdir(bdir) 1497 except Exception, err: 1498 error = unicode(err) 1499 if (error.find("550") == -1) and \ 1500 (error.find("File exist") == -1): 1501 1502 mytxt = "%s %s, %s: %s" % ( 1503 _("cannot create mirror directory"), 1504 bdir, 1505 _("error"), 1506 err, 1507 ) 1508 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1509 mytxt,)) 1510 raise

1511

1512 - def mirror_lock_check(self, uri, repo = None):

1513 1514 if repo == None: 1515 repo = self.Entropy.default_repository 1516 1517 gave_up = False 1518 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1519 try: 1520 ftp = FtpInterface(uri, self.Entropy) 1521 except ConnectionError: 1522 self.entropyTools.print_traceback() 1523 return True # gave up 1524 my_path = os.path.join( 1525 self.Entropy.get_remote_database_relative_path(repo), 1526 self.SystemSettings['repositories']['branch']) 1527 ftp.set_cwd(my_path, dodir = True) 1528 1529 lock_file = self.get_database_lockfile(repo) 1530 lock_filename = os.path.basename(lock_file) 1531 if not os.path.isfile(lock_file) and \ 1532 ftp.is_file_available(lock_filename): 1533 1534 self.Entropy.updateProgress( 1535 "[repo:%s|%s|%s] %s, %s" % ( 1536 brown(str(repo)), 1537 darkgreen(crippled_uri), 1538 red(_("locking")), 1539 darkblue(_("mirror already locked")), 1540 blue(_("waiting up to 2 minutes before giving up")), 1541 ), 1542 importance = 1, 1543 type = "warning", 1544 header = brown(" * "), 1545 back = True 1546 ) 1547 unlocked = False 1548 count = 0 1549 while count < 120: 1550 count += 1 1551 time.sleep(1) 1552 if not ftp.is_file_available(os.path.basename(lock_file)): 1553 self.Entropy.updateProgress( 1554 red("[repo:%s|%s|%s] %s !" % ( 1555 repo, 1556 crippled_uri, 1557 _("locking"), 1558 _("mirror unlocked"), 1559 ) 1560 ), 1561 importance = 1, 1562 type = "info", 1563 header = darkgreen(" * ") 1564 ) 1565 unlocked = True 1566 break 1567 if not unlocked: 1568 gave_up = True 1569 1570 ftp.close() 1571 return gave_up

1572

1573 - def shrink_database_and_close(self, repo = None):

1574 dbconn = self.Entropy.open_server_repository(read_only = False, 1575 no_upload = True, repo = repo, indexing = False, 1576 do_treeupdates = False) 1577 dbconn.dropAllIndexes() 1578 dbconn.vacuum() 1579 dbconn.vacuum() 1580 dbconn.commitChanges() 1581 self.Entropy.close_server_database(dbconn)

1582

1583 - def update_repository_timestamp(self, repo = None):

1584 if repo == None: 1585 repo = self.Entropy.default_repository 1586 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1587 current_ts = self.Entropy.get_current_timestamp() 1588 ts_f = open(ts_file,"w") 1589 ts_f.write(current_ts) 1590 ts_f.flush() 1591 ts_f.close()

1592

1593 - def sync_database_treeupdates(self, repo = None):

1594 1595 if repo == None: 1596 repo = self.Entropy.default_repository 1597 dbconn = self.Entropy.open_server_repository(read_only = False, 1598 no_upload = True, repo = repo, do_treeupdates = False) 1599 # grab treeupdates from other databases and inject 1600 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1601 server_repos = srv_set['repositories'].keys() 1602 all_actions = set() 1603 for myrepo in server_repos: 1604 1605 # avoid __default__ 1606 if myrepo == etpConst['clientserverrepoid']: 1607 continue 1608 1609 mydbc = self.Entropy.open_server_repository(just_reading = True, 1610 repo = myrepo) 1611 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1612 for data in actions: 1613 all_actions.add(data) 1614 if not actions: 1615 continue 1616 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1617 try: 1618 # clear first 1619 dbconn.removeTreeUpdatesActions(repo) 1620 dbconn.insertTreeUpdatesActions(all_actions, repo) 1621 except dbapi2.Error, err: 1622 self.entropyTools.print_traceback() 1623 mytxt = "%s, %s: %s. %s" % ( 1624 _("Troubles with treeupdates"), 1625 _("error"), 1626 err, 1627 _("Bumping old data back"), 1628 ) 1629 self.Entropy.updateProgress( 1630 mytxt, 1631 importance = 1, 1632 type = "warning" 1633 ) 1634 # restore previous data 1635 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1636 1637 dbconn.commitChanges() 1638 self.Entropy.close_server_database(dbconn)

1639

1640 - def upload_database(self, uris, lock_check = False, pretend = False, 1641 repo = None):

1642 1643 if repo == None: 1644 repo = self.Entropy.default_repository 1645 1646 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1647 if srv_set['rss']['enabled']: 1648 self.update_rss_feed(repo = repo) 1649 1650 upload_errors = False 1651 broken_uris = set() 1652 fine_uris = set() 1653 1654 disabled_eapis = sorted(srv_set['disabled_eapis']) 1655 1656 for uri in uris: 1657 1658 db_format = srv_set['database_file_format'] 1659 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1660 if cmethod == None: 1661 raise InvalidDataType("InvalidDataType: %s." % ( 1662 _("wrong database compression method passed"), 1663 ) 1664 ) 1665 1666 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1667 database_path = self.Entropy.get_local_database_file(repo) 1668 1669 if disabled_eapis: 1670 self.Entropy.updateProgress( 1671 "[repo:%s|%s|%s] %s: %s" % ( 1672 blue(repo), 1673 red(crippled_uri), 1674 darkgreen(_("upload")), 1675 darkred(_("disabled EAPI")), 1676 bold(', '.join(disabled_eapis)), 1677 ), 1678 importance = 1, 1679 type = "warning", 1680 header = darkgreen(" * ") 1681 ) 1682 1683 # create/update timestamp file 1684 self.update_repository_timestamp(repo) 1685 # create pkglist service file 1686 self.Entropy.create_repository_pkglist(repo) 1687 1688 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1689 repo = repo, disabled_eapis = disabled_eapis) 1690 1691 if lock_check: 1692 given_up = self.mirror_lock_check(uri, repo = repo) 1693 if given_up: 1694 upload_errors = True 1695 broken_uris.add(uri) 1696 continue 1697 1698 self.lock_mirrors_for_download(True, [uri], repo = repo) 1699 1700 self.Entropy.updateProgress( 1701 "[repo:%s|%s|%s] %s" % ( 1702 blue(repo), 1703 red(crippled_uri), 1704 darkgreen(_("upload")), 1705 darkgreen(_("preparing to upload database to mirror")), 1706 ), 1707 importance = 1, 1708 type = "info", 1709 header = darkgreen(" * ") 1710 ) 1711 1712 # Package Sets info 1713 self._show_package_sets_messages(repo) 1714 self.sync_database_treeupdates(repo) 1715 self.Entropy.update_database_package_sets(repo) 1716 self.Entropy.close_server_databases() 1717 1718 # backup current database to avoid re-indexing 1719 old_dbpath = self.Entropy.get_local_database_file(repo) 1720 backup_dbpath = old_dbpath + ".up_backup" 1721 copy_back = False 1722 if not pretend: 1723 try: 1724 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1725 os.remove(backup_dbpath) 1726 shutil.copy2(old_dbpath, backup_dbpath) 1727 copy_back = True 1728 except shutil.Error: 1729 copy_back = False 1730 1731 self.shrink_database_and_close(repo) 1732 1733 # EAPI 3 1734 if 3 not in disabled_eapis: 1735 self._create_metafiles_file(upload_data['metafiles_path'], 1736 text_files, repo) 1737 self._show_eapi3_upload_messages(crippled_uri, database_path, 1738 repo) 1739 1740 # EAPI 2 1741 if 2 not in disabled_eapis: 1742 self._show_eapi2_upload_messages(crippled_uri, database_path, 1743 upload_data, cmethod, repo) 1744 1745 # create compressed dump + checksum 1746 self.dump_database_to_file(database_path, 1747 upload_data['dump_path_light'], cmethod[0], 1748 exclude_tables = ["content"], repo = repo) 1749 self.create_file_checksum(upload_data['dump_path_light'], 1750 upload_data['dump_path_digest_light']) 1751 1752 1753 # EAPI 1 1754 if 1 not in disabled_eapis: 1755 self._show_eapi1_upload_messages(crippled_uri, database_path, 1756 upload_data, cmethod, repo) 1757 1758 # compress the database and create uncompressed 1759 # database checksum -- DEPRECATED 1760 self.compress_file(database_path, 1761 upload_data['compressed_database_path'], cmethod[0]) 1762 self.create_file_checksum(database_path, 1763 upload_data['database_path_digest']) 1764 1765 # create compressed database checksum 1766 self.create_file_checksum( 1767 upload_data['compressed_database_path'], 1768 upload_data['compressed_database_path_digest']) 1769 1770 # create light version of the compressed db 1771 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1772 temp_eapi1_dbfile = eapi1_dbfile+".light" 1773 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1774 # open and remove content table 1775 eapi1_tmp_dbconn = \ 1776 self.Entropy.ClientService.open_generic_database( 1777 temp_eapi1_dbfile, indexing_override = False, 1778 xcache = False) 1779 eapi1_tmp_dbconn.dropContent() 1780 eapi1_tmp_dbconn.commitChanges() 1781 eapi1_tmp_dbconn.vacuum() 1782 eapi1_tmp_dbconn.closeDB() 1783 1784 # compress 1785 self.compress_file(temp_eapi1_dbfile, 1786 upload_data['compressed_database_path_light'], cmethod[0]) 1787 # go away, we don't need you anymore 1788 os.remove(temp_eapi1_dbfile) 1789 # create compressed light database checksum 1790 self.create_file_checksum( 1791 upload_data['compressed_database_path_light'], 1792 upload_data['compressed_database_path_digest_light']) 1793 1794 if not pretend: 1795 # upload 1796 uploader = self.FtpServerHandler( 1797 FtpInterface, 1798 self.Entropy, 1799 [uri], 1800 [upload_data[x] for x in upload_data], 1801 critical_files = critical, 1802 repo = repo 1803 ) 1804 errors, m_fine_uris, m_broken_uris = uploader.go() 1805 if errors: 1806 my_broken_uris = sorted([ 1807 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1808 x[1]) for x in m_broken_uris]) 1809 self.Entropy.updateProgress( 1810 "[repo:%s|%s|%s] %s" % ( 1811 repo, 1812 crippled_uri, 1813 _("errors"), 1814 _("upload failed, not unlocking and continuing"), 1815 ), 1816 importance = 0, 1817 type = "error", 1818 header = darkred(" !!! ") 1819 ) 1820 # get reason 1821 reason = my_broken_uris[0][1] 1822 self.Entropy.updateProgress( 1823 blue("%s: %s" % (_("reason"), reason,)), 1824 importance = 0, 1825 type = "error", 1826 header = blue(" # ") 1827 ) 1828 upload_errors = True 1829 broken_uris |= m_broken_uris 1830 continue 1831 1832 # copy db back 1833 if copy_back and os.path.isfile(backup_dbpath): 1834 self.Entropy.close_server_databases() 1835 further_backup_dbpath = old_dbpath+".security_backup" 1836 if os.path.isfile(further_backup_dbpath): 1837 os.remove(further_backup_dbpath) 1838 shutil.copy2(old_dbpath, further_backup_dbpath) 1839 shutil.move(backup_dbpath, old_dbpath) 1840 1841 # unlock 1842 self.lock_mirrors_for_download(False, [uri], repo = repo) 1843 fine_uris |= m_fine_uris 1844 1845 if not fine_uris: 1846 upload_errors = True 1847 return upload_errors, broken_uris, fine_uris

1848 1849

1850 - def download_database(self, uris, lock_check = False, pretend = False, 1851 repo = None):

1852 1853 if repo == None: 1854 repo = self.Entropy.default_repository 1855 1856 download_errors = False 1857 broken_uris = set() 1858 fine_uris = set() 1859 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1860 disabled_eapis = sorted(srv_set['disabled_eapis']) 1861 1862 for uri in uris: 1863 1864 db_format = srv_set['database_file_format'] 1865 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1866 if cmethod == None: 1867 raise InvalidDataType("InvalidDataType: %s." % ( 1868 _("wrong database compression method passed"), 1869 ) 1870 ) 1871 1872 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1873 database_path = self.Entropy.get_local_database_file(repo) 1874 database_dir_path = os.path.dirname( 1875 self.Entropy.get_local_database_file(repo)) 1876 download_data, critical, text_files = self.get_files_to_sync( 1877 cmethod, download = True, 1878 repo = repo, disabled_eapis = disabled_eapis) 1879 mytmpdir = self.entropyTools.get_random_temp_file() 1880 os.makedirs(mytmpdir) 1881 1882 self.Entropy.updateProgress( 1883 "[repo:%s|%s|%s] %s" % ( 1884 brown(repo), 1885 darkgreen(crippled_uri), 1886 red(_("download")), 1887 blue(_("preparing to download database from mirror")), 1888 ), 1889 importance = 1, 1890 type = "info", 1891 header = darkgreen(" * ") 1892 ) 1893 files_to_sync = sorted(download_data.keys()) 1894 for myfile in files_to_sync: 1895 self.Entropy.updateProgress( 1896 "%s: %s" % ( 1897 blue(_("download path")), 1898 brown(unicode(download_data[myfile])), 1899 ), 1900 importance = 0, 1901 type = "info", 1902 header = brown(" # ") 1903 ) 1904 1905 if lock_check: 1906 given_up = self.mirror_lock_check(uri, repo = repo) 1907 if given_up: 1908 download_errors = True 1909 broken_uris.add(uri) 1910 continue 1911 1912 # avoid having others messing while we're downloading 1913 self.lock_mirrors(True, [uri], repo = repo) 1914 1915 if not pretend: 1916 # download 1917 downloader = self.FtpServerHandler( 1918 FtpInterface, self.Entropy, [uri], 1919 [download_data[x] for x in download_data], download = True, 1920 local_basedir = mytmpdir, critical_files = critical, 1921 repo = repo 1922 ) 1923 errors, m_fine_uris, m_broken_uris = downloader.go() 1924 if errors: 1925 my_broken_uris = sorted([ 1926 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1927 x[1]) for x in m_broken_uris]) 1928 self.Entropy.updateProgress( 1929 "[repo:%s|%s|%s] %s" % ( 1930 brown(repo), 1931 darkgreen(crippled_uri), 1932 red(_("errors")), 1933 blue(_("failed to download from mirror")), 1934 ), 1935 importance = 0, 1936 type = "error", 1937 header = darkred(" !!! ") 1938 ) 1939 # get reason 1940 reason = my_broken_uris[0][1] 1941 self.Entropy.updateProgress( 1942 blue("%s: %s" % (_("reason"), reason,)), 1943 importance = 0, 1944 type = "error", 1945 header = blue(" # ") 1946 ) 1947 download_errors = True 1948 broken_uris |= m_broken_uris 1949 self.lock_mirrors(False, [uri], repo = repo) 1950 continue 1951 1952 # all fine then, we need to move data from mytmpdir 1953 # to database_dir_path 1954 1955 # EAPI 1 -- unpack database 1956 if 1 not in disabled_eapis: 1957 compressed_db_filename = os.path.basename( 1958 download_data['compressed_database_path']) 1959 uncompressed_db_filename = os.path.basename(database_path) 1960 compressed_file = os.path.join(mytmpdir, 1961 compressed_db_filename) 1962 uncompressed_file = os.path.join(mytmpdir, 1963 uncompressed_db_filename) 1964 self.entropyTools.uncompress_file(compressed_file, 1965 uncompressed_file, cmethod[0]) 1966 1967 # now move 1968 for myfile in os.listdir(mytmpdir): 1969 fromfile = os.path.join(mytmpdir, myfile) 1970 tofile = os.path.join(database_dir_path, myfile) 1971 shutil.move(fromfile, tofile) 1972 self.Entropy.ClientService.setup_default_file_perms(tofile) 1973 1974 if os.path.isdir(mytmpdir): 1975 shutil.rmtree(mytmpdir) 1976 if os.path.isdir(mytmpdir): 1977 os.rmdir(mytmpdir) 1978 1979 1980 fine_uris.add(uri) 1981 self.lock_mirrors(False, [uri], repo = repo) 1982 1983 return download_errors, fine_uris, broken_uris

1984

1985 - def calculate_database_sync_queues(self, repo = None):

1986 1987 if repo == None: 1988 repo = self.Entropy.default_repository 1989 1990 remote_status = self.get_remote_databases_status(repo) 1991 local_revision = self.Entropy.get_local_database_revision(repo) 1992 upload_queue = [] 1993 download_latest = () 1994 1995 # all mirrors are empty ? I rule 1996 if not [x for x in remote_status if x[1]]: 1997 upload_queue = remote_status[:] 1998 else: 1999 highest_remote_revision = max([x[1] for x in remote_status]) 2000 2001 if local_revision < highest_remote_revision: 2002 for remote_item in remote_status: 2003 if remote_item[1] == highest_remote_revision: 2004 download_latest = x 2005 break 2006 2007 if download_latest: 2008 upload_queue = [x for x in remote_status if \ 2009 (x[1] < highest_remote_revision)] 2010 else: 2011 upload_queue = [x for x in remote_status if \ 2012 (x[1] < local_revision)] 2013 2014 return download_latest, upload_queue

2015

2016 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 2017 repo = None):

2018 2019 if repo == None: 2020 repo = self.Entropy.default_repository 2021 2022 while 1: 2023 2024 db_locked = False 2025 if self.is_local_database_locked(repo): 2026 db_locked = True 2027 2028 lock_data = self.get_mirrors_lock(repo) 2029 mirrors_locked = [x for x in lock_data if x[1]] 2030 2031 if not mirrors_locked and db_locked: 2032 # mirrors not locked remotely but only locally 2033 mylock_file = self.get_database_lockfile(repo) 2034 if os.access(mylock_file, os.W_OK | os.F_OK): 2035 os.remove(mylock_file) 2036 continue 2037 2038 break 2039 2040 if mirrors_locked and not db_locked: 2041 mytxt = "%s, %s %s" % ( 2042 _("Mirrors are locked, someone is working on the repository"), 2043 _("try again later"), 2044 "...", 2045 ) 2046 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 2047 2048 download_latest, upload_queue = self.calculate_database_sync_queues( 2049 repo) 2050 2051 if not download_latest and not upload_queue: 2052 self.Entropy.updateProgress( 2053 "[repo:%s|%s] %s" % ( 2054 brown(repo), 2055 red(_("sync")), # something short please 2056 blue(_("database already in sync")), 2057 ), 2058 importance = 1, 2059 type = "info", 2060 header = blue(" @@ ") 2061 ) 2062 return 0, set(), set() 2063 2064 if download_latest: 2065 download_uri = download_latest[0] 2066 download_errors, fine_uris, broken_uris = self.download_database( 2067 [download_uri], repo = repo) 2068 if download_errors: 2069 self.Entropy.updateProgress( 2070 "[repo:%s|%s] %s: %s" % ( 2071 brown(repo), 2072 red(_("sync")), 2073 blue(_("database sync failed")), 2074 red(_("download issues")), 2075 ), 2076 importance = 1, 2077 type = "error", 2078 header = darkred(" !!! ") 2079 ) 2080 return 1, fine_uris, broken_uris 2081 2082 if upload_queue and not no_upload: 2083 2084 # XXX QA checks, 2085 # please group them into entropy.qa 2086 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2087 if deps_not_found and not self.Entropy.community_repo: 2088 self.Entropy.updateProgress( 2089 "[repo:%s|%s] %s: %s" % ( 2090 brown(repo), 2091 red(_("sync")), 2092 blue(_("database sync forbidden")), 2093 red(_("dependencies_test() reported errors")), 2094 ), 2095 importance = 1, 2096 type = "error", 2097 header = darkred(" !!! ") 2098 ) 2099 return 3, set(), set() 2100 2101 problems = self.Entropy.check_config_file_updates() 2102 if problems: 2103 return 4, set(), set() 2104 2105 self.Entropy.updateProgress( 2106 "[repo:%s|%s] %s" % ( 2107 brown(repo), 2108 red(_("config files")), # something short please 2109 blue(_("no configuration files to commit. All fine.")), 2110 ), 2111 importance = 1, 2112 type = "info", 2113 header = blue(" @@ "), 2114 back = True 2115 ) 2116 2117 uris = [x[0] for x in upload_queue] 2118 errors, fine_uris, broken_uris = self.upload_database(uris, 2119 repo = repo) 2120 if errors: 2121 self.Entropy.updateProgress( 2122 "[repo:%s|%s] %s: %s" % ( 2123 brown(repo), 2124 red(_("sync")), 2125 blue(_("database sync failed")), 2126 red(_("upload issues")), 2127 ), 2128 importance = 1, 2129 type = "error", 2130 header = darkred(" !!! ") 2131 ) 2132 return 2, fine_uris, broken_uris 2133 2134 2135 self.Entropy.updateProgress( 2136 "[repo:%s|%s] %s" % ( 2137 brown(repo), 2138 red(_("sync")), 2139 blue(_("database sync completed successfully")), 2140 ), 2141 importance = 1, 2142 type = "info", 2143 header = darkgreen(" * ") 2144 ) 2145 2146 if unlock_mirrors: 2147 self.lock_mirrors(False, repo = repo) 2148 return 0, set(), set()

2149 2150

2151 - def calculate_local_upload_files(self, branch, repo = None):

2152 upload_files = 0 2153 upload_packages = set() 2154 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2155 branch) 2156 2157 # check if it exists 2158 if not os.path.isdir(upload_dir): 2159 return upload_files, upload_packages 2160 2161 pkg_ext = etpConst['packagesext'] 2162 pkg_md5_ext = etpConst['packagesmd5fileext'] 2163 for package in os.listdir(upload_dir): 2164 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2165 upload_packages.add(package) 2166 if package.endswith(pkg_ext): 2167 upload_files += 1 2168 2169 return upload_files, upload_packages

2170

2171 - def calculate_local_package_files(self, branch, repo = None):

2172 local_files = 0 2173 local_packages = set() 2174 packages_dir = os.path.join( 2175 self.Entropy.get_local_packages_directory(repo), branch) 2176 2177 if not os.path.isdir(packages_dir): 2178 os.makedirs(packages_dir) 2179 2180 pkg_ext = etpConst['packagesext'] 2181 pkg_md5_ext = etpConst['packagesmd5fileext'] 2182 for package in os.listdir(packages_dir): 2183 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2184 local_packages.add(package) 2185 if package.endswith(pkg_ext): 2186 local_files += 1 2187 2188 return local_files, local_packages

2189 2190

2191 - def _show_local_sync_stats(self, upload_files, local_files):

2192 self.Entropy.updateProgress( 2193 "%s:" % ( 2194 blue(_("Local statistics")), 2195 ), 2196 importance = 1, 2197 type = "info", 2198 header = red(" @@ ") 2199 ) 2200 self.Entropy.updateProgress( 2201 red("%s:\t\t%s %s" % ( 2202 blue(_("upload directory")), 2203 bold(str(upload_files)), 2204 red(_("files ready")), 2205 ) 2206 ), 2207 importance = 0, 2208 type = "info", 2209 header = red(" @@ ") 2210 ) 2211 self.Entropy.updateProgress( 2212 red("%s:\t\t%s %s" % ( 2213 blue(_("packages directory")), 2214 bold(str(local_files)), 2215 red(_("files ready")), 2216 ) 2217 ), 2218 importance = 0, 2219 type = "info", 2220 header = red(" @@ ") 2221 )

2222

2223 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2224 branch):

2225 2226 # show stats 2227 for package, size in upload: 2228 package = darkgreen(os.path.basename(package)) 2229 size = blue(self.entropyTools.bytes_into_human(size)) 2230 self.Entropy.updateProgress( 2231 "[branch:%s|%s] %s [%s]" % ( 2232 brown(branch), 2233 blue(_("upload")), 2234 darkgreen(package), 2235 size, 2236 ), 2237 importance = 0, 2238 type = "info", 2239 header = red(" # ") 2240 ) 2241 for package, size in download: 2242 package = darkred(os.path.basename(package)) 2243 size = blue(self.entropyTools.bytes_into_human(size)) 2244 self.Entropy.updateProgress( 2245 "[branch:%s|%s] %s [%s]" % ( 2246 brown(branch), 2247 darkred(_("download")), 2248 blue(package), 2249 size, 2250 ), 2251 importance = 0, 2252 type = "info", 2253 header = red(" # ") 2254 ) 2255 for package, size in copy: 2256 package = darkblue(os.path.basename(package)) 2257 size = blue(self.entropyTools.bytes_into_human(size)) 2258 self.Entropy.updateProgress( 2259 "[branch:%s|%s] %s [%s]" % ( 2260 brown(branch), 2261 darkgreen(_("copy")), 2262 brown(package), 2263 size, 2264 ), 2265 importance = 0, 2266 type = "info", 2267 header = red(" # ") 2268 ) 2269 for package, size in removal: 2270 package = brown(os.path.basename(package)) 2271 size = blue(self.entropyTools.bytes_into_human(size)) 2272 self.Entropy.updateProgress( 2273 "[branch:%s|%s] %s [%s]" % ( 2274 brown(branch), 2275 red(_("remove")), 2276 red(package), 2277 size, 2278 ), 2279 importance = 0, 2280 type = "info", 2281 header = red(" # ") 2282 ) 2283 2284 self.Entropy.updateProgress( 2285 "%s:\t\t\t%s" % ( 2286 blue(_("Packages to be removed")), 2287 darkred(str(len(removal))), 2288 ), 2289 importance = 0, 2290 type = "info", 2291 header = blue(" @@ ") 2292 ) 2293 self.Entropy.updateProgress( 2294 "%s:\t\t%s" % ( 2295 darkgreen(_("Packages to be moved locally")), 2296 darkgreen(str(len(copy))), 2297 ), 2298 importance = 0, 2299 type = "info", 2300 header = blue(" @@ ") 2301 ) 2302 self.Entropy.updateProgress( 2303 "%s:\t\t\t%s" % ( 2304 bold(_("Packages to be uploaded")), 2305 bold(str(len(upload))), 2306 ), 2307 importance = 0, 2308 type = "info", 2309 header = blue(" @@ ") 2310 ) 2311 2312 self.Entropy.updateProgress( 2313 "%s:\t\t\t%s" % ( 2314 darkred(_("Total removal size")), 2315 darkred( 2316 self.entropyTools.bytes_into_human(metainfo['removal']) 2317 ), 2318 ), 2319 importance = 0, 2320 type = "info", 2321 header = blue(" @@ ") 2322 ) 2323 2324 self.Entropy.updateProgress( 2325 "%s:\t\t\t%s" % ( 2326 blue(_("Total upload size")), 2327 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2328 ), 2329 importance = 0, 2330 type = "info", 2331 header = blue(" @@ ") 2332 ) 2333 self.Entropy.updateProgress( 2334 "%s:\t\t\t%s" % ( 2335 brown(_("Total download size")), 2336 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2337 ), 2338 importance = 0, 2339 type = "info", 2340 header = blue(" @@ ") 2341 )

2342 2343

2344 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2345 repo = None):

2346 2347 remote_files = 0 2348 close_conn = False 2349 remote_packages_data = {} 2350 2351 my_path = os.path.join( 2352 self.Entropy.get_remote_packages_relative_path(repo), branch) 2353 if ftp_connection == None: 2354 close_conn = True 2355 # let it raise an exception if things go bad 2356 ftp_connection = FtpInterface(uri, self.Entropy) 2357 ftp_connection.set_cwd(my_path, dodir = True) 2358 2359 remote_packages = ftp_connection.list_dir() 2360 remote_packages_info = ftp_connection.get_raw_list() 2361 if close_conn: 2362 ftp_connection.close() 2363 2364 for tbz2 in remote_packages: 2365 if tbz2.endswith(etpConst['packagesext']): 2366 remote_files += 1 2367 2368 for remote_package in remote_packages_info: 2369 remote_packages_data[remote_package.split()[8]] = \ 2370 int(remote_package.split()[4]) 2371 2372 return remote_files, remote_packages, remote_packages_data

2373

2374 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2375 2376 if repo == None: 2377 repo = self.Entropy.default_repository 2378 2379 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2380 upload_files, upload_packages = self.calculate_local_upload_files( 2381 branch, repo) 2382 local_files, local_packages = self.calculate_local_package_files(branch, 2383 repo) 2384 self._show_local_sync_stats(upload_files, local_files) 2385 2386 self.Entropy.updateProgress( 2387 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2388 importance = 1, 2389 type = "info", 2390 header = red(" @@ ") 2391 ) 2392 remote_files, remote_packages, remote_packages_data = \ 2393 self.calculate_remote_package_files(uri, branch, repo = repo) 2394 self.Entropy.updateProgress( 2395 "%s:\t\t\t%s %s" % ( 2396 blue(_("remote packages")), 2397 bold(str(remote_files)), 2398 red(_("files stored")), 2399 ), 2400 importance = 0, 2401 type = "info", 2402 header = red(" @@ ") 2403 ) 2404 2405 mytxt = blue("%s ...") % ( 2406 _("Calculating queues"), 2407 ) 2408 self.Entropy.updateProgress( 2409 mytxt, 2410 importance = 1, 2411 type = "info", 2412 header = red(" @@ ") 2413 ) 2414 2415 upload_queue, download_queue, removal_queue, fine_queue = \ 2416 self.calculate_sync_queues(upload_packages, local_packages, 2417 remote_packages, remote_packages_data, branch, repo) 2418 return upload_queue, download_queue, removal_queue, fine_queue, \ 2419 remote_packages_data

2420

2421 - def calculate_sync_queues( 2422 self, 2423 upload_packages, 2424 local_packages, 2425 remote_packages, 2426 remote_packages_data, 2427 branch, 2428 repo = None 2429 ):

2430 2431 upload_queue = set() 2432 download_queue = set() 2433 removal_queue = set() 2434 fine_queue = set() 2435 2436 for local_package in upload_packages: 2437 if local_package in remote_packages: 2438 2439 local_filepath = os.path.join( 2440 self.Entropy.get_local_upload_directory(repo), branch, 2441 local_package) 2442 2443 local_size = self.entropyTools.get_file_size(local_filepath) 2444 remote_size = remote_packages_data.get(local_package) 2445 if remote_size == None: 2446 remote_size = 0 2447 if (local_size != remote_size): 2448 # size does not match, adding to the upload queue 2449 upload_queue.add(local_package) 2450 else: 2451 # just move from upload to packages 2452 fine_queue.add(local_package) 2453 else: 2454 # always force upload of packages in uploaddir 2455 upload_queue.add(local_package) 2456 2457 # if a package is in the packages directory but not online, 2458 # we have to upload it we have local_packages and remote_packages 2459 for local_package in local_packages: 2460 if local_package in remote_packages: 2461 local_filepath = os.path.join( 2462 self.Entropy.get_local_packages_directory(repo), branch, 2463 local_package) 2464 local_size = self.entropyTools.get_file_size(local_filepath) 2465 remote_size = remote_packages_data.get(local_package) 2466 if remote_size == None: 2467 remote_size = 0 2468 if (local_size != remote_size) and (local_size != 0): 2469 # size does not match, adding to the upload queue 2470 if local_package not in fine_queue: 2471 upload_queue.add(local_package) 2472 else: 2473 # this means that the local package does not exist 2474 # so, we need to download it 2475 upload_queue.add(local_package) 2476 2477 # Fill download_queue and removal_queue 2478 for remote_package in remote_packages: 2479 if remote_package in local_packages: 2480 local_filepath = os.path.join( 2481 self.Entropy.get_local_packages_directory(repo), branch, 2482 remote_package) 2483 local_size = self.entropyTools.get_file_size(local_filepath) 2484 remote_size = remote_packages_data.get(remote_package) 2485 if remote_size == None: 2486 remote_size = 0 2487 if (local_size != remote_size) and (local_size != 0): 2488 # size does not match, remove first 2489 # do it only if the package has not been 2490 # added to the upload_queue 2491 if remote_package not in upload_queue: 2492 # remotePackage == localPackage 2493 # just remove something that differs 2494 # from the content of the mirror 2495 removal_queue.add(remote_package) 2496 # then add to the download queue 2497 download_queue.add(remote_package) 2498 else: 2499 # this means that the local package does not exist 2500 # so, we need to download it 2501 # ignore .tmp files 2502 if not remote_package.endswith(".tmp"): 2503 download_queue.add(remote_package) 2504 2505 # Collect packages that don't exist anymore in the database 2506 # so we can filter them out from the download queue 2507 dbconn = self.Entropy.open_server_repository(just_reading = True, 2508 repo = repo) 2509 db_files = dbconn.listAllDownloads(do_sort = False, 2510 full_path = True) 2511 db_files = set([os.path.basename(x) for x in db_files if \ 2512 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2513 2514 exclude = set() 2515 for myfile in download_queue: 2516 if myfile.endswith(etpConst['packagesext']): 2517 if myfile not in db_files: 2518 exclude.add(myfile) 2519 download_queue -= exclude 2520 2521 exclude = set() 2522 for myfile in upload_queue: 2523 if myfile.endswith(etpConst['packagesext']): 2524 if myfile not in db_files: 2525 exclude.add(myfile) 2526 upload_queue -= exclude 2527 2528 exclude = set() 2529 for myfile in download_queue: 2530 if myfile in upload_queue: 2531 exclude.add(myfile) 2532 download_queue -= exclude 2533 2534 return upload_queue, download_queue, removal_queue, fine_queue

2535 2536

2537 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2538 remote_packages_data, branch, repo):

2539 2540 metainfo = { 2541 'removal': 0, 2542 'download': 0, 2543 'upload': 0, 2544 } 2545 removal = [] 2546 download = [] 2547 do_copy = [] 2548 upload = [] 2549 2550 for item in removal_queue: 2551 if not item.endswith(etpConst['packagesext']): 2552 continue 2553 local_filepath = os.path.join( 2554 self.Entropy.get_local_packages_directory(repo), branch, item) 2555 size = self.entropyTools.get_file_size(local_filepath) 2556 metainfo['removal'] += size 2557 removal.append((local_filepath, size)) 2558 2559 for item in download_queue: 2560 if not item.endswith(etpConst['packagesext']): 2561 continue 2562 local_filepath = os.path.join( 2563 self.Entropy.get_local_upload_directory(repo), branch, item) 2564 if not os.path.isfile(local_filepath): 2565 size = remote_packages_data.get(item) 2566 if size == None: 2567 size = 0 2568 size = int(size) 2569 metainfo['removal'] += size 2570 download.append((local_filepath, size)) 2571 else: 2572 size = self.entropyTools.get_file_size(local_filepath) 2573 do_copy.append((local_filepath, size)) 2574 2575 for item in upload_queue: 2576 if not item.endswith(etpConst['packagesext']): 2577 continue 2578 local_filepath = os.path.join( 2579 self.Entropy.get_local_upload_directory(repo), branch, item) 2580 local_filepath_pkgs = os.path.join( 2581 self.Entropy.get_local_packages_directory(repo), branch, item) 2582 if os.path.isfile(local_filepath): 2583 size = self.entropyTools.get_file_size(local_filepath) 2584 upload.append((local_filepath, size)) 2585 else: 2586 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2587 upload.append((local_filepath_pkgs, size)) 2588 metainfo['upload'] += size 2589 2590 return upload, download, removal, do_copy, metainfo

2591 2592

2593 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2594 2595 if repo == None: 2596 repo = self.Entropy.default_repository 2597 2598 for itemdata in removal_queue: 2599 2600 remove_filename = itemdata[0] 2601 remove_filepath = os.path.join( 2602 self.Entropy.get_local_packages_directory(repo), branch, 2603 remove_filename) 2604 remove_filepath_hash = remove_filepath + \ 2605 etpConst['packagesmd5fileext'] 2606 self.Entropy.updateProgress( 2607 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2608 brown(repo), 2609 red("sync"), 2610 brown(branch), 2611 blue(_("removing package+hash")), 2612 darkgreen(remove_filename), 2613 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2614 ), 2615 importance = 0, 2616 type = "info", 2617 header = darkred(" * ") 2618 ) 2619 2620 if os.path.isfile(remove_filepath): 2621 os.remove(remove_filepath) 2622 if os.path.isfile(remove_filepath_hash): 2623 os.remove(remove_filepath_hash) 2624 2625 self.Entropy.updateProgress( 2626 "[repo:%s|%s|%s] %s" % ( 2627 brown(repo), 2628 red(_("sync")), 2629 brown(branch), 2630 blue(_("removal complete")), 2631 ), 2632 importance = 0, 2633 type = "info", 2634 header = darkred(" * ") 2635 )

2636 2637

2638 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2639 2640 if repo == None: 2641 repo = self.Entropy.default_repository 2642 2643 for itemdata in copy_queue: 2644 2645 from_file = itemdata[0] 2646 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2647 to_file = os.path.join( 2648 self.Entropy.get_local_packages_directory(repo), branch, 2649 os.path.basename(from_file)) 2650 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2651 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2652 self.Entropy.updateProgress( 2653 "[repo:%s|%s|%s] %s: %s" % ( 2654 brown(repo), 2655 red("sync"), 2656 brown(branch), 2657 blue(_("copying file+hash to repository")), 2658 darkgreen(from_file), 2659 ), 2660 importance = 0, 2661 type = "info", 2662 header = darkred(" * ") 2663 ) 2664 2665 if not os.path.isdir(os.path.dirname(to_file)): 2666 os.makedirs(os.path.dirname(to_file)) 2667 2668 shutil.copy2(from_file, to_file) 2669 if not os.path.isfile(from_file_hash): 2670 self.create_file_checksum(from_file, from_file_hash) 2671 shutil.copy2(from_file_hash, to_file_hash) 2672 2673 # clear expiration file 2674 if os.path.isfile(expiration_file): 2675 os.remove(expiration_file)

2676 2677

2678 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2679 2680 if repo == None: 2681 repo = self.Entropy.default_repository 2682 2683 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2684 myqueue = [] 2685 for itemdata in upload_queue: 2686 upload_item = itemdata[0] 2687 hash_file = upload_item + etpConst['packagesmd5fileext'] 2688 if not os.path.isfile(hash_file): 2689 self.entropyTools.create_md5_file(upload_item) 2690 myqueue.append(hash_file) 2691 myqueue.append(upload_item) 2692 2693 ftp_basedir = os.path.join( 2694 self.Entropy.get_remote_packages_relative_path(repo), branch) 2695 uploader = self.FtpServerHandler(FtpInterface, 2696 self.Entropy, [uri], myqueue, critical_files = myqueue, 2697 use_handlers = True, ftp_basedir = ftp_basedir, 2698 handlers_data = {'branch': branch }, repo = repo) 2699 errors, m_fine_uris, m_broken_uris = uploader.go() 2700 if errors: 2701 my_broken_uris = [ 2702 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2703 x in m_broken_uris] 2704 reason = my_broken_uris[0][1] 2705 self.Entropy.updateProgress( 2706 "[branch:%s] %s: %s, %s: %s" % ( 2707 brown(branch), 2708 blue(_("upload errors")), 2709 red(crippled_uri), 2710 blue(_("reason")), 2711 darkgreen(unicode(reason)), 2712 ), 2713 importance = 1, 2714 type = "error", 2715 header = darkred(" !!! ") 2716 ) 2717 return errors, m_fine_uris, m_broken_uris 2718 2719 self.Entropy.updateProgress( 2720 "[branch:%s] %s: %s" % ( 2721 brown(branch), 2722 blue(_("upload completed successfully")), 2723 red(crippled_uri), 2724 ), 2725 importance = 1, 2726 type = "info", 2727 header = blue(" @@ ") 2728 ) 2729 return errors, m_fine_uris, m_broken_uris

2730 2731

2732 - def _sync_run_download_queue(self, uri, download_queue, branch, 2733 repo = None):

2734 2735 if repo == None: 2736 repo = self.Entropy.default_repository 2737 2738 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2739 myqueue = [] 2740 for package in download_queue: 2741 hash_file = package + etpConst['packagesmd5fileext'] 2742 myqueue.append(package) 2743 myqueue.append(hash_file) 2744 2745 ftp_basedir = os.path.join( 2746 self.Entropy.get_remote_packages_relative_path(repo), branch) 2747 local_basedir = os.path.join( 2748 self.Entropy.get_local_packages_directory(repo), branch) 2749 downloader = self.FtpServerHandler( 2750 FtpInterface, self.Entropy, [uri], myqueue, 2751 critical_files = myqueue, use_handlers = True, 2752 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2753 handlers_data = {'branch': branch }, download = True, repo = repo) 2754 2755 errors, m_fine_uris, m_broken_uris = downloader.go() 2756 if errors: 2757 my_broken_uris = [ 2758 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2759 for x, y in m_broken_uris] 2760 reason = my_broken_uris[0][1] 2761 self.Entropy.updateProgress( 2762 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2763 brown(repo), 2764 red(_("sync")), 2765 brown(branch), 2766 blue(_("download errors")), 2767 darkgreen(crippled_uri), 2768 blue(_("reason")), 2769 reason, 2770 ), 2771 importance = 1, 2772 type = "error", 2773 header = darkred(" !!! ") 2774 ) 2775 return errors, m_fine_uris, m_broken_uris 2776 2777 self.Entropy.updateProgress( 2778 "[repo:%s|%s|%s] %s: %s" % ( 2779 brown(repo), 2780 red(_("sync")), 2781 brown(branch), 2782 blue(_("download completed successfully")), 2783 darkgreen(crippled_uri), 2784 ), 2785 importance = 1, 2786 type = "info", 2787 header = darkgreen(" * ") 2788 ) 2789 return errors, m_fine_uris, m_broken_uris

2790

2791 - def run_package_files_qa_checks(self, packages_list, repo = None):

2792 2793 if repo == None: 2794 repo = self.Entropy.default_repository 2795 2796 my_qa = self.Entropy.QA() 2797 2798 qa_total = len(packages_list) 2799 qa_count = 0 2800 qa_some_faulty = [] 2801 2802 for upload_package in packages_list: 2803 qa_count += 1 2804 2805 self.Entropy.updateProgress( 2806 "%s: %s" % ( 2807 purple(_("QA checking package file")), 2808 darkgreen(os.path.basename(upload_package)), 2809 ), 2810 importance = 0, 2811 type = "info", 2812 header = purple(" @@ "), 2813 back = True, 2814 count = (qa_count, qa_total,) 2815 ) 2816 2817 result = my_qa.entropy_package_checks(upload_package) 2818 if not result: 2819 # call wolfman-911 2820 qa_some_faulty.append(os.path.basename(upload_package)) 2821 2822 if qa_some_faulty: 2823 2824 for qa_faulty_pkg in qa_some_faulty: 2825 self.Entropy.updateProgress( 2826 "[repo:%s|branch:%s] %s: %s" % ( 2827 brown(repo), 2828 self.SystemSettings['repositories']['branch'], 2829 red(_("faulty package file, please fix")), 2830 blue(os.path.basename(qa_faulty_pkg)), 2831 ), 2832 importance = 1, 2833 type = "error", 2834 header = darkred(" @@ ") 2835 ) 2836 raise EntropyPackageException( 2837 'EntropyPackageException: cannot continue')

2838 2839

2840 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2841 repo = None):

2842 2843 if repo == None: 2844 repo = self.Entropy.default_repository 2845 2846 self.Entropy.updateProgress( 2847 "[repo:%s|%s] %s" % ( 2848 repo, 2849 red(_("sync")), 2850 darkgreen(_("starting packages sync")), 2851 ), 2852 importance = 1, 2853 type = "info", 2854 header = red(" @@ "), 2855 back = True 2856 ) 2857 2858 2859 successfull_mirrors = set() 2860 broken_mirrors = set() 2861 check_data = () 2862 upload_queue_qa_checked = set() 2863 mirrors_tainted = False 2864 mirror_errors = False 2865 mirrors_errors = False 2866 2867 for uri in self.Entropy.get_remote_mirrors(repo): 2868 2869 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2870 mirror_errors = False 2871 2872 self.Entropy.updateProgress( 2873 "[repo:%s|%s|branch:%s] %s: %s" % ( 2874 repo, 2875 red(_("sync")), 2876 brown(self.SystemSettings['repositories']['branch']), 2877 blue(_("packages sync")), 2878 bold(crippled_uri), 2879 ), 2880 importance = 1, 2881 type = "info", 2882 header = red(" @@ ") 2883 ) 2884 2885 try: 2886 upload_queue, download_queue, removal_queue, fine_queue, \ 2887 remote_packages_data = self.calculate_packages_to_sync(uri, 2888 self.SystemSettings['repositories']['branch'], repo) 2889 except self.socket.error, err: 2890 self.Entropy.updateProgress( 2891 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2892 repo, 2893 red(_("sync")), 2894 self.SystemSettings['repositories']['branch'], 2895 darkred(_("socket error")), 2896 err, 2897 darkred(_("on")), 2898 crippled_uri, 2899 ), 2900 importance = 1, 2901 type = "error", 2902 header = darkgreen(" * ") 2903 ) 2904 continue 2905 2906 if (not upload_queue) and (not download_queue) and \ 2907 (not removal_queue): 2908 self.Entropy.updateProgress( 2909 "[repo:%s|%s|branch:%s] %s: %s" % ( 2910 repo, 2911 red(_("sync")), 2912 self.SystemSettings['repositories']['branch'], 2913 darkgreen(_("nothing to do on")), 2914 crippled_uri, 2915 ), 2916 importance = 1, 2917 type = "info", 2918 header = darkgreen(" * ") 2919 ) 2920 successfull_mirrors.add(uri) 2921 continue 2922 2923 self.Entropy.updateProgress( 2924 "%s:" % (blue(_("Expanding queues")),), 2925 importance = 1, 2926 type = "info", 2927 header = red(" ** ") 2928 ) 2929 2930 upload, download, removal, copy, metainfo = self.expand_queues( 2931 upload_queue, 2932 download_queue, 2933 removal_queue, 2934 remote_packages_data, 2935 self.SystemSettings['repositories']['branch'], 2936 repo 2937 ) 2938 del upload_queue, download_queue, removal_queue, \ 2939 remote_packages_data 2940 self._show_sync_queues(upload, download, removal, copy, metainfo, 2941 self.SystemSettings['repositories']['branch']) 2942 2943 if not len(upload)+len(download)+len(removal)+len(copy): 2944 2945 self.Entropy.updateProgress( 2946 "[repo:%s|%s|branch:%s] %s %s" % ( 2947 self.Entropy.default_repository, 2948 red(_("sync")), 2949 self.SystemSettings['repositories']['branch'], 2950 blue(_("nothing to sync for")), 2951 crippled_uri, 2952 ), 2953 importance = 1, 2954 type = "info", 2955 header = darkgreen(" @@ ") 2956 ) 2957 2958 successfull_mirrors.add(uri) 2959 continue 2960 2961 if pretend: 2962 successfull_mirrors.add(uri) 2963 continue 2964 2965 if ask: 2966 rc_sync = self.Entropy.askQuestion( 2967 _("Would you like to run the steps above ?")) 2968 if rc_sync == "No": 2969 continue 2970 2971 try: 2972 2973 # QA checks 2974 qa_package_files = [x[0] for x in upload if x[0] \ 2975 not in upload_queue_qa_checked] 2976 upload_queue_qa_checked |= set(qa_package_files) 2977 2978 self.run_package_files_qa_checks(qa_package_files, repo = repo) 2979 2980 if removal: 2981 self._sync_run_removal_queue(removal, 2982 self.SystemSettings['repositories']['branch'], repo) 2983 2984 if copy: 2985 self._sync_run_copy_queue(copy, 2986 self.SystemSettings['repositories']['branch'], repo) 2987 2988 if upload or download: 2989 mirrors_tainted = True 2990 2991 if upload: 2992 d_errors, m_fine_uris, \ 2993 m_broken_uris = self._sync_run_upload_queue( 2994 uri, upload, 2995 self.SystemSettings['repositories']['branch'], repo) 2996 2997 if d_errors: 2998 mirror_errors = True 2999 3000 if download: 3001 my_downlist = [x[0] for x in download] 3002 d_errors, m_fine_uris, \ 3003 m_broken_uris = self._sync_run_download_queue( 3004 uri, my_downlist, 3005 self.SystemSettings['repositories']['branch'], repo) 3006 3007 if d_errors: 3008 mirror_errors = True 3009 if not mirror_errors: 3010 successfull_mirrors.add(uri) 3011 else: 3012 mirrors_errors = True 3013 3014 except KeyboardInterrupt: 3015 self.Entropy.updateProgress( 3016 "[repo:%s|%s|branch:%s] %s" % ( 3017 repo, 3018 red(_("sync")), 3019 self.SystemSettings['repositories']['branch'], 3020 darkgreen(_("keyboard interrupt !")), 3021 ), 3022 importance = 1, 3023 type = "info", 3024 header = darkgreen(" * ") 3025 ) 3026 continue 3027 3028 except EntropyPackageException: 3029 3030 mirrors_errors = True 3031 broken_mirrors.add(uri) 3032 successfull_mirrors.clear() 3033 # so that people will realize this is a very bad thing 3034 self.Entropy.updateProgress( 3035 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3036 repo, 3037 red(_("sync")), 3038 self.SystemSettings['repositories']['branch'], 3039 darkred(_("you must package them again")), 3040 Exception, 3041 _("error"), 3042 err, 3043 ), 3044 importance = 1, 3045 type = "error", 3046 header = darkred(" !!! ") 3047 ) 3048 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3049 broken_mirrors, check_data 3050 3051 except Exception, err: 3052 3053 self.entropyTools.print_traceback() 3054 mirrors_errors = True 3055 broken_mirrors.add(uri) 3056 self.Entropy.updateProgress( 3057 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3058 repo, 3059 red(_("sync")), 3060 self.SystemSettings['repositories']['branch'], 3061 darkred(_("exception caught")), 3062 Exception, 3063 _("error"), 3064 err, 3065 ), 3066 importance = 1, 3067 type = "error", 3068 header = darkred(" !!! ") 3069 ) 3070 3071 exc_txt = self.Entropy.entropyTools.print_exception( 3072 returndata = True) 3073 for line in exc_txt: 3074 self.Entropy.updateProgress( 3075 unicode(line), 3076 importance = 1, 3077 type = "error", 3078 header = darkred(": ") 3079 ) 3080 3081 if len(successfull_mirrors) > 0: 3082 self.Entropy.updateProgress( 3083 "[repo:%s|%s|branch:%s] %s" % ( 3084 repo, 3085 red(_("sync")), 3086 self.SystemSettings['repositories']['branch'], 3087 darkred( 3088 _("at least one mirror synced properly!")), 3089 ), 3090 importance = 1, 3091 type = "error", 3092 header = darkred(" !!! ") 3093 ) 3094 continue 3095 3096 # if at least one server has been synced successfully, move files 3097 if (len(successfull_mirrors) > 0) and not pretend: 3098 self.remove_expiration_files( 3099 self.SystemSettings['repositories']['branch'], repo) 3100 3101 if packages_check: 3102 check_data = self.Entropy.verify_local_packages([], ask = ask, 3103 repo = repo) 3104 3105 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3106 broken_mirrors, check_data

3107

3108 - def remove_expiration_files(self, branch, repo = None):

3109 3110 if repo == None: 3111 repo = self.Entropy.default_repository 3112 3113 branch_dir = os.path.join( 3114 self.Entropy.get_local_upload_directory(repo), branch) 3115 3116 # check if it exists 3117 if not os.path.isdir(branch_dir): 3118 return None 3119 3120 branchcontent = os.listdir(branch_dir) 3121 for xfile in branchcontent: 3122 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3123 branch, xfile) 3124 destdir = os.path.join( 3125 self.Entropy.get_local_packages_directory(repo), branch) 3126 if not os.path.isdir(destdir): 3127 os.makedirs(destdir) 3128 dest = os.path.join(destdir, xfile) 3129 shutil.move(source, dest) 3130 # clear expiration file 3131 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3132 if os.path.isfile(dest_expiration): 3133 os.remove(dest_expiration)

3134 3135

3136 - def is_package_expired(self, package_file, branch, repo = None):

3137 3138 pkg_path = os.path.join( 3139 self.Entropy.get_local_packages_directory(repo), branch, 3140 package_file) 3141 pkg_path += etpConst['packagesexpirationfileext'] 3142 if not os.path.isfile(pkg_path): 3143 return False 3144 3145 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3146 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3147 days = srv_set['packages_expiration_days'] 3148 delta = int(days)*24*3600 3149 currmtime = time.time() 3150 file_delta = currmtime - mtime 3151 3152 if file_delta > delta: 3153 return True 3154 return False

3155

3156 - def create_expiration_file(self, package_file, branch, repo = None, 3157 gentle = False):

3158 3159 pkg_path = os.path.join( 3160 self.Entropy.get_local_packages_directory(repo), branch, 3161 package_file) 3162 pkg_path += etpConst['packagesexpirationfileext'] 3163 if gentle and os.path.isfile(pkg_path): 3164 return 3165 f_exp = open(pkg_path,"w") 3166 f_exp.flush() 3167 f_exp.close()

3168 3169

3170 - def collect_expiring_packages(self, branch, repo = None):

3171 3172 dbconn = self.Entropy.open_server_repository(just_reading = True, 3173 repo = repo) 3174 database_bins = dbconn.listAllDownloads(do_sort = False, 3175 full_path = True) 3176 bins_dir = os.path.join( 3177 self.Entropy.get_local_packages_directory(repo), branch) 3178 repo_bins = set() 3179 3180 if os.path.isdir(bins_dir): 3181 repo_bins = os.listdir(bins_dir) 3182 repo_bins = set([ 3183 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3184 in repo_bins if x.endswith(etpConst['packagesext'])]) 3185 repo_bins -= database_bins 3186 3187 return set([os.path.basename(x) for x in repo_bins])

3188 3189

3190 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3191 3192 if repo == None: 3193 repo = self.Entropy.default_repository 3194 3195 self.Entropy.updateProgress( 3196 "[repo:%s|%s|branch:%s] %s" % ( 3197 brown(repo), 3198 red(_("tidy")), 3199 blue(self.SystemSettings['repositories']['branch']), 3200 blue(_("collecting expired packages")), 3201 ), 3202 importance = 1, 3203 type = "info", 3204 header = red(" @@ ") 3205 ) 3206 3207 branch_data = {} 3208 errors = False 3209 branch_data['errors'] = False 3210 branch = self.SystemSettings['repositories']['branch'] 3211 3212 self.Entropy.updateProgress( 3213 "[branch:%s] %s" % ( 3214 brown(branch), 3215 blue(_("collecting expired packages in the selected branches")), 3216 ), 3217 importance = 1, 3218 type = "info", 3219 header = blue(" @@ ") 3220 ) 3221 3222 # collect removed packages 3223 expiring_packages = self.collect_expiring_packages(branch, repo) 3224 if expiring_packages: 3225 3226 # filter expired packages used by other branches 3227 # this is done for the sake of consistency 3228 # --- read packages.db.pkglist, make sure your repository 3229 # has been ported to latest Entropy 3230 3231 branch_pkglist_data = self.read_remote_file_in_branches( 3232 etpConst['etpdatabasepkglist'], repo = repo, 3233 excluded_branches = [branch]) 3234 # format data 3235 for key, val in branch_pkglist_data.items(): 3236 branch_pkglist_data[key] = val.split("\n") 3237 3238 3239 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3240 branch) 3241 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3242 expiring_packages]) 3243 3244 for other_branch in branch_pkglist_data: 3245 branch_pkglist = set(branch_pkglist_data[other_branch]) 3246 my_expiring_pkgs -= branch_pkglist 3247 3248 # fallback to normality, set new expiring packages var 3249 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3250 3251 removal = [] 3252 for package in expiring_packages: 3253 expired = self.is_package_expired(package, branch, repo) 3254 if expired: 3255 removal.append(package) 3256 else: 3257 self.create_expiration_file(package, branch, repo, 3258 gentle = True) 3259 3260 # fill returning data 3261 branch_data['removal'] = removal[:] 3262 3263 if not removal: 3264 self.Entropy.updateProgress( 3265 "[branch:%s] %s" % ( 3266 brown(branch), 3267 blue(_("nothing to remove on this branch")), 3268 ), 3269 importance = 1, 3270 type = "info", 3271 header = blue(" @@ ") 3272 ) 3273 return errors, branch_data 3274 else: 3275 self.Entropy.updateProgress( 3276 "[branch:%s] %s:" % ( 3277 brown(branch), 3278 blue(_("these are the expired packages")), 3279 ), 3280 importance = 1, 3281 type = "info", 3282 header = blue(" @@ ") 3283 ) 3284 for package in removal: 3285 self.Entropy.updateProgress( 3286 "[branch:%s] %s: %s" % ( 3287 brown(branch), 3288 blue(_("remove")), 3289 darkgreen(package), 3290 ), 3291 importance = 1, 3292 type = "info", 3293 header = brown(" # ") 3294 ) 3295 3296 if pretend: 3297 return errors, branch_data 3298 3299 if ask: 3300 rc_question = self.Entropy.askQuestion( 3301 _("Would you like to continue ?")) 3302 if rc_question == "No": 3303 return errors, branch_data 3304 3305 myqueue = [] 3306 for package in removal: 3307 myqueue.append(package+etpConst['packagesmd5fileext']) 3308 myqueue.append(package) 3309 3310 ftp_basedir = os.path.join( 3311 self.Entropy.get_remote_packages_relative_path(repo), branch) 3312 for uri in self.Entropy.get_remote_mirrors(repo): 3313 3314 self.Entropy.updateProgress( 3315 "[branch:%s] %s..." % ( 3316 brown(branch), 3317 blue(_("removing packages remotely")), 3318 ), 3319 importance = 1, 3320 type = "info", 3321 header = blue(" @@ ") 3322 ) 3323 3324 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3325 destroyer = self.FtpServerHandler( 3326 FtpInterface, 3327 self.Entropy, 3328 [uri], 3329 myqueue, 3330 critical_files = [], 3331 ftp_basedir = ftp_basedir, 3332 remove = True, 3333 repo = repo 3334 ) 3335 errors, m_fine_uris, m_broken_uris = destroyer.go() 3336 if errors: 3337 my_broken_uris = [ 3338 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3339 for x in m_broken_uris] 3340 3341 reason = my_broken_uris[0][1] 3342 self.Entropy.updateProgress( 3343 "[branch:%s] %s: %s, %s: %s" % ( 3344 brown(branch), 3345 blue(_("remove errors")), 3346 red(crippled_uri), 3347 blue(_("reason")), 3348 reason, 3349 ), 3350 importance = 1, 3351 type = "warning", 3352 header = brown(" !!! ") 3353 ) 3354 branch_data['errors'] = True 3355 errors = True 3356 3357 self.Entropy.updateProgress( 3358 "[branch:%s] %s..." % ( 3359 brown(branch), 3360 blue(_("removing packages locally")), 3361 ), 3362 importance = 1, 3363 type = "info", 3364 header = blue(" @@ ") 3365 ) 3366 3367 branch_data['removed'] = set() 3368 for package in removal: 3369 package_path = os.path.join( 3370 self.Entropy.get_local_packages_directory(repo), 3371 branch, package) 3372 package_path_hash = package_path + \ 3373 etpConst['packagesmd5fileext'] 3374 package_path_expired = package_path + \ 3375 etpConst['packagesexpirationfileext'] 3376 3377 my_rm_list = (package_path_hash, package_path, 3378 package_path_expired) 3379 for myfile in my_rm_list: 3380 if os.path.isfile(myfile): 3381 self.Entropy.updateProgress( 3382 "[branch:%s] %s: %s" % ( 3383 brown(branch), 3384 blue(_("removing")), 3385 darkgreen(myfile), 3386 ), 3387 importance = 1, 3388 type = "info", 3389 header = brown(" @@ ") 3390 ) 3391 os.remove(myfile) 3392 branch_data['removed'].add(myfile) 3393 3394 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors