entropy.server.interfaces.mirrors

39 40 import socket 41 import entropy.dump as dumpTools 42 import entropy.tools as entropyTools

43 - def __init__(self, ServerInstance, repo = None):

44 45 from entropy.server.interfaces.main import Server as MainServer 46 if not isinstance(ServerInstance, MainServer): 47 mytxt = _("entropy.server.interfaces.main.Server needed") 48 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 49 50 self.Entropy = ServerInstance 51 from entropy.transceivers import FtpServerHandler 52 self.FtpServerHandler = FtpServerHandler 53 from entropy.cache import EntropyCacher 54 self.Cacher = EntropyCacher() 55 self.sys_settings_plugin_id = \ 56 etpConst['system_settings_plugins_ids']['server_plugin'] 57 self.SystemSettings = self.Entropy.SystemSettings 58 59 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 60 self.Entropy.updateProgress( 61 mytxt, 62 importance = 2, 63 type = "info", 64 header = red(" @@ ") 65 ) 66 mytxt = _("mirror") 67 for mirror in self.Entropy.get_remote_mirrors(repo): 68 mirror = self.entropyTools.hide_ftp_password(mirror) 69 self.Entropy.updateProgress( 70 blue("%s: %s") % (mytxt, darkgreen(mirror),), 71 importance = 0, 72 type = "info", 73 header = brown(" # ") 74 )

75

76 - def get_remote_branches(self, repo = None):

77 """ 78 Returns a list of remotely available branches for the provided 79 repository identifier. 80 81 @keyword repo: repository identifier 82 @type repo: string 83 @return: list of valid, available remote branches 84 @rtype: set 85 """ 86 87 if repo == None: 88 repo = self.Entropy.default_repository 89 90 remote_branches = set() 91 # this is used to validate Entropy repository path 92 ts_file = etpConst['etpdatabasetimestampfile'] 93 mirrors = self.Entropy.get_remote_mirrors(repo) 94 for uri in mirrors: 95 96 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 97 98 self.Entropy.updateProgress( 99 "[repo:%s] %s: %s" % ( 100 brown(repo), 101 blue(_("listing branches in mirror")), 102 darkgreen(crippled_uri), 103 ), 104 importance = 1, 105 type = "info", 106 header = brown(" @@ ") 107 ) 108 109 ftp = FtpInterface(uri, self.Entropy, verbose = False) 110 try: 111 112 branches_path = self.Entropy.get_remote_database_relative_path( 113 repo) 114 try: 115 ftp.set_cwd(branches_path) 116 except FtpError: 117 continue # no branches in this mirror 118 119 branches = ftp.list_dir() 120 for branch in branches: 121 mypath = os.path.join("/", branches_path, branch) 122 try: 123 ftp.set_cwd(mypath) 124 except FtpError: 125 # not a dir 126 continue 127 if ftp.is_file_available(ts_file): 128 remote_branches.add(branch) 129 130 finally: 131 ftp.close() 132 133 return remote_branches

134

135 - def read_remote_file_in_branches(self, filename, repo = None, 136 excluded_branches = None):

137 """ 138 Reads a file remotely located in all the available branches. 139 140 @param filename: name of the file that should be located inside 141 repository database directory 142 @type filename: string 143 @keyword repo: repository identifier 144 @type repo: string 145 @keyword excluded_branches: list of branch identifiers excluded or None 146 @type excluded_branches: list or None 147 @return: dictionary with branches as key and raw file content as value: 148 {'4': 'abcd\n', '5': 'defg\n'} 149 @rtype: dict 150 """ 151 if repo == None: 152 repo = self.Entropy.default_repository 153 if excluded_branches == None: 154 excluded_branches = [] 155 156 branch_data = {} 157 mirrors = self.Entropy.get_remote_mirrors(repo) 158 for uri in mirrors: 159 160 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 161 162 self.Entropy.updateProgress( 163 "[repo:%s] %s: %s => %s" % ( 164 brown(repo), 165 blue(_("looking for file in mirror")), 166 darkgreen(crippled_uri), 167 filename, 168 ), 169 importance = 1, 170 type = "info", 171 header = brown(" @@ ") 172 ) 173 174 ftp = FtpInterface(uri, self.Entropy, verbose = False) 175 try: 176 177 branches_path = self.Entropy.get_remote_database_relative_path( 178 repo) 179 try: 180 ftp.set_cwd(branches_path) 181 except FtpError: 182 continue # no branches in this mirror 183 184 branches = ftp.list_dir() 185 for branch in branches: 186 187 # is branch excluded ? 188 if branch in excluded_branches: 189 continue 190 191 if branch_data.get(branch) != None: 192 # already read 193 continue 194 195 mypath = os.path.join("/", branches_path, branch) 196 try: 197 ftp.set_cwd(mypath) 198 except FtpError: 199 # not a dir 200 continue 201 202 if not ftp.is_file_available(filename): 203 # nothing to do, no file here 204 continue 205 206 tmp_dir = tempfile.mkdtemp() 207 tries = 4 208 success = False 209 while tries: 210 downloaded = ftp.download_file(filename, tmp_dir) 211 if not downloaded: 212 tries -= 1 213 continue # argh! 214 success = True 215 break 216 217 down_path = os.path.join(tmp_dir, filename) 218 if success and os.access(down_path, os.F_OK): 219 down_f = open(down_path) 220 branch_data[branch] = down_f.read() 221 down_f.close() 222 shutil.rmtree(tmp_dir) 223 224 finally: 225 ftp.close() 226 227 return branch_data

228

229 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

230 231 if repo == None: 232 repo = self.Entropy.default_repository 233 234 if not mirrors: 235 mirrors = self.Entropy.get_remote_mirrors(repo) 236 237 issues = False 238 for uri in mirrors: 239 240 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 241 242 lock_text = _("unlocking") 243 if lock: 244 lock_text = _("locking") 245 self.Entropy.updateProgress( 246 "[repo:%s|%s] %s %s" % ( 247 brown(repo), 248 darkgreen(crippled_uri), 249 bold(lock_text), 250 blue("%s...") % (_("mirror"),), 251 ), 252 importance = 1, 253 type = "info", 254 header = brown(" * "), 255 back = True 256 ) 257 258 try: 259 ftp = FtpInterface(uri, self.Entropy) 260 except ConnectionError: 261 self.entropyTools.print_traceback() 262 return True # issues 263 my_path = os.path.join( 264 self.Entropy.get_remote_database_relative_path(repo), 265 self.SystemSettings['repositories']['branch']) 266 ftp.set_cwd(my_path, dodir = True) 267 268 lock_file = etpConst['etpdatabaselockfile'] 269 if lock and ftp.is_file_available(lock_file): 270 self.Entropy.updateProgress( 271 "[repo:%s|%s] %s" % ( 272 brown(repo), 273 darkgreen(crippled_uri), 274 blue(_("mirror already locked")), 275 ), 276 importance = 1, 277 type = "info", 278 header = darkgreen(" * ") 279 ) 280 ftp.close() 281 continue 282 elif not lock and not ftp.is_file_available(lock_file): 283 self.Entropy.updateProgress( 284 "[repo:%s|%s] %s" % ( 285 brown(repo), 286 darkgreen(crippled_uri), 287 blue(_("mirror already unlocked")), 288 ), 289 importance = 1, 290 type = "info", 291 header = darkgreen(" * ") 292 ) 293 ftp.close() 294 continue 295 296 if lock: 297 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 298 else: 299 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 300 ftp.close() 301 if not rc_lock: 302 issues = True 303 304 if not issues: 305 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 306 if os.path.isfile(db_taint_file): 307 os.remove(db_taint_file) 308 309 return issues

310 311

312 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 313 repo = None):

314 # this functions makes entropy clients to not download anything 315 # from the chosen mirrors. it is used to avoid clients to 316 # download databases while we're uploading a new one 317 318 if mirrors == None: 319 mirrors = [] 320 321 if repo == None: 322 repo = self.Entropy.default_repository 323 324 if not mirrors: 325 mirrors = self.Entropy.get_remote_mirrors(repo) 326 327 issues = False 328 for uri in mirrors: 329 330 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 331 332 lock_text = _("unlocking") 333 if lock: 334 lock_text = _("locking") 335 self.Entropy.updateProgress( 336 "[repo:%s|%s] %s %s..." % ( 337 blue(repo), 338 red(crippled_uri), 339 bold(lock_text), 340 blue(_("mirror for download")), 341 ), 342 importance = 1, 343 type = "info", 344 header = red(" @@ "), 345 back = True 346 ) 347 348 try: 349 ftp = FtpInterface(uri, self.Entropy) 350 except ConnectionError: 351 self.entropyTools.print_traceback() 352 return True # issues 353 my_path = os.path.join( 354 self.Entropy.get_remote_database_relative_path(repo), 355 self.SystemSettings['repositories']['branch']) 356 ftp.set_cwd(my_path, dodir = True) 357 358 lock_file = etpConst['etpdatabasedownloadlockfile'] 359 360 if lock and ftp.is_file_available(lock_file): 361 self.Entropy.updateProgress( 362 "[repo:%s|%s] %s" % ( 363 blue(repo), 364 red(crippled_uri), 365 blue(_("mirror already locked for download")), 366 ), 367 importance = 1, 368 type = "info", 369 header = red(" @@ ") 370 ) 371 ftp.close() 372 continue 373 374 elif not lock and not ftp.is_file_available(lock_file): 375 self.Entropy.updateProgress( 376 "[repo:%s|%s] %s" % ( 377 blue(repo), 378 red(crippled_uri), 379 blue(_("mirror already unlocked for download")), 380 ), 381 importance = 1, 382 type = "info", 383 header = red(" @@ ") 384 ) 385 ftp.close() 386 continue 387 388 if lock: 389 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 390 repo = repo) 391 else: 392 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 393 repo = repo) 394 ftp.close() 395 if not rc_lock: 396 issues = True 397 398 return issues

399

400 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 401 repo = None):

402 403 if repo == None: 404 repo = self.Entropy.default_repository 405 406 my_path = os.path.join( 407 self.Entropy.get_remote_database_relative_path(repo), 408 self.SystemSettings['repositories']['branch']) 409 if not ftp_connection: 410 try: 411 ftp_connection = FtpInterface(uri, self.Entropy) 412 except ConnectionError: 413 self.entropyTools.print_traceback() 414 return False # issues 415 ftp_connection.set_cwd(my_path, dodir = True) 416 else: 417 mycwd = ftp_connection.get_cwd() 418 if mycwd != my_path: 419 ftp_connection.set_basedir() 420 ftp_connection.set_cwd(my_path, dodir = True) 421 422 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 423 lock_string = u'' 424 425 if dblock: 426 self.create_local_database_lockfile(repo) 427 lock_file = self.get_database_lockfile(repo) 428 else: 429 # locking/unlocking mirror1 for download 430 lock_string = _('for download') 431 self.create_local_database_download_lockfile(repo) 432 lock_file = self.get_database_download_lockfile(repo) 433 434 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 435 if rc_upload: 436 self.Entropy.updateProgress( 437 "[repo:%s|%s] %s %s" % ( 438 blue(repo), 439 red(crippled_uri), 440 blue(_("mirror successfully locked")), 441 blue(lock_string), 442 ), 443 importance = 1, 444 type = "info", 445 header = red(" @@ ") 446 ) 447 else: 448 self.Entropy.updateProgress( 449 "[repo:%s|%s] %s: %s - %s %s" % ( 450 blue(repo), 451 red(crippled_uri), 452 blue("lock error"), 453 rc_upload, 454 blue(_("mirror not locked")), 455 blue(lock_string), 456 ), 457 importance = 1, 458 type = "error", 459 header = darkred(" * ") 460 ) 461 self.remove_local_database_lockfile(repo) 462 463 return rc_upload

464 465

466 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

467 468 if repo == None: 469 repo = self.Entropy.default_repository 470 471 my_path = os.path.join( 472 self.Entropy.get_remote_database_relative_path(repo), 473 self.SystemSettings['repositories']['branch']) 474 if not ftp_connection: 475 try: 476 ftp_connection = FtpInterface(uri, self.Entropy) 477 except ConnectionError: 478 self.entropyTools.print_traceback() 479 return False # issues 480 ftp_connection.set_cwd(my_path) 481 else: 482 mycwd = ftp_connection.get_cwd() 483 if mycwd != my_path: 484 ftp_connection.set_basedir() 485 ftp_connection.set_cwd(my_path) 486 487 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 488 489 if dblock: 490 dbfile = etpConst['etpdatabaselockfile'] 491 else: 492 dbfile = etpConst['etpdatabasedownloadlockfile'] 493 rc_delete = ftp_connection.delete_file(dbfile) 494 if rc_delete: 495 self.Entropy.updateProgress( 496 "[repo:%s|%s] %s" % ( 497 blue(repo), 498 red(crippled_uri), 499 blue(_("mirror successfully unlocked")), 500 ), 501 importance = 1, 502 type = "info", 503 header = darkgreen(" * ") 504 ) 505 if dblock: 506 self.remove_local_database_lockfile(repo) 507 else: 508 self.remove_local_database_download_lockfile(repo) 509 else: 510 self.Entropy.updateProgress( 511 "[repo:%s|%s] %s: %s - %s" % ( 512 blue(repo), 513 red(crippled_uri), 514 blue(_("unlock error")), 515 rc_delete, 516 blue(_("mirror not unlocked")), 517 ), 518 importance = 1, 519 type = "error", 520 header = darkred(" * ") 521 ) 522 523 return rc_delete

524

525 - def get_database_lockfile(self, repo = None):

526 if repo == None: 527 repo = self.Entropy.default_repository 528 return os.path.join(self.Entropy.get_local_database_dir(repo), 529 etpConst['etpdatabaselockfile'])

530

531 - def get_database_download_lockfile(self, repo = None):

532 if repo == None: 533 repo = self.Entropy.default_repository 534 return os.path.join(self.Entropy.get_local_database_dir(repo), 535 etpConst['etpdatabasedownloadlockfile'])

536

537 - def create_local_database_download_lockfile(self, repo = None):

538 if repo == None: 539 repo = self.Entropy.default_repository 540 lock_file = self.get_database_download_lockfile(repo) 541 f_lock = open(lock_file, "w") 542 f_lock.write("download locked") 543 f_lock.flush() 544 f_lock.close()

545

546 - def create_local_database_lockfile(self, repo = None):

547 if repo == None: 548 repo = self.Entropy.default_repository 549 lock_file = self.get_database_lockfile(repo) 550 f_lock = open(lock_file, "w") 551 f_lock.write("database locked") 552 f_lock.flush() 553 f_lock.close()

554

555 - def remove_local_database_lockfile(self, repo = None):

556 if repo == None: 557 repo = self.Entropy.default_repository 558 lock_file = self.get_database_lockfile(repo) 559 if os.path.isfile(lock_file): 560 os.remove(lock_file)

561

562 - def remove_local_database_download_lockfile(self, repo = None):

563 if repo == None: 564 repo = self.Entropy.default_repository 565 lock_file = self.get_database_download_lockfile(repo) 566 if os.path.isfile(lock_file): 567 os.remove(lock_file)

568

569 - def download_package(self, uri, pkg_relative_path, repo = None):

570 571 if repo == None: 572 repo = self.Entropy.default_repository 573 574 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 575 576 tries = 0 577 578 try: 579 ftp = FtpInterface(uri, self.Entropy) 580 except ConnectionError: 581 self.entropyTools.print_traceback() 582 return False # issues 583 584 while tries < 5: 585 tries += 1 586 587 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 588 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 589 pkgfile = os.path.basename(pkg_relative_path) 590 591 self.Entropy.updateProgress( 592 "[repo:%s|%s|#%s] %s: %s" % ( 593 brown(repo), 594 darkgreen(crippled_uri), 595 brown(str(tries)), 596 blue(_("connecting to download package")), 597 darkgreen(pkg_to_join_path), 598 ), 599 importance = 1, 600 type = "info", 601 header = darkgreen(" * "), 602 back = True 603 ) 604 605 dirpath = os.path.join( 606 self.Entropy.get_remote_packages_relative_path(repo), 607 pkg_to_join_dirpath) 608 ftp.set_cwd(dirpath, dodir = True) 609 610 self.Entropy.updateProgress( 611 "[repo:%s|%s|#%s] %s: %s" % ( 612 brown(repo), 613 darkgreen(crippled_uri), 614 brown(str(tries)), 615 blue(_("downloading package")), 616 darkgreen(pkg_to_join_path), 617 ), 618 importance = 1, 619 type = "info", 620 header = darkgreen(" * ") 621 ) 622 623 download_path = os.path.join( 624 self.Entropy.get_local_packages_directory(repo), 625 pkg_to_join_dirpath) 626 if (not os.path.isdir(download_path)) and \ 627 (not os.access(download_path, os.R_OK)): 628 os.makedirs(download_path) 629 rc_download = ftp.download_file(pkgfile, download_path) 630 if not rc_download: 631 self.Entropy.updateProgress( 632 "[repo:%s|%s|#%s] %s: %s %s" % ( 633 brown(repo), 634 darkgreen(crippled_uri), 635 brown(str(tries)), 636 blue(_("package")), 637 darkgreen(pkg_to_join_path), 638 blue(_("does not exist")), 639 ), 640 importance = 1, 641 type = "error", 642 header = darkred(" !!! ") 643 ) 644 ftp.close() 645 return rc_download 646 647 dbconn = self.Entropy.open_server_repository(read_only = True, 648 no_upload = True, repo = repo) 649 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 650 if idpackage == -1: 651 self.Entropy.updateProgress( 652 "[repo:%s|%s|#%s] %s: %s %s" % ( 653 brown(repo), 654 darkgreen(crippled_uri), 655 brown(str(tries)), 656 blue(_("package")), 657 darkgreen(pkgfile), 658 blue(_("is not listed in the repository !")), 659 ), 660 importance = 1, 661 type = "error", 662 header = darkred(" !!! ") 663 ) 664 ftp.close() 665 return False 666 667 storedmd5 = dbconn.retrieveDigest(idpackage) 668 self.Entropy.updateProgress( 669 "[repo:%s|%s|#%s] %s: %s" % ( 670 brown(repo), 671 darkgreen(crippled_uri), 672 brown(str(tries)), 673 blue(_("verifying checksum of package")), 674 darkgreen(pkgfile), 675 ), 676 importance = 1, 677 type = "info", 678 header = darkgreen(" * "), 679 back = True 680 ) 681 682 pkg_path = os.path.join(download_path, pkgfile) 683 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 684 if md5check: 685 self.Entropy.updateProgress( 686 "[repo:%s|%s|#%s] %s: %s %s" % ( 687 brown(repo), 688 darkgreen(crippled_uri), 689 brown(str(tries)), 690 blue(_("package")), 691 darkgreen(pkgfile), 692 blue(_("downloaded successfully")), 693 ), 694 importance = 1, 695 type = "info", 696 header = darkgreen(" * ") 697 ) 698 ftp.close() 699 return True 700 else: 701 self.Entropy.updateProgress( 702 "[repo:%s|%s|#%s] %s: %s %s" % ( 703 brown(repo), 704 darkgreen(crippled_uri), 705 brown(str(tries)), 706 blue(_("package")), 707 darkgreen(pkgfile), 708 blue(_("checksum does not match. re-downloading...")), 709 ), 710 importance = 1, 711 type = "warning", 712 header = darkred(" * ") 713 ) 714 if os.path.isfile(pkg_path): 715 os.remove(pkg_path) 716 717 continue 718 719 # if we get here it means the files hasn't been downloaded properly 720 self.Entropy.updateProgress( 721 "[repo:%s|%s|#%s] %s: %s %s" % ( 722 brown(repo), 723 darkgreen(crippled_uri), 724 brown(str(tries)), 725 blue(_("package")), 726 darkgreen(pkgfile), 727 blue(_("seems broken. Consider to re-package it. Giving up!")), 728 ), 729 importance = 1, 730 type = "error", 731 header = darkred(" !!! ") 732 ) 733 ftp.close() 734 return False

735 736

737 - def get_remote_databases_status(self, repo = None, mirrors = None):

738 739 if repo == None: 740 repo = self.Entropy.default_repository 741 742 if not mirrors: 743 mirrors = self.Entropy.get_remote_mirrors(repo) 744 745 data = [] 746 for uri in mirrors: 747 748 # let it raise an exception if connection is impossible 749 ftp = FtpInterface(uri, self.Entropy) 750 try: 751 my_path = os.path.join( 752 self.Entropy.get_remote_database_relative_path(repo), 753 self.SystemSettings['repositories']['branch']) 754 ftp.set_cwd(my_path, dodir = True) 755 except ftp.ftplib.error_perm: 756 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 757 self.Entropy.updateProgress( 758 "[repo:%s|%s] %s !" % ( 759 brown(repo), 760 darkgreen(crippled_uri), 761 blue(_("mirror has invalid directory structure")), 762 ), 763 importance = 1, 764 type = "warning", 765 header = darkred(" !!! ") 766 ) 767 ftp.close() 768 continue 769 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 770 db_format = sys_set['database_file_format'] 771 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 772 if cmethod == None: 773 raise InvalidDataType("InvalidDataType: %s." % ( 774 _("Wrong database compression method passed"), 775 ) 776 ) 777 compressedfile = etpConst[cmethod[2]] 778 779 revision = 0 780 rc1 = ftp.is_file_available(compressedfile) 781 revfilename = os.path.basename( 782 self.Entropy.get_local_database_revision_file(repo)) 783 rc2 = ftp.is_file_available(revfilename) 784 if rc1 and rc2: 785 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 786 revfilename) 787 dlcount = 5 788 while dlcount: 789 dled = ftp.download_file(revfilename, 790 etpConst['packagestmpdir'], True) 791 if dled: 792 break 793 dlcount -= 1 794 795 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 796 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 797 f_rev = open(revision_localtmppath,"r") 798 try: 799 revision = int(f_rev.readline().strip()) 800 except ValueError: 801 mytxt = _("mirror hasn't valid database revision file") 802 self.Entropy.updateProgress( 803 "[repo:%s|%s] %s: %s" % ( 804 brown(repo), 805 darkgreen(crippled_uri), 806 blue(mytxt), 807 bold(revision), 808 ), 809 importance = 1, 810 type = "error", 811 header = darkred(" !!! ") 812 ) 813 revision = 0 814 f_rev.close() 815 elif dlcount == 0: 816 self.Entropy.updateProgress( 817 "[repo:%s|%s] %s: %s" % ( 818 brown(repo), 819 darkgreen(crippled_uri), 820 blue(_("unable to download repository revision")), 821 bold(revision), 822 ), 823 importance = 1, 824 type = "error", 825 header = darkred(" !!! ") 826 ) 827 revision = 0 828 else: 829 self.Entropy.updateProgress( 830 "[repo:%s|%s] %s: %s" % ( 831 brown(repo), 832 darkgreen(crippled_uri), 833 blue(_("mirror doesn't have valid revision file")), 834 bold(revision), 835 ), 836 importance = 1, 837 type = "error", 838 header = darkred(" !!! ") 839 ) 840 revision = 0 841 if os.path.isfile(revision_localtmppath): 842 os.remove(revision_localtmppath) 843 844 info = [uri, revision] 845 data.append(info) 846 ftp.close() 847 848 return data

849

850 - def is_local_database_locked(self, repo = None):

851 local_repo = repo 852 if local_repo == None: 853 local_repo = self.Entropy.default_repository 854 lock_file = self.get_database_lockfile(local_repo) 855 return os.path.isfile(lock_file)

856

857 - def get_mirrors_lock(self, repo = None):

858 859 dbstatus = [] 860 for uri in self.Entropy.get_remote_mirrors(repo): 861 data = [uri, False, False] 862 ftp = FtpInterface(uri, self.Entropy) 863 try: 864 my_path = os.path.join( 865 self.Entropy.get_remote_database_relative_path(repo), 866 self.SystemSettings['repositories']['branch']) 867 ftp.set_cwd(my_path) 868 except ftp.ftplib.error_perm: 869 ftp.close() 870 continue 871 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 872 # upload locked 873 data[1] = True 874 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 875 # download locked 876 data[2] = True 877 ftp.close() 878 dbstatus.append(data) 879 return dbstatus

880

881 - def download_notice_board(self, repo = None):

882 883 if repo == None: 884 repo = self.Entropy.default_repository 885 mirrors = self.Entropy.get_remote_mirrors(repo) 886 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 887 mytmpdir = self.entropyTools.get_random_temp_file() 888 os.makedirs(mytmpdir) 889 890 self.Entropy.updateProgress( 891 "[repo:%s] %s %s" % ( 892 brown(repo), 893 blue(_("downloading notice board from mirrors to")), 894 red(rss_path), 895 ), 896 importance = 1, 897 type = "info", 898 header = blue(" @@ ") 899 ) 900 901 downloaded = False 902 for uri in mirrors: 903 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 904 downloader = self.FtpServerHandler( 905 FtpInterface, self.Entropy, [uri], 906 [rss_path], download = True, 907 local_basedir = mytmpdir, critical_files = [rss_path], 908 repo = repo 909 ) 910 errors, m_fine_uris, m_broken_uris = downloader.go() 911 if not errors: 912 self.Entropy.updateProgress( 913 "[repo:%s] %s: %s" % ( 914 brown(repo), 915 blue(_("notice board downloaded successfully from")), 916 red(crippled_uri), 917 ), 918 importance = 1, 919 type = "info", 920 header = blue(" @@ ") 921 ) 922 downloaded = True 923 break 924 925 if downloaded: 926 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 927 rss_path) 928 929 return downloaded

930

931 - def upload_notice_board(self, repo = None):

932 933 if repo == None: 934 repo = self.Entropy.default_repository 935 mirrors = self.Entropy.get_remote_mirrors(repo) 936 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 937 938 self.Entropy.updateProgress( 939 "[repo:%s] %s %s" % ( 940 brown(repo), 941 blue(_("uploading notice board from")), 942 red(rss_path), 943 ), 944 importance = 1, 945 type = "info", 946 header = blue(" @@ ") 947 ) 948 949 uploader = self.FtpServerHandler( 950 FtpInterface, 951 self.Entropy, 952 mirrors, 953 [rss_path], 954 critical_files = [rss_path], 955 repo = repo 956 ) 957 errors, m_fine_uris, m_broken_uris = uploader.go() 958 if errors: 959 m_broken_uris = sorted(m_broken_uris) 960 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 961 for x in m_broken_uris] 962 self.Entropy.updateProgress( 963 "[repo:%s] %s %s" % ( 964 brown(repo), 965 blue(_("notice board upload failed on")), 966 red(', '.join(m_broken_uris)), 967 ), 968 importance = 1, 969 type = "info", 970 header = blue(" @@ ") 971 ) 972 return False 973 self.Entropy.updateProgress( 974 "[repo:%s] %s" % ( 975 brown(repo), 976 blue(_("notice board upload success")), 977 ), 978 importance = 1, 979 type = "info", 980 header = blue(" @@ ") 981 ) 982 return True

983 984

985 - def update_notice_board(self, title, notice_text, link = None, repo = None):

986 987 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 988 rss_description = "Inform about important distribution activities." 989 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 990 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 991 if not link: 992 link = srv_set['rss']['website_url'] 993 994 self.download_notice_board(repo) 995 rss_main = RSS(rss_path, rss_title, rss_description, 996 maxentries = 20) 997 rss_main.add_item(title, link, description = notice_text) 998 rss_main.write_changes() 999 status = self.upload_notice_board(repo) 1000 return status

1001

1002 - def read_notice_board(self, do_download = True, repo = None):

1003 1004 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1005 if do_download: 1006 self.download_notice_board(repo) 1007 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1008 return None 1009 rss_main = RSS(rss_path, '', '') 1010 return rss_main.get_entries()

1011

1012 - def remove_from_notice_board(self, identifier, repo = None):

1013 1014 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1015 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 1016 rss_description = "Inform about important distribution activities." 1017 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1018 return 0 1019 rss_main = RSS(rss_path, rss_title, rss_description) 1020 data = rss_main.remove_entry(identifier) 1021 rss_main.write_changes() 1022 return data

1023

1024 - def update_rss_feed(self, repo = None):

1025 1026 if repo == None: 1027 repo = self.Entropy.default_repository 1028 1029 product = self.SystemSettings['repositories']['product'] 1030 #db_dir = self.Entropy.get_local_database_dir(repo) 1031 rss_path = self.Entropy.get_local_database_rss_file(repo) 1032 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 1033 rss_dump_name = repo + etpConst['rss-dump-name'] 1034 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 1035 1036 rss_title = u"%s Online Repository Status" % ( 1037 self.SystemSettings['system']['name'],) 1038 rss_description = \ 1039 u"Keep you updated on what's going on in the %s Repository." % ( 1040 self.SystemSettings['system']['name'],) 1041 1042 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1043 1044 rss_main = RSS(rss_path, rss_title, rss_description, 1045 maxentries = srv_set['rss']['max_entries']) 1046 # load dump 1047 db_actions = self.Cacher.pop(rss_dump_name) 1048 if db_actions: 1049 try: 1050 f_rev = open(db_revision_path) 1051 revision = f_rev.readline().strip() 1052 f_rev.close() 1053 except (IOError, OSError): 1054 revision = u"N/A" 1055 commitmessage = u'' 1056 if self.Entropy.rssMessages['commitmessage']: 1057 commitmessage = u' :: ' + \ 1058 self.Entropy.rssMessages['commitmessage'] 1059 1060 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 1061 product[0].upper() + product[1:] + " " + \ 1062 self.SystemSettings['repositories']['branch'] + \ 1063 " :: Revision: " + revision + commitmessage 1064 1065 link = srv_set['rss']['base_url'] 1066 # create description 1067 added_items = db_actions.get("added") 1068 1069 if added_items: 1070 for atom in sorted(added_items): 1071 mylink = link + "?search=" + atom.split("~")[0] + \ 1072 "&arch=" + etpConst['currentarch'] + "&product="+product 1073 description = atom + ": " + added_items[atom]['description'] 1074 rss_main.add_item(title = "Added/Updated" + title, 1075 link = mylink, description = description) 1076 removed_items = db_actions.get("removed") 1077 1078 if removed_items: 1079 for atom in sorted(removed_items): 1080 description = atom + ": " + \ 1081 removed_items[atom]['description'] 1082 rss_main.add_item(title = "Removed" + title, link = link, 1083 description = description) 1084 1085 light_items = db_actions.get('light') 1086 if light_items: 1087 rss_light = RSS(rss_light_path, rss_title, rss_description, 1088 maxentries = srv_set['rss']['light_max_entries']) 1089 for atom in sorted(light_items): 1090 mylink = link + "?search=" + atom.split("~")[0] + \ 1091 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1092 product 1093 description = light_items[atom]['description'] 1094 rss_light.add_item(title = "[" + revision + "] " + atom, 1095 link = mylink, description = description) 1096 rss_light.write_changes() 1097 1098 rss_main.write_changes() 1099 self.Entropy.rssMessages.clear() 1100 self.dumpTools.removeobj(rss_dump_name)

1101 1102

1103 - def dump_database_to_file(self, db_path, destination_path, opener, 1104 exclude_tables = None, repo = None):

1105 1106 if not exclude_tables: 1107 exclude_tables = [] 1108 1109 f_out = opener(destination_path, "wb") 1110 dbconn = self.Entropy.open_server_repository(db_path, 1111 just_reading = True, repo = repo, do_treeupdates = False) 1112 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1113 self.Entropy.close_server_database(dbconn) 1114 f_out.close()

1115

1116 - def create_file_checksum(self, file_path, checksum_path):

1117 mydigest = self.entropyTools.md5sum(file_path) 1118 f_ck = open(checksum_path, "w") 1119 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1120 f_ck.write(mystring) 1121 f_ck.flush() 1122 f_ck.close()

1123

1124 - def compress_file(self, file_path, destination_path, opener):

1125 f_out = opener(destination_path, "wb") 1126 f_in = open(file_path,"rb") 1127 data = f_in.read(8192) 1128 while data: 1129 f_out.write(data) 1130 data = f_in.read(8192) 1131 f_in.close() 1132 if hasattr(f_out,'flush'): 1133 f_out.flush() 1134 f_out.close()

1135

1136 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1137 disabled_eapis = None):

1138 1139 if disabled_eapis == None: 1140 disabled_eapis = [] 1141 1142 critical = [] 1143 extra_text_files = [] 1144 data = {} 1145 data['database_revision_file'] = \ 1146 self.Entropy.get_local_database_revision_file(repo) 1147 extra_text_files.append(data['database_revision_file']) 1148 critical.append(data['database_revision_file']) 1149 1150 1151 # branch migration support scripts 1152 post_branch_mig_file = self.Entropy.get_local_post_branch_mig_script( 1153 repo) 1154 if os.path.isfile(post_branch_mig_file) or download: 1155 data['database_post_branch_hop_script'] = post_branch_mig_file 1156 extra_text_files.append(data['database_post_branch_hop_script']) 1157 1158 post_branch_upg_file = self.Entropy.get_local_post_branch_upg_script( 1159 repo) 1160 if os.path.isfile(post_branch_upg_file) or download: 1161 data['database_post_branch_upgrade_script'] = post_branch_upg_file 1162 extra_text_files.append(data['database_post_branch_upgrade_script']) 1163 1164 1165 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1166 if os.path.isfile(database_ts_file) or download: 1167 data['database_timestamp_file'] = database_ts_file 1168 if not download: 1169 critical.append(database_ts_file) 1170 1171 database_package_mask_file = \ 1172 self.Entropy.get_local_database_mask_file(repo) 1173 extra_text_files.append(database_package_mask_file) 1174 if os.path.isfile(database_package_mask_file) or download: 1175 data['database_package_mask_file'] = database_package_mask_file 1176 if not download: 1177 critical.append(data['database_package_mask_file']) 1178 1179 database_package_system_mask_file = \ 1180 self.Entropy.get_local_database_system_mask_file(repo) 1181 extra_text_files.append(database_package_system_mask_file) 1182 if os.path.isfile(database_package_system_mask_file) or download: 1183 data['database_package_system_mask_file'] = \ 1184 database_package_system_mask_file 1185 if not download: 1186 critical.append(data['database_package_system_mask_file']) 1187 1188 database_package_confl_tagged_file = \ 1189 self.Entropy.get_local_database_confl_tagged_file(repo) 1190 extra_text_files.append(database_package_confl_tagged_file) 1191 if os.path.isfile(database_package_confl_tagged_file) or download: 1192 data['database_package_confl_tagged_file'] = \ 1193 database_package_confl_tagged_file 1194 if not download: 1195 critical.append(data['database_package_confl_tagged_file']) 1196 1197 database_license_whitelist_file = \ 1198 self.Entropy.get_local_database_licensewhitelist_file(repo) 1199 extra_text_files.append(database_license_whitelist_file) 1200 if os.path.isfile(database_license_whitelist_file) or download: 1201 data['database_license_whitelist_file'] = \ 1202 database_license_whitelist_file 1203 if not download: 1204 critical.append(data['database_license_whitelist_file']) 1205 1206 exp_based_pkgs_removal_file = \ 1207 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1208 extra_text_files.append(exp_based_pkgs_removal_file) 1209 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1210 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1211 if not download: 1212 critical.append(data['exp_based_pkgs_removal_file']) 1213 1214 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1215 if os.path.isfile(database_rss_file) or download: 1216 data['database_rss_file'] = database_rss_file 1217 if not download: 1218 critical.append(data['database_rss_file']) 1219 database_rss_light_file = \ 1220 self.Entropy.get_local_database_rsslight_file(repo) 1221 1222 extra_text_files.append(database_rss_light_file) 1223 if os.path.isfile(database_rss_light_file) or download: 1224 data['database_rss_light_file'] = database_rss_light_file 1225 if not download: 1226 critical.append(data['database_rss_light_file']) 1227 1228 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1229 data['pkglist_file'] = pkglist_file 1230 if not download: 1231 critical.append(data['pkglist_file']) 1232 1233 # EAPI 2,3 1234 if not download: # we don't need to get the dump 1235 1236 if 3 not in disabled_eapis: 1237 1238 data['metafiles_path'] = \ 1239 self.Entropy.get_local_database_compressed_metafiles_file( 1240 repo) 1241 critical.append(data['metafiles_path']) 1242 1243 if 2 not in disabled_eapis: 1244 1245 data['dump_path_light'] = os.path.join( 1246 self.Entropy.get_local_database_dir(repo), 1247 etpConst[cmethod[5]]) 1248 critical.append(data['dump_path_light']) 1249 1250 data['dump_path_digest_light'] = os.path.join( 1251 self.Entropy.get_local_database_dir(repo), 1252 etpConst[cmethod[6]]) 1253 critical.append(data['dump_path_digest_light']) 1254 1255 # EAPI 1 1256 if 1 not in disabled_eapis: 1257 1258 data['compressed_database_path'] = os.path.join( 1259 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1260 critical.append(data['compressed_database_path']) 1261 data['compressed_database_path_light'] = os.path.join( 1262 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1263 1264 data['database_path_digest'] = os.path.join( 1265 self.Entropy.get_local_database_dir(repo), 1266 etpConst['etpdatabasehashfile'] 1267 ) 1268 critical.append(data['database_path_digest']) 1269 1270 data['compressed_database_path_digest'] = os.path.join( 1271 self.Entropy.get_local_database_dir(repo), 1272 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1273 ) 1274 critical.append(data['compressed_database_path_digest']) 1275 1276 data['compressed_database_path_digest_light'] = os.path.join( 1277 self.Entropy.get_local_database_dir(repo), 1278 etpConst[cmethod[8]] 1279 ) 1280 critical.append(data['compressed_database_path_digest_light']) 1281 1282 1283 # SSL cert file, just for reference 1284 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1285 extra_text_files.append(ssl_ca_cert) 1286 if os.path.isfile(ssl_ca_cert): 1287 data['ssl_ca_cert_file'] = ssl_ca_cert 1288 if not download: 1289 critical.append(ssl_ca_cert) 1290 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1291 extra_text_files.append(ssl_server_cert) 1292 if os.path.isfile(ssl_server_cert): 1293 data['ssl_server_cert_file'] = ssl_server_cert 1294 if not download: 1295 critical.append(ssl_server_cert) 1296 1297 # Some information regarding how packages are built 1298 spm_files = [ 1299 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1300 (etpConst['spm']['global_package_keywords'], 1301 "global_package_keywords"), 1302 (etpConst['spm']['global_package_use'], "global_package_use"), 1303 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1304 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1305 ] 1306 for myfile, myname in spm_files: 1307 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1308 data[myname] = myfile 1309 extra_text_files.append(myfile) 1310 1311 make_profile = etpConst['spm']['global_make_profile'] 1312 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1313 mytmpdir = os.path.dirname(rnd_tmp_file) 1314 mytmpfile = os.path.join(mytmpdir, 1315 etpConst['spm']['global_make_profile_link_name']) 1316 extra_text_files.append(mytmpfile) 1317 1318 if os.path.islink(make_profile): 1319 mylink = os.readlink(make_profile) 1320 f_mkp = open(mytmpfile,"w") 1321 f_mkp.write(mylink) 1322 f_mkp.flush() 1323 f_mkp.close() 1324 data['global_make_profile'] = mytmpfile 1325 1326 return data, critical, extra_text_files

1327

1328 - def _show_package_sets_messages(self, repo):

1329 1330 self.Entropy.updateProgress( 1331 "[repo:%s] %s:" % ( 1332 brown(repo), 1333 blue(_("configured package sets")), 1334 ), 1335 importance = 0, 1336 type = "info", 1337 header = darkgreen(" * ") 1338 ) 1339 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1340 if not sets_data: 1341 self.Entropy.updateProgress( 1342 "%s" % (_("None configured"),), 1343 importance = 0, 1344 type = "info", 1345 header = brown(" # ") 1346 ) 1347 return 1348 for s_repo, s_name, s_sets in sets_data: 1349 self.Entropy.updateProgress( 1350 blue("%s" % (s_name,)), 1351 importance = 0, 1352 type = "info", 1353 header = brown(" # ") 1354 )

1355

1356 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1357 1358 self.Entropy.updateProgress( 1359 "[repo:%s|%s|%s:%s] %s" % ( 1360 brown(repo), 1361 darkgreen(crippled_uri), 1362 red("EAPI"), 1363 bold("3"), 1364 blue(_("preparing uncompressed database for the upload")), 1365 ), 1366 importance = 0, 1367 type = "info", 1368 header = darkgreen(" * ") 1369 ) 1370 self.Entropy.updateProgress( 1371 "%s: %s" % (_("database path"), blue(database_path),), 1372 importance = 0, 1373 type = "info", 1374 header = brown(" # ") 1375 )

1376

1377 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1378 upload_data, cmethod, repo):

1379 1380 if repo == None: 1381 repo = self.Entropy.default_repository 1382 1383 self.Entropy.updateProgress( 1384 "[repo:%s|%s|%s:%s] %s" % ( 1385 brown(repo), 1386 darkgreen(crippled_uri), 1387 red("EAPI"), 1388 bold("2"), 1389 blue(_("creating compressed database dump + checksum")), 1390 ), 1391 importance = 0, 1392 type = "info", 1393 header = darkgreen(" * ") 1394 ) 1395 self.Entropy.updateProgress( 1396 "%s: %s" % (_("database path"), blue(database_path),), 1397 importance = 0, 1398 type = "info", 1399 header = brown(" # ") 1400 ) 1401 self.Entropy.updateProgress( 1402 "%s: %s" % ( 1403 _("dump light"), 1404 blue(upload_data['dump_path_light']), 1405 ), 1406 importance = 0, 1407 type = "info", 1408 header = brown(" # ") 1409 ) 1410 self.Entropy.updateProgress( 1411 "%s: %s" % ( 1412 _("dump light checksum"), 1413 blue(upload_data['dump_path_digest_light']), 1414 ), 1415 importance = 0, 1416 type = "info", 1417 header = brown(" # ") 1418 ) 1419 1420 self.Entropy.updateProgress( 1421 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1422 importance = 0, 1423 type = "info", 1424 header = brown(" # ") 1425 )

1426

1427 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1428 upload_data, cmethod, repo):

1429 1430 self.Entropy.updateProgress( 1431 "[repo:%s|%s|%s:%s] %s" % ( 1432 brown(repo), 1433 darkgreen(crippled_uri), 1434 red("EAPI"), 1435 bold("1"), 1436 blue(_("compressing database + checksum")), 1437 ), 1438 importance = 0, 1439 type = "info", 1440 header = darkgreen(" * "), 1441 back = True 1442 ) 1443 self.Entropy.updateProgress( 1444 "%s: %s" % (_("database path"), blue(database_path),), 1445 importance = 0, 1446 type = "info", 1447 header = brown(" # ") 1448 ) 1449 self.Entropy.updateProgress( 1450 "%s: %s" % ( 1451 _("compressed database path"), 1452 blue(upload_data['compressed_database_path']), 1453 ), 1454 importance = 0, 1455 type = "info", 1456 header = brown(" # ") 1457 ) 1458 self.Entropy.updateProgress( 1459 "%s: %s" % ( 1460 _("database checksum"), 1461 blue(upload_data['database_path_digest']), 1462 ), 1463 importance = 0, 1464 type = "info", 1465 header = brown(" # ") 1466 ) 1467 self.Entropy.updateProgress( 1468 "%s: %s" % ( 1469 _("compressed checksum"), 1470 blue(upload_data['compressed_database_path_digest']), 1471 ), 1472 importance = 0, 1473 type = "info", 1474 header = brown(" # ") 1475 ) 1476 self.Entropy.updateProgress( 1477 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1478 importance = 0, 1479 type = "info", 1480 header = brown(" # ") 1481 )

1482

1483 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1484 1485 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1486 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1487 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1488 file_list if x not in found_file_list] 1489 metafile_not_found_file = \ 1490 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1491 1492 f_meta = open(metafile_not_found_file,"w") 1493 f_meta.writelines(not_found_file_list) 1494 f_meta.flush() 1495 f_meta.close() 1496 found_file_list.append(metafile_not_found_file) 1497 if os.path.isfile(compressed_dest_path): 1498 os.remove(compressed_dest_path) 1499 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1500

1501 - def create_mirror_directories(self, ftp_connection, path_to_create):

1502 bdir = "" 1503 for mydir in path_to_create.split("/"): 1504 bdir += "/"+mydir 1505 if not ftp_connection.is_file_available(bdir): 1506 try: 1507 ftp_connection.mkdir(bdir) 1508 except Exception, err: 1509 error = unicode(err) 1510 if (error.find("550") == -1) and \ 1511 (error.find("File exist") == -1): 1512 1513 mytxt = "%s %s, %s: %s" % ( 1514 _("cannot create mirror directory"), 1515 bdir, 1516 _("error"), 1517 err, 1518 ) 1519 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1520 mytxt,)) 1521 raise

1522

1523 - def mirror_lock_check(self, uri, repo = None):

1524 1525 if repo == None: 1526 repo = self.Entropy.default_repository 1527 1528 gave_up = False 1529 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1530 try: 1531 ftp = FtpInterface(uri, self.Entropy) 1532 except ConnectionError: 1533 self.entropyTools.print_traceback() 1534 return True # gave up 1535 my_path = os.path.join( 1536 self.Entropy.get_remote_database_relative_path(repo), 1537 self.SystemSettings['repositories']['branch']) 1538 ftp.set_cwd(my_path, dodir = True) 1539 1540 lock_file = self.get_database_lockfile(repo) 1541 lock_filename = os.path.basename(lock_file) 1542 if not os.path.isfile(lock_file) and \ 1543 ftp.is_file_available(lock_filename): 1544 1545 self.Entropy.updateProgress( 1546 "[repo:%s|%s|%s] %s, %s" % ( 1547 brown(str(repo)), 1548 darkgreen(crippled_uri), 1549 red(_("locking")), 1550 darkblue(_("mirror already locked")), 1551 blue(_("waiting up to 2 minutes before giving up")), 1552 ), 1553 importance = 1, 1554 type = "warning", 1555 header = brown(" * "), 1556 back = True 1557 ) 1558 unlocked = False 1559 count = 0 1560 while count < 120: 1561 count += 1 1562 time.sleep(1) 1563 if not ftp.is_file_available(os.path.basename(lock_file)): 1564 self.Entropy.updateProgress( 1565 red("[repo:%s|%s|%s] %s !" % ( 1566 repo, 1567 crippled_uri, 1568 _("locking"), 1569 _("mirror unlocked"), 1570 ) 1571 ), 1572 importance = 1, 1573 type = "info", 1574 header = darkgreen(" * ") 1575 ) 1576 unlocked = True 1577 break 1578 if not unlocked: 1579 gave_up = True 1580 1581 ftp.close() 1582 return gave_up

1583

1584 - def shrink_database_and_close(self, repo = None):

1585 dbconn = self.Entropy.open_server_repository(read_only = False, 1586 no_upload = True, repo = repo, indexing = False, 1587 do_treeupdates = False) 1588 dbconn.dropAllIndexes() 1589 dbconn.vacuum() 1590 dbconn.vacuum() 1591 dbconn.commitChanges() 1592 self.Entropy.close_server_database(dbconn)

1593

1594 - def update_repository_timestamp(self, repo = None):

1595 if repo == None: 1596 repo = self.Entropy.default_repository 1597 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1598 current_ts = self.Entropy.get_current_timestamp() 1599 ts_f = open(ts_file,"w") 1600 ts_f.write(current_ts) 1601 ts_f.flush() 1602 ts_f.close()

1603

1604 - def sync_database_treeupdates(self, repo = None):

1605 1606 if repo == None: 1607 repo = self.Entropy.default_repository 1608 dbconn = self.Entropy.open_server_repository(read_only = False, 1609 no_upload = True, repo = repo, do_treeupdates = False) 1610 # grab treeupdates from other databases and inject 1611 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1612 server_repos = srv_set['repositories'].keys() 1613 all_actions = set() 1614 for myrepo in server_repos: 1615 1616 # avoid __default__ 1617 if myrepo == etpConst['clientserverrepoid']: 1618 continue 1619 1620 mydbc = self.Entropy.open_server_repository(just_reading = True, 1621 repo = myrepo) 1622 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1623 for data in actions: 1624 all_actions.add(data) 1625 if not actions: 1626 continue 1627 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1628 try: 1629 # clear first 1630 dbconn.removeTreeUpdatesActions(repo) 1631 dbconn.insertTreeUpdatesActions(all_actions, repo) 1632 except dbapi2.Error, err: 1633 self.entropyTools.print_traceback() 1634 mytxt = "%s, %s: %s. %s" % ( 1635 _("Troubles with treeupdates"), 1636 _("error"), 1637 err, 1638 _("Bumping old data back"), 1639 ) 1640 self.Entropy.updateProgress( 1641 mytxt, 1642 importance = 1, 1643 type = "warning" 1644 ) 1645 # restore previous data 1646 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1647 1648 dbconn.commitChanges() 1649 self.Entropy.close_server_database(dbconn)

1650

1651 - def upload_database(self, uris, lock_check = False, pretend = False, 1652 repo = None):

1653 1654 if repo == None: 1655 repo = self.Entropy.default_repository 1656 1657 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1658 if srv_set['rss']['enabled']: 1659 self.update_rss_feed(repo = repo) 1660 1661 upload_errors = False 1662 broken_uris = set() 1663 fine_uris = set() 1664 1665 disabled_eapis = sorted(srv_set['disabled_eapis']) 1666 1667 for uri in uris: 1668 1669 db_format = srv_set['database_file_format'] 1670 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1671 if cmethod == None: 1672 raise InvalidDataType("InvalidDataType: %s." % ( 1673 _("wrong database compression method passed"), 1674 ) 1675 ) 1676 1677 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1678 database_path = self.Entropy.get_local_database_file(repo) 1679 1680 if disabled_eapis: 1681 self.Entropy.updateProgress( 1682 "[repo:%s|%s|%s] %s: %s" % ( 1683 blue(repo), 1684 red(crippled_uri), 1685 darkgreen(_("upload")), 1686 darkred(_("disabled EAPI")), 1687 bold(', '.join(disabled_eapis)), 1688 ), 1689 importance = 1, 1690 type = "warning", 1691 header = darkgreen(" * ") 1692 ) 1693 1694 # create/update timestamp file 1695 self.update_repository_timestamp(repo) 1696 # create pkglist service file 1697 self.Entropy.create_repository_pkglist(repo) 1698 1699 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1700 repo = repo, disabled_eapis = disabled_eapis) 1701 1702 if lock_check: 1703 given_up = self.mirror_lock_check(uri, repo = repo) 1704 if given_up: 1705 upload_errors = True 1706 broken_uris.add(uri) 1707 continue 1708 1709 self.lock_mirrors_for_download(True, [uri], repo = repo) 1710 1711 self.Entropy.updateProgress( 1712 "[repo:%s|%s|%s] %s" % ( 1713 blue(repo), 1714 red(crippled_uri), 1715 darkgreen(_("upload")), 1716 darkgreen(_("preparing to upload database to mirror")), 1717 ), 1718 importance = 1, 1719 type = "info", 1720 header = darkgreen(" * ") 1721 ) 1722 1723 # Package Sets info 1724 self._show_package_sets_messages(repo) 1725 self.sync_database_treeupdates(repo) 1726 self.Entropy.update_database_package_sets(repo) 1727 self.Entropy.close_server_databases() 1728 1729 # backup current database to avoid re-indexing 1730 old_dbpath = self.Entropy.get_local_database_file(repo) 1731 backup_dbpath = old_dbpath + ".up_backup" 1732 copy_back = False 1733 if not pretend: 1734 try: 1735 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1736 os.remove(backup_dbpath) 1737 shutil.copy2(old_dbpath, backup_dbpath) 1738 copy_back = True 1739 except shutil.Error: 1740 copy_back = False 1741 1742 self.shrink_database_and_close(repo) 1743 1744 # EAPI 3 1745 if 3 not in disabled_eapis: 1746 self._create_metafiles_file(upload_data['metafiles_path'], 1747 text_files, repo) 1748 self._show_eapi3_upload_messages(crippled_uri, database_path, 1749 repo) 1750 1751 # EAPI 2 1752 if 2 not in disabled_eapis: 1753 self._show_eapi2_upload_messages(crippled_uri, database_path, 1754 upload_data, cmethod, repo) 1755 1756 # create compressed dump + checksum 1757 self.dump_database_to_file(database_path, 1758 upload_data['dump_path_light'], cmethod[0], 1759 exclude_tables = ["content"], repo = repo) 1760 self.create_file_checksum(upload_data['dump_path_light'], 1761 upload_data['dump_path_digest_light']) 1762 1763 1764 # EAPI 1 1765 if 1 not in disabled_eapis: 1766 self._show_eapi1_upload_messages(crippled_uri, database_path, 1767 upload_data, cmethod, repo) 1768 1769 # compress the database and create uncompressed 1770 # database checksum -- DEPRECATED 1771 self.compress_file(database_path, 1772 upload_data['compressed_database_path'], cmethod[0]) 1773 self.create_file_checksum(database_path, 1774 upload_data['database_path_digest']) 1775 1776 # create compressed database checksum 1777 self.create_file_checksum( 1778 upload_data['compressed_database_path'], 1779 upload_data['compressed_database_path_digest']) 1780 1781 # create light version of the compressed db 1782 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1783 temp_eapi1_dbfile = eapi1_dbfile+".light" 1784 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1785 # open and remove content table 1786 eapi1_tmp_dbconn = \ 1787 self.Entropy.ClientService.open_generic_database( 1788 temp_eapi1_dbfile, indexing_override = False, 1789 xcache = False) 1790 eapi1_tmp_dbconn.dropContent() 1791 eapi1_tmp_dbconn.commitChanges() 1792 eapi1_tmp_dbconn.vacuum() 1793 eapi1_tmp_dbconn.closeDB() 1794 1795 # compress 1796 self.compress_file(temp_eapi1_dbfile, 1797 upload_data['compressed_database_path_light'], cmethod[0]) 1798 # go away, we don't need you anymore 1799 os.remove(temp_eapi1_dbfile) 1800 # create compressed light database checksum 1801 self.create_file_checksum( 1802 upload_data['compressed_database_path_light'], 1803 upload_data['compressed_database_path_digest_light']) 1804 1805 if not pretend: 1806 # upload 1807 uploader = self.FtpServerHandler( 1808 FtpInterface, 1809 self.Entropy, 1810 [uri], 1811 [upload_data[x] for x in upload_data], 1812 critical_files = critical, 1813 repo = repo 1814 ) 1815 errors, m_fine_uris, m_broken_uris = uploader.go() 1816 if errors: 1817 my_broken_uris = sorted([ 1818 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1819 x[1]) for x in m_broken_uris]) 1820 self.Entropy.updateProgress( 1821 "[repo:%s|%s|%s] %s" % ( 1822 repo, 1823 crippled_uri, 1824 _("errors"), 1825 _("upload failed, not unlocking and continuing"), 1826 ), 1827 importance = 0, 1828 type = "error", 1829 header = darkred(" !!! ") 1830 ) 1831 # get reason 1832 reason = my_broken_uris[0][1] 1833 self.Entropy.updateProgress( 1834 blue("%s: %s" % (_("reason"), reason,)), 1835 importance = 0, 1836 type = "error", 1837 header = blue(" # ") 1838 ) 1839 upload_errors = True 1840 broken_uris |= m_broken_uris 1841 continue 1842 1843 # copy db back 1844 if copy_back and os.path.isfile(backup_dbpath): 1845 self.Entropy.close_server_databases() 1846 further_backup_dbpath = old_dbpath+".security_backup" 1847 if os.path.isfile(further_backup_dbpath): 1848 os.remove(further_backup_dbpath) 1849 shutil.copy2(old_dbpath, further_backup_dbpath) 1850 shutil.move(backup_dbpath, old_dbpath) 1851 1852 # unlock 1853 self.lock_mirrors_for_download(False, [uri], repo = repo) 1854 fine_uris |= m_fine_uris 1855 1856 if not fine_uris: 1857 upload_errors = True 1858 return upload_errors, broken_uris, fine_uris

1859 1860

1861 - def download_database(self, uris, lock_check = False, pretend = False, 1862 repo = None):

1863 1864 if repo == None: 1865 repo = self.Entropy.default_repository 1866 1867 download_errors = False 1868 broken_uris = set() 1869 fine_uris = set() 1870 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1871 disabled_eapis = sorted(srv_set['disabled_eapis']) 1872 1873 for uri in uris: 1874 1875 db_format = srv_set['database_file_format'] 1876 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1877 if cmethod == None: 1878 raise InvalidDataType("InvalidDataType: %s." % ( 1879 _("wrong database compression method passed"), 1880 ) 1881 ) 1882 1883 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1884 database_path = self.Entropy.get_local_database_file(repo) 1885 database_dir_path = os.path.dirname( 1886 self.Entropy.get_local_database_file(repo)) 1887 download_data, critical, text_files = self.get_files_to_sync( 1888 cmethod, download = True, 1889 repo = repo, disabled_eapis = disabled_eapis) 1890 mytmpdir = self.entropyTools.get_random_temp_file() 1891 os.makedirs(mytmpdir) 1892 1893 self.Entropy.updateProgress( 1894 "[repo:%s|%s|%s] %s" % ( 1895 brown(repo), 1896 darkgreen(crippled_uri), 1897 red(_("download")), 1898 blue(_("preparing to download database from mirror")), 1899 ), 1900 importance = 1, 1901 type = "info", 1902 header = darkgreen(" * ") 1903 ) 1904 files_to_sync = sorted(download_data.keys()) 1905 for myfile in files_to_sync: 1906 self.Entropy.updateProgress( 1907 "%s: %s" % ( 1908 blue(_("download path")), 1909 brown(unicode(download_data[myfile])), 1910 ), 1911 importance = 0, 1912 type = "info", 1913 header = brown(" # ") 1914 ) 1915 1916 if lock_check: 1917 given_up = self.mirror_lock_check(uri, repo = repo) 1918 if given_up: 1919 download_errors = True 1920 broken_uris.add(uri) 1921 continue 1922 1923 # avoid having others messing while we're downloading 1924 self.lock_mirrors(True, [uri], repo = repo) 1925 1926 if not pretend: 1927 # download 1928 downloader = self.FtpServerHandler( 1929 FtpInterface, self.Entropy, [uri], 1930 [download_data[x] for x in download_data], download = True, 1931 local_basedir = mytmpdir, critical_files = critical, 1932 repo = repo 1933 ) 1934 errors, m_fine_uris, m_broken_uris = downloader.go() 1935 if errors: 1936 my_broken_uris = sorted([ 1937 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1938 x[1]) for x in m_broken_uris]) 1939 self.Entropy.updateProgress( 1940 "[repo:%s|%s|%s] %s" % ( 1941 brown(repo), 1942 darkgreen(crippled_uri), 1943 red(_("errors")), 1944 blue(_("failed to download from mirror")), 1945 ), 1946 importance = 0, 1947 type = "error", 1948 header = darkred(" !!! ") 1949 ) 1950 # get reason 1951 reason = my_broken_uris[0][1] 1952 self.Entropy.updateProgress( 1953 blue("%s: %s" % (_("reason"), reason,)), 1954 importance = 0, 1955 type = "error", 1956 header = blue(" # ") 1957 ) 1958 download_errors = True 1959 broken_uris |= m_broken_uris 1960 self.lock_mirrors(False, [uri], repo = repo) 1961 continue 1962 1963 # all fine then, we need to move data from mytmpdir 1964 # to database_dir_path 1965 1966 # EAPI 1 -- unpack database 1967 if 1 not in disabled_eapis: 1968 compressed_db_filename = os.path.basename( 1969 download_data['compressed_database_path']) 1970 uncompressed_db_filename = os.path.basename(database_path) 1971 compressed_file = os.path.join(mytmpdir, 1972 compressed_db_filename) 1973 uncompressed_file = os.path.join(mytmpdir, 1974 uncompressed_db_filename) 1975 self.entropyTools.uncompress_file(compressed_file, 1976 uncompressed_file, cmethod[0]) 1977 1978 # now move 1979 for myfile in os.listdir(mytmpdir): 1980 fromfile = os.path.join(mytmpdir, myfile) 1981 tofile = os.path.join(database_dir_path, myfile) 1982 shutil.move(fromfile, tofile) 1983 self.Entropy.ClientService.setup_default_file_perms(tofile) 1984 1985 if os.path.isdir(mytmpdir): 1986 shutil.rmtree(mytmpdir) 1987 if os.path.isdir(mytmpdir): 1988 os.rmdir(mytmpdir) 1989 1990 1991 fine_uris.add(uri) 1992 self.lock_mirrors(False, [uri], repo = repo) 1993 1994 return download_errors, fine_uris, broken_uris

1995

1996 - def calculate_database_sync_queues(self, repo = None):

1997 1998 if repo == None: 1999 repo = self.Entropy.default_repository 2000 2001 remote_status = self.get_remote_databases_status(repo) 2002 local_revision = self.Entropy.get_local_database_revision(repo) 2003 upload_queue = [] 2004 download_latest = () 2005 2006 # all mirrors are empty ? I rule 2007 if not [x for x in remote_status if x[1]]: 2008 upload_queue = remote_status[:] 2009 else: 2010 highest_remote_revision = max([x[1] for x in remote_status]) 2011 2012 if local_revision < highest_remote_revision: 2013 for remote_item in remote_status: 2014 if remote_item[1] == highest_remote_revision: 2015 download_latest = x 2016 break 2017 2018 if download_latest: 2019 upload_queue = [x for x in remote_status if \ 2020 (x[1] < highest_remote_revision)] 2021 else: 2022 upload_queue = [x for x in remote_status if \ 2023 (x[1] < local_revision)] 2024 2025 return download_latest, upload_queue

2026

2027 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 2028 repo = None):

2029 2030 if repo == None: 2031 repo = self.Entropy.default_repository 2032 2033 while 1: 2034 2035 db_locked = False 2036 if self.is_local_database_locked(repo): 2037 db_locked = True 2038 2039 lock_data = self.get_mirrors_lock(repo) 2040 mirrors_locked = [x for x in lock_data if x[1]] 2041 2042 if not mirrors_locked and db_locked: 2043 # mirrors not locked remotely but only locally 2044 mylock_file = self.get_database_lockfile(repo) 2045 if os.access(mylock_file, os.W_OK | os.F_OK): 2046 os.remove(mylock_file) 2047 continue 2048 2049 break 2050 2051 if mirrors_locked and not db_locked: 2052 mytxt = "%s, %s %s" % ( 2053 _("Mirrors are locked, someone is working on the repository"), 2054 _("try again later"), 2055 "...", 2056 ) 2057 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 2058 2059 download_latest, upload_queue = self.calculate_database_sync_queues( 2060 repo) 2061 2062 if not download_latest and not upload_queue: 2063 self.Entropy.updateProgress( 2064 "[repo:%s|%s] %s" % ( 2065 brown(repo), 2066 red(_("sync")), # something short please 2067 blue(_("database already in sync")), 2068 ), 2069 importance = 1, 2070 type = "info", 2071 header = blue(" @@ ") 2072 ) 2073 return 0, set(), set() 2074 2075 if download_latest: 2076 download_uri = download_latest[0] 2077 download_errors, fine_uris, broken_uris = self.download_database( 2078 [download_uri], repo = repo) 2079 if download_errors: 2080 self.Entropy.updateProgress( 2081 "[repo:%s|%s] %s: %s" % ( 2082 brown(repo), 2083 red(_("sync")), 2084 blue(_("database sync failed")), 2085 red(_("download issues")), 2086 ), 2087 importance = 1, 2088 type = "error", 2089 header = darkred(" !!! ") 2090 ) 2091 return 1, fine_uris, broken_uris 2092 2093 if upload_queue and not no_upload: 2094 2095 # XXX QA checks, 2096 # please group them into entropy.qa 2097 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2098 if deps_not_found and not self.Entropy.community_repo: 2099 self.Entropy.updateProgress( 2100 "[repo:%s|%s] %s: %s" % ( 2101 brown(repo), 2102 red(_("sync")), 2103 blue(_("database sync forbidden")), 2104 red(_("dependencies_test() reported errors")), 2105 ), 2106 importance = 1, 2107 type = "error", 2108 header = darkred(" !!! ") 2109 ) 2110 return 3, set(), set() 2111 2112 problems = self.Entropy.check_config_file_updates() 2113 if problems: 2114 return 4, set(), set() 2115 2116 self.Entropy.updateProgress( 2117 "[repo:%s|%s] %s" % ( 2118 brown(repo), 2119 red(_("config files")), # something short please 2120 blue(_("no configuration files to commit. All fine.")), 2121 ), 2122 importance = 1, 2123 type = "info", 2124 header = blue(" @@ "), 2125 back = True 2126 ) 2127 2128 uris = [x[0] for x in upload_queue] 2129 errors, fine_uris, broken_uris = self.upload_database(uris, 2130 repo = repo) 2131 if errors: 2132 self.Entropy.updateProgress( 2133 "[repo:%s|%s] %s: %s" % ( 2134 brown(repo), 2135 red(_("sync")), 2136 blue(_("database sync failed")), 2137 red(_("upload issues")), 2138 ), 2139 importance = 1, 2140 type = "error", 2141 header = darkred(" !!! ") 2142 ) 2143 return 2, fine_uris, broken_uris 2144 2145 2146 self.Entropy.updateProgress( 2147 "[repo:%s|%s] %s" % ( 2148 brown(repo), 2149 red(_("sync")), 2150 blue(_("database sync completed successfully")), 2151 ), 2152 importance = 1, 2153 type = "info", 2154 header = darkgreen(" * ") 2155 ) 2156 2157 if unlock_mirrors: 2158 self.lock_mirrors(False, repo = repo) 2159 return 0, set(), set()

2160 2161

2162 - def calculate_local_upload_files(self, branch, repo = None):

2163 upload_files = 0 2164 upload_packages = set() 2165 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2166 branch) 2167 2168 # check if it exists 2169 if not os.path.isdir(upload_dir): 2170 return upload_files, upload_packages 2171 2172 pkg_ext = etpConst['packagesext'] 2173 pkg_md5_ext = etpConst['packagesmd5fileext'] 2174 for package in os.listdir(upload_dir): 2175 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2176 upload_packages.add(package) 2177 if package.endswith(pkg_ext): 2178 upload_files += 1 2179 2180 return upload_files, upload_packages

2181

2182 - def calculate_local_package_files(self, branch, repo = None):

2183 local_files = 0 2184 local_packages = set() 2185 packages_dir = os.path.join( 2186 self.Entropy.get_local_packages_directory(repo), branch) 2187 2188 if not os.path.isdir(packages_dir): 2189 os.makedirs(packages_dir) 2190 2191 pkg_ext = etpConst['packagesext'] 2192 pkg_md5_ext = etpConst['packagesmd5fileext'] 2193 for package in os.listdir(packages_dir): 2194 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2195 local_packages.add(package) 2196 if package.endswith(pkg_ext): 2197 local_files += 1 2198 2199 return local_files, local_packages

2200 2201

2202 - def _show_local_sync_stats(self, upload_files, local_files):

2203 self.Entropy.updateProgress( 2204 "%s:" % ( 2205 blue(_("Local statistics")), 2206 ), 2207 importance = 1, 2208 type = "info", 2209 header = red(" @@ ") 2210 ) 2211 self.Entropy.updateProgress( 2212 red("%s:\t\t%s %s" % ( 2213 blue(_("upload directory")), 2214 bold(str(upload_files)), 2215 red(_("files ready")), 2216 ) 2217 ), 2218 importance = 0, 2219 type = "info", 2220 header = red(" @@ ") 2221 ) 2222 self.Entropy.updateProgress( 2223 red("%s:\t\t%s %s" % ( 2224 blue(_("packages directory")), 2225 bold(str(local_files)), 2226 red(_("files ready")), 2227 ) 2228 ), 2229 importance = 0, 2230 type = "info", 2231 header = red(" @@ ") 2232 )

2233

2234 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2235 branch):

2236 2237 # show stats 2238 for package, size in upload: 2239 package = darkgreen(os.path.basename(package)) 2240 size = blue(self.entropyTools.bytes_into_human(size)) 2241 self.Entropy.updateProgress( 2242 "[branch:%s|%s] %s [%s]" % ( 2243 brown(branch), 2244 blue(_("upload")), 2245 darkgreen(package), 2246 size, 2247 ), 2248 importance = 0, 2249 type = "info", 2250 header = red(" # ") 2251 ) 2252 for package, size in download: 2253 package = darkred(os.path.basename(package)) 2254 size = blue(self.entropyTools.bytes_into_human(size)) 2255 self.Entropy.updateProgress( 2256 "[branch:%s|%s] %s [%s]" % ( 2257 brown(branch), 2258 darkred(_("download")), 2259 blue(package), 2260 size, 2261 ), 2262 importance = 0, 2263 type = "info", 2264 header = red(" # ") 2265 ) 2266 for package, size in copy: 2267 package = darkblue(os.path.basename(package)) 2268 size = blue(self.entropyTools.bytes_into_human(size)) 2269 self.Entropy.updateProgress( 2270 "[branch:%s|%s] %s [%s]" % ( 2271 brown(branch), 2272 darkgreen(_("copy")), 2273 brown(package), 2274 size, 2275 ), 2276 importance = 0, 2277 type = "info", 2278 header = red(" # ") 2279 ) 2280 for package, size in removal: 2281 package = brown(os.path.basename(package)) 2282 size = blue(self.entropyTools.bytes_into_human(size)) 2283 self.Entropy.updateProgress( 2284 "[branch:%s|%s] %s [%s]" % ( 2285 brown(branch), 2286 red(_("remove")), 2287 red(package), 2288 size, 2289 ), 2290 importance = 0, 2291 type = "info", 2292 header = red(" # ") 2293 ) 2294 2295 self.Entropy.updateProgress( 2296 "%s:\t\t\t%s" % ( 2297 blue(_("Packages to be removed")), 2298 darkred(str(len(removal))), 2299 ), 2300 importance = 0, 2301 type = "info", 2302 header = blue(" @@ ") 2303 ) 2304 self.Entropy.updateProgress( 2305 "%s:\t\t%s" % ( 2306 darkgreen(_("Packages to be moved locally")), 2307 darkgreen(str(len(copy))), 2308 ), 2309 importance = 0, 2310 type = "info", 2311 header = blue(" @@ ") 2312 ) 2313 self.Entropy.updateProgress( 2314 "%s:\t\t\t%s" % ( 2315 bold(_("Packages to be uploaded")), 2316 bold(str(len(upload))), 2317 ), 2318 importance = 0, 2319 type = "info", 2320 header = blue(" @@ ") 2321 ) 2322 2323 self.Entropy.updateProgress( 2324 "%s:\t\t\t%s" % ( 2325 darkred(_("Total removal size")), 2326 darkred( 2327 self.entropyTools.bytes_into_human(metainfo['removal']) 2328 ), 2329 ), 2330 importance = 0, 2331 type = "info", 2332 header = blue(" @@ ") 2333 ) 2334 2335 self.Entropy.updateProgress( 2336 "%s:\t\t\t%s" % ( 2337 blue(_("Total upload size")), 2338 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2339 ), 2340 importance = 0, 2341 type = "info", 2342 header = blue(" @@ ") 2343 ) 2344 self.Entropy.updateProgress( 2345 "%s:\t\t\t%s" % ( 2346 brown(_("Total download size")), 2347 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2348 ), 2349 importance = 0, 2350 type = "info", 2351 header = blue(" @@ ") 2352 )

2353 2354

2355 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2356 repo = None):

2357 2358 remote_files = 0 2359 close_conn = False 2360 remote_packages_data = {} 2361 2362 my_path = os.path.join( 2363 self.Entropy.get_remote_packages_relative_path(repo), branch) 2364 if ftp_connection == None: 2365 close_conn = True 2366 # let it raise an exception if things go bad 2367 ftp_connection = FtpInterface(uri, self.Entropy) 2368 ftp_connection.set_cwd(my_path, dodir = True) 2369 2370 remote_packages = ftp_connection.list_dir() 2371 remote_packages_info = ftp_connection.get_raw_list() 2372 if close_conn: 2373 ftp_connection.close() 2374 2375 for tbz2 in remote_packages: 2376 if tbz2.endswith(etpConst['packagesext']): 2377 remote_files += 1 2378 2379 for remote_package in remote_packages_info: 2380 remote_packages_data[remote_package.split()[8]] = \ 2381 int(remote_package.split()[4]) 2382 2383 return remote_files, remote_packages, remote_packages_data

2384

2385 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2386 2387 if repo == None: 2388 repo = self.Entropy.default_repository 2389 2390 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2391 upload_files, upload_packages = self.calculate_local_upload_files( 2392 branch, repo) 2393 local_files, local_packages = self.calculate_local_package_files(branch, 2394 repo) 2395 self._show_local_sync_stats(upload_files, local_files) 2396 2397 self.Entropy.updateProgress( 2398 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2399 importance = 1, 2400 type = "info", 2401 header = red(" @@ ") 2402 ) 2403 remote_files, remote_packages, remote_packages_data = \ 2404 self.calculate_remote_package_files(uri, branch, repo = repo) 2405 self.Entropy.updateProgress( 2406 "%s:\t\t\t%s %s" % ( 2407 blue(_("remote packages")), 2408 bold(str(remote_files)), 2409 red(_("files stored")), 2410 ), 2411 importance = 0, 2412 type = "info", 2413 header = red(" @@ ") 2414 ) 2415 2416 mytxt = blue("%s ...") % ( 2417 _("Calculating queues"), 2418 ) 2419 self.Entropy.updateProgress( 2420 mytxt, 2421 importance = 1, 2422 type = "info", 2423 header = red(" @@ ") 2424 ) 2425 2426 upload_queue, download_queue, removal_queue, fine_queue = \ 2427 self.calculate_sync_queues(upload_packages, local_packages, 2428 remote_packages, remote_packages_data, branch, repo) 2429 return upload_queue, download_queue, removal_queue, fine_queue, \ 2430 remote_packages_data

2431

2432 - def calculate_sync_queues( 2433 self, 2434 upload_packages, 2435 local_packages, 2436 remote_packages, 2437 remote_packages_data, 2438 branch, 2439 repo = None 2440 ):

2441 2442 upload_queue = set() 2443 download_queue = set() 2444 removal_queue = set() 2445 fine_queue = set() 2446 2447 for local_package in upload_packages: 2448 if local_package in remote_packages: 2449 2450 local_filepath = os.path.join( 2451 self.Entropy.get_local_upload_directory(repo), branch, 2452 local_package) 2453 2454 local_size = self.entropyTools.get_file_size(local_filepath) 2455 remote_size = remote_packages_data.get(local_package) 2456 if remote_size == None: 2457 remote_size = 0 2458 if (local_size != remote_size): 2459 # size does not match, adding to the upload queue 2460 upload_queue.add(local_package) 2461 else: 2462 # just move from upload to packages 2463 fine_queue.add(local_package) 2464 else: 2465 # always force upload of packages in uploaddir 2466 upload_queue.add(local_package) 2467 2468 # if a package is in the packages directory but not online, 2469 # we have to upload it we have local_packages and remote_packages 2470 for local_package in local_packages: 2471 if local_package in remote_packages: 2472 local_filepath = os.path.join( 2473 self.Entropy.get_local_packages_directory(repo), branch, 2474 local_package) 2475 local_size = self.entropyTools.get_file_size(local_filepath) 2476 remote_size = remote_packages_data.get(local_package) 2477 if remote_size == None: 2478 remote_size = 0 2479 if (local_size != remote_size) and (local_size != 0): 2480 # size does not match, adding to the upload queue 2481 if local_package not in fine_queue: 2482 upload_queue.add(local_package) 2483 else: 2484 # this means that the local package does not exist 2485 # so, we need to download it 2486 upload_queue.add(local_package) 2487 2488 # Fill download_queue and removal_queue 2489 for remote_package in remote_packages: 2490 if remote_package in local_packages: 2491 local_filepath = os.path.join( 2492 self.Entropy.get_local_packages_directory(repo), branch, 2493 remote_package) 2494 local_size = self.entropyTools.get_file_size(local_filepath) 2495 remote_size = remote_packages_data.get(remote_package) 2496 if remote_size == None: 2497 remote_size = 0 2498 if (local_size != remote_size) and (local_size != 0): 2499 # size does not match, remove first 2500 # do it only if the package has not been 2501 # added to the upload_queue 2502 if remote_package not in upload_queue: 2503 # remotePackage == localPackage 2504 # just remove something that differs 2505 # from the content of the mirror 2506 removal_queue.add(remote_package) 2507 # then add to the download queue 2508 download_queue.add(remote_package) 2509 else: 2510 # this means that the local package does not exist 2511 # so, we need to download it 2512 # ignore .tmp files 2513 if not remote_package.endswith(".tmp"): 2514 download_queue.add(remote_package) 2515 2516 # Collect packages that don't exist anymore in the database 2517 # so we can filter them out from the download queue 2518 dbconn = self.Entropy.open_server_repository(just_reading = True, 2519 repo = repo) 2520 db_files = dbconn.listAllDownloads(do_sort = False, 2521 full_path = True) 2522 db_files = set([os.path.basename(x) for x in db_files if \ 2523 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2524 2525 exclude = set() 2526 for myfile in download_queue: 2527 if myfile.endswith(etpConst['packagesext']): 2528 if myfile not in db_files: 2529 exclude.add(myfile) 2530 download_queue -= exclude 2531 2532 exclude = set() 2533 for myfile in upload_queue: 2534 if myfile.endswith(etpConst['packagesext']): 2535 if myfile not in db_files: 2536 exclude.add(myfile) 2537 upload_queue -= exclude 2538 2539 exclude = set() 2540 for myfile in download_queue: 2541 if myfile in upload_queue: 2542 exclude.add(myfile) 2543 download_queue -= exclude 2544 2545 return upload_queue, download_queue, removal_queue, fine_queue

2546 2547

2548 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2549 remote_packages_data, branch, repo):

2550 2551 metainfo = { 2552 'removal': 0, 2553 'download': 0, 2554 'upload': 0, 2555 } 2556 removal = [] 2557 download = [] 2558 do_copy = [] 2559 upload = [] 2560 2561 for item in removal_queue: 2562 if not item.endswith(etpConst['packagesext']): 2563 continue 2564 local_filepath = os.path.join( 2565 self.Entropy.get_local_packages_directory(repo), branch, item) 2566 size = self.entropyTools.get_file_size(local_filepath) 2567 metainfo['removal'] += size 2568 removal.append((local_filepath, size)) 2569 2570 for item in download_queue: 2571 if not item.endswith(etpConst['packagesext']): 2572 continue 2573 local_filepath = os.path.join( 2574 self.Entropy.get_local_upload_directory(repo), branch, item) 2575 if not os.path.isfile(local_filepath): 2576 size = remote_packages_data.get(item) 2577 if size == None: 2578 size = 0 2579 size = int(size) 2580 metainfo['removal'] += size 2581 download.append((local_filepath, size)) 2582 else: 2583 size = self.entropyTools.get_file_size(local_filepath) 2584 do_copy.append((local_filepath, size)) 2585 2586 for item in upload_queue: 2587 if not item.endswith(etpConst['packagesext']): 2588 continue 2589 local_filepath = os.path.join( 2590 self.Entropy.get_local_upload_directory(repo), branch, item) 2591 local_filepath_pkgs = os.path.join( 2592 self.Entropy.get_local_packages_directory(repo), branch, item) 2593 if os.path.isfile(local_filepath): 2594 size = self.entropyTools.get_file_size(local_filepath) 2595 upload.append((local_filepath, size)) 2596 else: 2597 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2598 upload.append((local_filepath_pkgs, size)) 2599 metainfo['upload'] += size 2600 2601 return upload, download, removal, do_copy, metainfo

2602 2603

2604 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2605 2606 if repo == None: 2607 repo = self.Entropy.default_repository 2608 2609 for itemdata in removal_queue: 2610 2611 remove_filename = itemdata[0] 2612 remove_filepath = os.path.join( 2613 self.Entropy.get_local_packages_directory(repo), branch, 2614 remove_filename) 2615 remove_filepath_hash = remove_filepath + \ 2616 etpConst['packagesmd5fileext'] 2617 self.Entropy.updateProgress( 2618 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2619 brown(repo), 2620 red("sync"), 2621 brown(branch), 2622 blue(_("removing package+hash")), 2623 darkgreen(remove_filename), 2624 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2625 ), 2626 importance = 0, 2627 type = "info", 2628 header = darkred(" * ") 2629 ) 2630 2631 if os.path.isfile(remove_filepath): 2632 os.remove(remove_filepath) 2633 if os.path.isfile(remove_filepath_hash): 2634 os.remove(remove_filepath_hash) 2635 2636 self.Entropy.updateProgress( 2637 "[repo:%s|%s|%s] %s" % ( 2638 brown(repo), 2639 red(_("sync")), 2640 brown(branch), 2641 blue(_("removal complete")), 2642 ), 2643 importance = 0, 2644 type = "info", 2645 header = darkred(" * ") 2646 )

2647 2648

2649 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2650 2651 if repo == None: 2652 repo = self.Entropy.default_repository 2653 2654 for itemdata in copy_queue: 2655 2656 from_file = itemdata[0] 2657 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2658 to_file = os.path.join( 2659 self.Entropy.get_local_packages_directory(repo), branch, 2660 os.path.basename(from_file)) 2661 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2662 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2663 self.Entropy.updateProgress( 2664 "[repo:%s|%s|%s] %s: %s" % ( 2665 brown(repo), 2666 red("sync"), 2667 brown(branch), 2668 blue(_("copying file+hash to repository")), 2669 darkgreen(from_file), 2670 ), 2671 importance = 0, 2672 type = "info", 2673 header = darkred(" * ") 2674 ) 2675 2676 if not os.path.isdir(os.path.dirname(to_file)): 2677 os.makedirs(os.path.dirname(to_file)) 2678 2679 shutil.copy2(from_file, to_file) 2680 if not os.path.isfile(from_file_hash): 2681 self.create_file_checksum(from_file, from_file_hash) 2682 shutil.copy2(from_file_hash, to_file_hash) 2683 2684 # clear expiration file 2685 if os.path.isfile(expiration_file): 2686 os.remove(expiration_file)

2687 2688

2689 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2690 2691 if repo == None: 2692 repo = self.Entropy.default_repository 2693 2694 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2695 myqueue = [] 2696 for itemdata in upload_queue: 2697 upload_item = itemdata[0] 2698 hash_file = upload_item + etpConst['packagesmd5fileext'] 2699 if not os.path.isfile(hash_file): 2700 self.entropyTools.create_md5_file(upload_item) 2701 myqueue.append(hash_file) 2702 myqueue.append(upload_item) 2703 2704 ftp_basedir = os.path.join( 2705 self.Entropy.get_remote_packages_relative_path(repo), branch) 2706 uploader = self.FtpServerHandler(FtpInterface, 2707 self.Entropy, [uri], myqueue, critical_files = myqueue, 2708 use_handlers = True, ftp_basedir = ftp_basedir, 2709 handlers_data = {'branch': branch }, repo = repo) 2710 errors, m_fine_uris, m_broken_uris = uploader.go() 2711 if errors: 2712 my_broken_uris = [ 2713 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2714 x in m_broken_uris] 2715 reason = my_broken_uris[0][1] 2716 self.Entropy.updateProgress( 2717 "[branch:%s] %s: %s, %s: %s" % ( 2718 brown(branch), 2719 blue(_("upload errors")), 2720 red(crippled_uri), 2721 blue(_("reason")), 2722 darkgreen(unicode(reason)), 2723 ), 2724 importance = 1, 2725 type = "error", 2726 header = darkred(" !!! ") 2727 ) 2728 return errors, m_fine_uris, m_broken_uris 2729 2730 self.Entropy.updateProgress( 2731 "[branch:%s] %s: %s" % ( 2732 brown(branch), 2733 blue(_("upload completed successfully")), 2734 red(crippled_uri), 2735 ), 2736 importance = 1, 2737 type = "info", 2738 header = blue(" @@ ") 2739 ) 2740 return errors, m_fine_uris, m_broken_uris

2741 2742

2743 - def _sync_run_download_queue(self, uri, download_queue, branch, 2744 repo = None):

2745 2746 if repo == None: 2747 repo = self.Entropy.default_repository 2748 2749 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2750 myqueue = [] 2751 for package in download_queue: 2752 hash_file = package + etpConst['packagesmd5fileext'] 2753 myqueue.append(package) 2754 myqueue.append(hash_file) 2755 2756 ftp_basedir = os.path.join( 2757 self.Entropy.get_remote_packages_relative_path(repo), branch) 2758 local_basedir = os.path.join( 2759 self.Entropy.get_local_packages_directory(repo), branch) 2760 downloader = self.FtpServerHandler( 2761 FtpInterface, self.Entropy, [uri], myqueue, 2762 critical_files = myqueue, use_handlers = True, 2763 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2764 handlers_data = {'branch': branch }, download = True, repo = repo) 2765 2766 errors, m_fine_uris, m_broken_uris = downloader.go() 2767 if errors: 2768 my_broken_uris = [ 2769 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2770 for x, y in m_broken_uris] 2771 reason = my_broken_uris[0][1] 2772 self.Entropy.updateProgress( 2773 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2774 brown(repo), 2775 red(_("sync")), 2776 brown(branch), 2777 blue(_("download errors")), 2778 darkgreen(crippled_uri), 2779 blue(_("reason")), 2780 reason, 2781 ), 2782 importance = 1, 2783 type = "error", 2784 header = darkred(" !!! ") 2785 ) 2786 return errors, m_fine_uris, m_broken_uris 2787 2788 self.Entropy.updateProgress( 2789 "[repo:%s|%s|%s] %s: %s" % ( 2790 brown(repo), 2791 red(_("sync")), 2792 brown(branch), 2793 blue(_("download completed successfully")), 2794 darkgreen(crippled_uri), 2795 ), 2796 importance = 1, 2797 type = "info", 2798 header = darkgreen(" * ") 2799 ) 2800 return errors, m_fine_uris, m_broken_uris

2801

2802 - def run_package_files_qa_checks(self, packages_list, repo = None):

2803 2804 if repo == None: 2805 repo = self.Entropy.default_repository 2806 2807 my_qa = self.Entropy.QA() 2808 2809 qa_total = len(packages_list) 2810 qa_count = 0 2811 qa_some_faulty = [] 2812 2813 for upload_package in packages_list: 2814 qa_count += 1 2815 2816 self.Entropy.updateProgress( 2817 "%s: %s" % ( 2818 purple(_("QA checking package file")), 2819 darkgreen(os.path.basename(upload_package)), 2820 ), 2821 importance = 0, 2822 type = "info", 2823 header = purple(" @@ "), 2824 back = True, 2825 count = (qa_count, qa_total,) 2826 ) 2827 2828 result = my_qa.entropy_package_checks(upload_package) 2829 if not result: 2830 # call wolfman-911 2831 qa_some_faulty.append(os.path.basename(upload_package)) 2832 2833 if qa_some_faulty: 2834 2835 for qa_faulty_pkg in qa_some_faulty: 2836 self.Entropy.updateProgress( 2837 "[repo:%s|branch:%s] %s: %s" % ( 2838 brown(repo), 2839 self.SystemSettings['repositories']['branch'], 2840 red(_("faulty package file, please fix")), 2841 blue(os.path.basename(qa_faulty_pkg)), 2842 ), 2843 importance = 1, 2844 type = "error", 2845 header = darkred(" @@ ") 2846 ) 2847 raise EntropyPackageException( 2848 'EntropyPackageException: cannot continue')

2849 2850

2851 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2852 repo = None):

2853 2854 if repo == None: 2855 repo = self.Entropy.default_repository 2856 2857 self.Entropy.updateProgress( 2858 "[repo:%s|%s] %s" % ( 2859 repo, 2860 red(_("sync")), 2861 darkgreen(_("starting packages sync")), 2862 ), 2863 importance = 1, 2864 type = "info", 2865 header = red(" @@ "), 2866 back = True 2867 ) 2868 2869 2870 successfull_mirrors = set() 2871 broken_mirrors = set() 2872 check_data = () 2873 upload_queue_qa_checked = set() 2874 mirrors_tainted = False 2875 mirror_errors = False 2876 mirrors_errors = False 2877 2878 for uri in self.Entropy.get_remote_mirrors(repo): 2879 2880 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2881 mirror_errors = False 2882 2883 self.Entropy.updateProgress( 2884 "[repo:%s|%s|branch:%s] %s: %s" % ( 2885 repo, 2886 red(_("sync")), 2887 brown(self.SystemSettings['repositories']['branch']), 2888 blue(_("packages sync")), 2889 bold(crippled_uri), 2890 ), 2891 importance = 1, 2892 type = "info", 2893 header = red(" @@ ") 2894 ) 2895 2896 try: 2897 upload_queue, download_queue, removal_queue, fine_queue, \ 2898 remote_packages_data = self.calculate_packages_to_sync(uri, 2899 self.SystemSettings['repositories']['branch'], repo) 2900 except self.socket.error, err: 2901 self.Entropy.updateProgress( 2902 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2903 repo, 2904 red(_("sync")), 2905 self.SystemSettings['repositories']['branch'], 2906 darkred(_("socket error")), 2907 err, 2908 darkred(_("on")), 2909 crippled_uri, 2910 ), 2911 importance = 1, 2912 type = "error", 2913 header = darkgreen(" * ") 2914 ) 2915 continue 2916 2917 if (not upload_queue) and (not download_queue) and \ 2918 (not removal_queue): 2919 self.Entropy.updateProgress( 2920 "[repo:%s|%s|branch:%s] %s: %s" % ( 2921 repo, 2922 red(_("sync")), 2923 self.SystemSettings['repositories']['branch'], 2924 darkgreen(_("nothing to do on")), 2925 crippled_uri, 2926 ), 2927 importance = 1, 2928 type = "info", 2929 header = darkgreen(" * ") 2930 ) 2931 successfull_mirrors.add(uri) 2932 continue 2933 2934 self.Entropy.updateProgress( 2935 "%s:" % (blue(_("Expanding queues")),), 2936 importance = 1, 2937 type = "info", 2938 header = red(" ** ") 2939 ) 2940 2941 upload, download, removal, copy, metainfo = self.expand_queues( 2942 upload_queue, 2943 download_queue, 2944 removal_queue, 2945 remote_packages_data, 2946 self.SystemSettings['repositories']['branch'], 2947 repo 2948 ) 2949 del upload_queue, download_queue, removal_queue, \ 2950 remote_packages_data 2951 self._show_sync_queues(upload, download, removal, copy, metainfo, 2952 self.SystemSettings['repositories']['branch']) 2953 2954 if not len(upload)+len(download)+len(removal)+len(copy): 2955 2956 self.Entropy.updateProgress( 2957 "[repo:%s|%s|branch:%s] %s %s" % ( 2958 self.Entropy.default_repository, 2959 red(_("sync")), 2960 self.SystemSettings['repositories']['branch'], 2961 blue(_("nothing to sync for")), 2962 crippled_uri, 2963 ), 2964 importance = 1, 2965 type = "info", 2966 header = darkgreen(" @@ ") 2967 ) 2968 2969 successfull_mirrors.add(uri) 2970 continue 2971 2972 if pretend: 2973 successfull_mirrors.add(uri) 2974 continue 2975 2976 if ask: 2977 rc_sync = self.Entropy.askQuestion( 2978 _("Would you like to run the steps above ?")) 2979 if rc_sync == "No": 2980 continue 2981 2982 try: 2983 2984 # QA checks 2985 qa_package_files = [x[0] for x in upload if x[0] \ 2986 not in upload_queue_qa_checked] 2987 upload_queue_qa_checked |= set(qa_package_files) 2988 2989 self.run_package_files_qa_checks(qa_package_files, repo = repo) 2990 2991 if removal: 2992 self._sync_run_removal_queue(removal, 2993 self.SystemSettings['repositories']['branch'], repo) 2994 2995 if copy: 2996 self._sync_run_copy_queue(copy, 2997 self.SystemSettings['repositories']['branch'], repo) 2998 2999 if upload or download: 3000 mirrors_tainted = True 3001 3002 if upload: 3003 d_errors, m_fine_uris, \ 3004 m_broken_uris = self._sync_run_upload_queue( 3005 uri, upload, 3006 self.SystemSettings['repositories']['branch'], repo) 3007 3008 if d_errors: 3009 mirror_errors = True 3010 3011 if download: 3012 my_downlist = [x[0] for x in download] 3013 d_errors, m_fine_uris, \ 3014 m_broken_uris = self._sync_run_download_queue( 3015 uri, my_downlist, 3016 self.SystemSettings['repositories']['branch'], repo) 3017 3018 if d_errors: 3019 mirror_errors = True 3020 if not mirror_errors: 3021 successfull_mirrors.add(uri) 3022 else: 3023 mirrors_errors = True 3024 3025 except KeyboardInterrupt: 3026 self.Entropy.updateProgress( 3027 "[repo:%s|%s|branch:%s] %s" % ( 3028 repo, 3029 red(_("sync")), 3030 self.SystemSettings['repositories']['branch'], 3031 darkgreen(_("keyboard interrupt !")), 3032 ), 3033 importance = 1, 3034 type = "info", 3035 header = darkgreen(" * ") 3036 ) 3037 continue 3038 3039 except EntropyPackageException: 3040 3041 mirrors_errors = True 3042 broken_mirrors.add(uri) 3043 successfull_mirrors.clear() 3044 # so that people will realize this is a very bad thing 3045 self.Entropy.updateProgress( 3046 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3047 repo, 3048 red(_("sync")), 3049 self.SystemSettings['repositories']['branch'], 3050 darkred(_("you must package them again")), 3051 Exception, 3052 _("error"), 3053 err, 3054 ), 3055 importance = 1, 3056 type = "error", 3057 header = darkred(" !!! ") 3058 ) 3059 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3060 broken_mirrors, check_data 3061 3062 except Exception, err: 3063 3064 self.entropyTools.print_traceback() 3065 mirrors_errors = True 3066 broken_mirrors.add(uri) 3067 self.Entropy.updateProgress( 3068 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3069 repo, 3070 red(_("sync")), 3071 self.SystemSettings['repositories']['branch'], 3072 darkred(_("exception caught")), 3073 Exception, 3074 _("error"), 3075 err, 3076 ), 3077 importance = 1, 3078 type = "error", 3079 header = darkred(" !!! ") 3080 ) 3081 3082 exc_txt = self.Entropy.entropyTools.print_exception( 3083 returndata = True) 3084 for line in exc_txt: 3085 self.Entropy.updateProgress( 3086 unicode(line), 3087 importance = 1, 3088 type = "error", 3089 header = darkred(": ") 3090 ) 3091 3092 if len(successfull_mirrors) > 0: 3093 self.Entropy.updateProgress( 3094 "[repo:%s|%s|branch:%s] %s" % ( 3095 repo, 3096 red(_("sync")), 3097 self.SystemSettings['repositories']['branch'], 3098 darkred( 3099 _("at least one mirror synced properly!")), 3100 ), 3101 importance = 1, 3102 type = "error", 3103 header = darkred(" !!! ") 3104 ) 3105 continue 3106 3107 # if at least one server has been synced successfully, move files 3108 if (len(successfull_mirrors) > 0) and not pretend: 3109 self.remove_expiration_files( 3110 self.SystemSettings['repositories']['branch'], repo) 3111 3112 if packages_check: 3113 check_data = self.Entropy.verify_local_packages([], ask = ask, 3114 repo = repo) 3115 3116 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3117 broken_mirrors, check_data

3118

3119 - def remove_expiration_files(self, branch, repo = None):

3120 3121 if repo == None: 3122 repo = self.Entropy.default_repository 3123 3124 branch_dir = os.path.join( 3125 self.Entropy.get_local_upload_directory(repo), branch) 3126 3127 # check if it exists 3128 if not os.path.isdir(branch_dir): 3129 return None 3130 3131 branchcontent = os.listdir(branch_dir) 3132 for xfile in branchcontent: 3133 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3134 branch, xfile) 3135 destdir = os.path.join( 3136 self.Entropy.get_local_packages_directory(repo), branch) 3137 if not os.path.isdir(destdir): 3138 os.makedirs(destdir) 3139 dest = os.path.join(destdir, xfile) 3140 shutil.move(source, dest) 3141 # clear expiration file 3142 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3143 if os.path.isfile(dest_expiration): 3144 os.remove(dest_expiration)

3145 3146

3147 - def is_package_expired(self, package_file, branch, repo = None):

3148 3149 pkg_path = os.path.join( 3150 self.Entropy.get_local_packages_directory(repo), branch, 3151 package_file) 3152 pkg_path += etpConst['packagesexpirationfileext'] 3153 if not os.path.isfile(pkg_path): 3154 return False 3155 3156 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3157 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3158 days = srv_set['packages_expiration_days'] 3159 delta = int(days)*24*3600 3160 currmtime = time.time() 3161 file_delta = currmtime - mtime 3162 3163 if file_delta > delta: 3164 return True 3165 return False

3166

3167 - def create_expiration_file(self, package_file, branch, repo = None, 3168 gentle = False):

3169 3170 pkg_path = os.path.join( 3171 self.Entropy.get_local_packages_directory(repo), branch, 3172 package_file) 3173 pkg_path += etpConst['packagesexpirationfileext'] 3174 if gentle and os.path.isfile(pkg_path): 3175 return 3176 f_exp = open(pkg_path,"w") 3177 f_exp.flush() 3178 f_exp.close()

3179 3180

3181 - def collect_expiring_packages(self, branch, repo = None):

3182 3183 dbconn = self.Entropy.open_server_repository(just_reading = True, 3184 repo = repo) 3185 database_bins = dbconn.listAllDownloads(do_sort = False, 3186 full_path = True) 3187 bins_dir = os.path.join( 3188 self.Entropy.get_local_packages_directory(repo), branch) 3189 repo_bins = set() 3190 3191 if os.path.isdir(bins_dir): 3192 repo_bins = os.listdir(bins_dir) 3193 repo_bins = set([ 3194 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3195 in repo_bins if x.endswith(etpConst['packagesext'])]) 3196 repo_bins -= database_bins 3197 3198 return set([os.path.basename(x) for x in repo_bins])

3199 3200

3201 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3202 3203 if repo == None: 3204 repo = self.Entropy.default_repository 3205 3206 self.Entropy.updateProgress( 3207 "[repo:%s|%s|branch:%s] %s" % ( 3208 brown(repo), 3209 red(_("tidy")), 3210 blue(self.SystemSettings['repositories']['branch']), 3211 blue(_("collecting expired packages")), 3212 ), 3213 importance = 1, 3214 type = "info", 3215 header = red(" @@ ") 3216 ) 3217 3218 branch_data = {} 3219 errors = False 3220 branch_data['errors'] = False 3221 branch = self.SystemSettings['repositories']['branch'] 3222 3223 self.Entropy.updateProgress( 3224 "[branch:%s] %s" % ( 3225 brown(branch), 3226 blue(_("collecting expired packages in the selected branches")), 3227 ), 3228 importance = 1, 3229 type = "info", 3230 header = blue(" @@ ") 3231 ) 3232 3233 # collect removed packages 3234 expiring_packages = self.collect_expiring_packages(branch, repo) 3235 if expiring_packages: 3236 3237 # filter expired packages used by other branches 3238 # this is done for the sake of consistency 3239 # --- read packages.db.pkglist, make sure your repository 3240 # has been ported to latest Entropy 3241 3242 branch_pkglist_data = self.read_remote_file_in_branches( 3243 etpConst['etpdatabasepkglist'], repo = repo, 3244 excluded_branches = [branch]) 3245 # format data 3246 for key, val in branch_pkglist_data.items(): 3247 branch_pkglist_data[key] = val.split("\n") 3248 3249 3250 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3251 branch) 3252 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3253 expiring_packages]) 3254 3255 for other_branch in branch_pkglist_data: 3256 branch_pkglist = set(branch_pkglist_data[other_branch]) 3257 my_expiring_pkgs -= branch_pkglist 3258 3259 # fallback to normality, set new expiring packages var 3260 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3261 3262 removal = [] 3263 for package in expiring_packages: 3264 expired = self.is_package_expired(package, branch, repo) 3265 if expired: 3266 removal.append(package) 3267 else: 3268 self.create_expiration_file(package, branch, repo, 3269 gentle = True) 3270 3271 # fill returning data 3272 branch_data['removal'] = removal[:] 3273 3274 if not removal: 3275 self.Entropy.updateProgress( 3276 "[branch:%s] %s" % ( 3277 brown(branch), 3278 blue(_("nothing to remove on this branch")), 3279 ), 3280 importance = 1, 3281 type = "info", 3282 header = blue(" @@ ") 3283 ) 3284 return errors, branch_data 3285 else: 3286 self.Entropy.updateProgress( 3287 "[branch:%s] %s:" % ( 3288 brown(branch), 3289 blue(_("these are the expired packages")), 3290 ), 3291 importance = 1, 3292 type = "info", 3293 header = blue(" @@ ") 3294 ) 3295 for package in removal: 3296 self.Entropy.updateProgress( 3297 "[branch:%s] %s: %s" % ( 3298 brown(branch), 3299 blue(_("remove")), 3300 darkgreen(package), 3301 ), 3302 importance = 1, 3303 type = "info", 3304 header = brown(" # ") 3305 ) 3306 3307 if pretend: 3308 return errors, branch_data 3309 3310 if ask: 3311 rc_question = self.Entropy.askQuestion( 3312 _("Would you like to continue ?")) 3313 if rc_question == "No": 3314 return errors, branch_data 3315 3316 myqueue = [] 3317 for package in removal: 3318 myqueue.append(package+etpConst['packagesmd5fileext']) 3319 myqueue.append(package) 3320 3321 ftp_basedir = os.path.join( 3322 self.Entropy.get_remote_packages_relative_path(repo), branch) 3323 for uri in self.Entropy.get_remote_mirrors(repo): 3324 3325 self.Entropy.updateProgress( 3326 "[branch:%s] %s..." % ( 3327 brown(branch), 3328 blue(_("removing packages remotely")), 3329 ), 3330 importance = 1, 3331 type = "info", 3332 header = blue(" @@ ") 3333 ) 3334 3335 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3336 destroyer = self.FtpServerHandler( 3337 FtpInterface, 3338 self.Entropy, 3339 [uri], 3340 myqueue, 3341 critical_files = [], 3342 ftp_basedir = ftp_basedir, 3343 remove = True, 3344 repo = repo 3345 ) 3346 errors, m_fine_uris, m_broken_uris = destroyer.go() 3347 if errors: 3348 my_broken_uris = [ 3349 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3350 for x in m_broken_uris] 3351 3352 reason = my_broken_uris[0][1] 3353 self.Entropy.updateProgress( 3354 "[branch:%s] %s: %s, %s: %s" % ( 3355 brown(branch), 3356 blue(_("remove errors")), 3357 red(crippled_uri), 3358 blue(_("reason")), 3359 reason, 3360 ), 3361 importance = 1, 3362 type = "warning", 3363 header = brown(" !!! ") 3364 ) 3365 branch_data['errors'] = True 3366 errors = True 3367 3368 self.Entropy.updateProgress( 3369 "[branch:%s] %s..." % ( 3370 brown(branch), 3371 blue(_("removing packages locally")), 3372 ), 3373 importance = 1, 3374 type = "info", 3375 header = blue(" @@ ") 3376 ) 3377 3378 branch_data['removed'] = set() 3379 for package in removal: 3380 package_path = os.path.join( 3381 self.Entropy.get_local_packages_directory(repo), 3382 branch, package) 3383 package_path_hash = package_path + \ 3384 etpConst['packagesmd5fileext'] 3385 package_path_expired = package_path + \ 3386 etpConst['packagesexpirationfileext'] 3387 3388 my_rm_list = (package_path_hash, package_path, 3389 package_path_expired) 3390 for myfile in my_rm_list: 3391 if os.path.isfile(myfile): 3392 self.Entropy.updateProgress( 3393 "[branch:%s] %s: %s" % ( 3394 brown(branch), 3395 blue(_("removing")), 3396 darkgreen(myfile), 3397 ), 3398 importance = 1, 3399 type = "info", 3400 header = brown(" @@ ") 3401 ) 3402 os.remove(myfile) 3403 branch_data['removed'].add(myfile) 3404 3405 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors