entropy.server.interfaces.mirrors

39 40 import socket 41 import entropy.dump as dumpTools 42 import entropy.tools as entropyTools

43 - def __init__(self, ServerInstance, repo = None):

44 45 from entropy.server.interfaces.main import Server as MainServer 46 if not isinstance(ServerInstance, MainServer): 47 mytxt = _("entropy.server.interfaces.main.Server needed") 48 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 49 50 self.Entropy = ServerInstance 51 from entropy.transceivers import FtpServerHandler 52 self.FtpServerHandler = FtpServerHandler 53 from entropy.cache import EntropyCacher 54 self.Cacher = EntropyCacher() 55 self.sys_settings_plugin_id = \ 56 etpConst['system_settings_plugins_ids']['server_plugin'] 57 self.SystemSettings = self.Entropy.SystemSettings 58 59 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 60 self.Entropy.updateProgress( 61 mytxt, 62 importance = 2, 63 type = "info", 64 header = red(" @@ ") 65 ) 66 mytxt = _("mirror") 67 for mirror in self.Entropy.get_remote_mirrors(repo): 68 mirror = self.entropyTools.hide_ftp_password(mirror) 69 self.Entropy.updateProgress( 70 blue("%s: %s") % (mytxt, darkgreen(mirror),), 71 importance = 0, 72 type = "info", 73 header = brown(" # ") 74 )

75

76 - def get_remote_branches(self, repo = None):

77 """ 78 Returns a list of remotely available branches for the provided 79 repository identifier. 80 81 @keyword repo: repository identifier 82 @type repo: string 83 @return: list of valid, available remote branches 84 @rtype: set 85 """ 86 87 if repo == None: 88 repo = self.Entropy.default_repository 89 90 remote_branches = set() 91 # this is used to validate Entropy repository path 92 ts_file = etpConst['etpdatabasetimestampfile'] 93 mirrors = self.Entropy.get_remote_mirrors(repo) 94 for uri in mirrors: 95 96 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 97 98 self.Entropy.updateProgress( 99 "[repo:%s] %s: %s" % ( 100 brown(repo), 101 blue(_("listing branches in mirror")), 102 darkgreen(crippled_uri), 103 ), 104 importance = 1, 105 type = "info", 106 header = brown(" @@ ") 107 ) 108 109 ftp = FtpInterface(uri, self.Entropy, verbose = False) 110 try: 111 112 branches_path = self.Entropy.get_remote_database_relative_path( 113 repo) 114 try: 115 ftp.set_cwd(branches_path) 116 except FtpError: 117 continue # no branches in this mirror 118 119 branches = ftp.list_dir() 120 for branch in branches: 121 mypath = os.path.join("/", branches_path, branch) 122 try: 123 ftp.set_cwd(mypath) 124 except FtpError: 125 # not a dir 126 continue 127 if ftp.is_file_available(ts_file): 128 remote_branches.add(branch) 129 130 finally: 131 ftp.close() 132 133 return remote_branches

134

135 - def read_remote_file_in_branches(self, filename, repo = None, 136 excluded_branches = None):

137 """ 138 Reads a file remotely located in all the available branches. 139 140 @param filename: name of the file that should be located inside 141 repository database directory 142 @type filename: string 143 @keyword repo: repository identifier 144 @type repo: string 145 @keyword excluded_branches: list of branch identifiers excluded or None 146 @type excluded_branches: list or None 147 @return: dictionary with branches as key and raw file content as value: 148 {'4': 'abcd\n', '5': 'defg\n'} 149 @rtype: dict 150 """ 151 if repo == None: 152 repo = self.Entropy.default_repository 153 if excluded_branches == None: 154 excluded_branches = [] 155 156 branch_data = {} 157 mirrors = self.Entropy.get_remote_mirrors(repo) 158 for uri in mirrors: 159 160 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 161 162 self.Entropy.updateProgress( 163 "[repo:%s] %s: %s => %s" % ( 164 brown(repo), 165 blue(_("looking for file in mirror")), 166 darkgreen(crippled_uri), 167 filename, 168 ), 169 importance = 1, 170 type = "info", 171 header = brown(" @@ ") 172 ) 173 174 ftp = FtpInterface(uri, self.Entropy, verbose = False) 175 try: 176 177 branches_path = self.Entropy.get_remote_database_relative_path( 178 repo) 179 try: 180 ftp.set_cwd(branches_path) 181 except FtpError: 182 continue # no branches in this mirror 183 184 branches = ftp.list_dir() 185 for branch in branches: 186 187 # is branch excluded ? 188 if branch in excluded_branches: 189 continue 190 191 if branch_data.get(branch) != None: 192 # already read 193 continue 194 195 mypath = os.path.join("/", branches_path, branch) 196 try: 197 ftp.set_cwd(mypath) 198 except FtpError: 199 # not a dir 200 continue 201 202 if not ftp.is_file_available(filename): 203 # nothing to do, no file here 204 continue 205 206 tmp_dir = tempfile.mkdtemp() 207 tries = 4 208 success = False 209 while tries: 210 downloaded = ftp.download_file(filename, tmp_dir) 211 if not downloaded: 212 tries -= 1 213 continue # argh! 214 success = True 215 break 216 217 down_path = os.path.join(tmp_dir, filename) 218 if success and os.access(down_path, os.F_OK): 219 down_f = open(down_path) 220 branch_data[branch] = down_f.read() 221 down_f.close() 222 shutil.rmtree(tmp_dir) 223 224 finally: 225 ftp.close() 226 227 return branch_data

228

229 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

230 231 if repo == None: 232 repo = self.Entropy.default_repository 233 234 if not mirrors: 235 mirrors = self.Entropy.get_remote_mirrors(repo) 236 237 issues = False 238 for uri in mirrors: 239 240 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 241 242 lock_text = _("unlocking") 243 if lock: 244 lock_text = _("locking") 245 self.Entropy.updateProgress( 246 "[repo:%s|%s] %s %s" % ( 247 brown(repo), 248 darkgreen(crippled_uri), 249 bold(lock_text), 250 blue("%s...") % (_("mirror"),), 251 ), 252 importance = 1, 253 type = "info", 254 header = brown(" * "), 255 back = True 256 ) 257 258 try: 259 ftp = FtpInterface(uri, self.Entropy) 260 except ConnectionError: 261 self.entropyTools.print_traceback() 262 return True # issues 263 my_path = os.path.join( 264 self.Entropy.get_remote_database_relative_path(repo), 265 self.SystemSettings['repositories']['branch']) 266 ftp.set_cwd(my_path, dodir = True) 267 268 lock_file = etpConst['etpdatabaselockfile'] 269 if lock and ftp.is_file_available(lock_file): 270 self.Entropy.updateProgress( 271 "[repo:%s|%s] %s" % ( 272 brown(repo), 273 darkgreen(crippled_uri), 274 blue(_("mirror already locked")), 275 ), 276 importance = 1, 277 type = "info", 278 header = darkgreen(" * ") 279 ) 280 ftp.close() 281 continue 282 elif not lock and not ftp.is_file_available(lock_file): 283 self.Entropy.updateProgress( 284 "[repo:%s|%s] %s" % ( 285 brown(repo), 286 darkgreen(crippled_uri), 287 blue(_("mirror already unlocked")), 288 ), 289 importance = 1, 290 type = "info", 291 header = darkgreen(" * ") 292 ) 293 ftp.close() 294 continue 295 296 if lock: 297 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 298 else: 299 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 300 ftp.close() 301 if not rc_lock: 302 issues = True 303 304 if not issues: 305 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 306 if os.path.isfile(db_taint_file): 307 os.remove(db_taint_file) 308 309 return issues

310 311

312 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 313 repo = None):

314 # this functions makes entropy clients to not download anything 315 # from the chosen mirrors. it is used to avoid clients to 316 # download databases while we're uploading a new one 317 318 if mirrors == None: 319 mirrors = [] 320 321 if repo == None: 322 repo = self.Entropy.default_repository 323 324 if not mirrors: 325 mirrors = self.Entropy.get_remote_mirrors(repo) 326 327 issues = False 328 for uri in mirrors: 329 330 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 331 332 lock_text = _("unlocking") 333 if lock: 334 lock_text = _("locking") 335 self.Entropy.updateProgress( 336 "[repo:%s|%s] %s %s..." % ( 337 blue(repo), 338 red(crippled_uri), 339 bold(lock_text), 340 blue(_("mirror for download")), 341 ), 342 importance = 1, 343 type = "info", 344 header = red(" @@ "), 345 back = True 346 ) 347 348 try: 349 ftp = FtpInterface(uri, self.Entropy) 350 except ConnectionError: 351 self.entropyTools.print_traceback() 352 return True # issues 353 my_path = os.path.join( 354 self.Entropy.get_remote_database_relative_path(repo), 355 self.SystemSettings['repositories']['branch']) 356 ftp.set_cwd(my_path, dodir = True) 357 358 lock_file = etpConst['etpdatabasedownloadlockfile'] 359 360 if lock and ftp.is_file_available(lock_file): 361 self.Entropy.updateProgress( 362 "[repo:%s|%s] %s" % ( 363 blue(repo), 364 red(crippled_uri), 365 blue(_("mirror already locked for download")), 366 ), 367 importance = 1, 368 type = "info", 369 header = red(" @@ ") 370 ) 371 ftp.close() 372 continue 373 374 elif not lock and not ftp.is_file_available(lock_file): 375 self.Entropy.updateProgress( 376 "[repo:%s|%s] %s" % ( 377 blue(repo), 378 red(crippled_uri), 379 blue(_("mirror already unlocked for download")), 380 ), 381 importance = 1, 382 type = "info", 383 header = red(" @@ ") 384 ) 385 ftp.close() 386 continue 387 388 if lock: 389 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 390 repo = repo) 391 else: 392 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 393 repo = repo) 394 ftp.close() 395 if not rc_lock: 396 issues = True 397 398 return issues

399

400 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 401 repo = None):

402 403 if repo == None: 404 repo = self.Entropy.default_repository 405 406 my_path = os.path.join( 407 self.Entropy.get_remote_database_relative_path(repo), 408 self.SystemSettings['repositories']['branch']) 409 if not ftp_connection: 410 try: 411 ftp_connection = FtpInterface(uri, self.Entropy) 412 except ConnectionError: 413 self.entropyTools.print_traceback() 414 return False # issues 415 ftp_connection.set_cwd(my_path, dodir = True) 416 else: 417 mycwd = ftp_connection.get_cwd() 418 if mycwd != my_path: 419 ftp_connection.set_basedir() 420 ftp_connection.set_cwd(my_path, dodir = True) 421 422 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 423 lock_string = u'' 424 425 if dblock: 426 self.create_local_database_lockfile(repo) 427 lock_file = self.get_database_lockfile(repo) 428 else: 429 # locking/unlocking mirror1 for download 430 lock_string = _('for download') 431 self.create_local_database_download_lockfile(repo) 432 lock_file = self.get_database_download_lockfile(repo) 433 434 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 435 if rc_upload: 436 self.Entropy.updateProgress( 437 "[repo:%s|%s] %s %s" % ( 438 blue(repo), 439 red(crippled_uri), 440 blue(_("mirror successfully locked")), 441 blue(lock_string), 442 ), 443 importance = 1, 444 type = "info", 445 header = red(" @@ ") 446 ) 447 else: 448 self.Entropy.updateProgress( 449 "[repo:%s|%s] %s: %s - %s %s" % ( 450 blue(repo), 451 red(crippled_uri), 452 blue("lock error"), 453 rc_upload, 454 blue(_("mirror not locked")), 455 blue(lock_string), 456 ), 457 importance = 1, 458 type = "error", 459 header = darkred(" * ") 460 ) 461 self.remove_local_database_lockfile(repo) 462 463 return rc_upload

464 465

466 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

467 468 if repo == None: 469 repo = self.Entropy.default_repository 470 471 my_path = os.path.join( 472 self.Entropy.get_remote_database_relative_path(repo), 473 self.SystemSettings['repositories']['branch']) 474 if not ftp_connection: 475 try: 476 ftp_connection = FtpInterface(uri, self.Entropy) 477 except ConnectionError: 478 self.entropyTools.print_traceback() 479 return False # issues 480 ftp_connection.set_cwd(my_path) 481 else: 482 mycwd = ftp_connection.get_cwd() 483 if mycwd != my_path: 484 ftp_connection.set_basedir() 485 ftp_connection.set_cwd(my_path) 486 487 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 488 489 if dblock: 490 dbfile = etpConst['etpdatabaselockfile'] 491 else: 492 dbfile = etpConst['etpdatabasedownloadlockfile'] 493 rc_delete = ftp_connection.delete_file(dbfile) 494 if rc_delete: 495 self.Entropy.updateProgress( 496 "[repo:%s|%s] %s" % ( 497 blue(repo), 498 red(crippled_uri), 499 blue(_("mirror successfully unlocked")), 500 ), 501 importance = 1, 502 type = "info", 503 header = darkgreen(" * ") 504 ) 505 if dblock: 506 self.remove_local_database_lockfile(repo) 507 else: 508 self.remove_local_database_download_lockfile(repo) 509 else: 510 self.Entropy.updateProgress( 511 "[repo:%s|%s] %s: %s - %s" % ( 512 blue(repo), 513 red(crippled_uri), 514 blue(_("unlock error")), 515 rc_delete, 516 blue(_("mirror not unlocked")), 517 ), 518 importance = 1, 519 type = "error", 520 header = darkred(" * ") 521 ) 522 523 return rc_delete

524

525 - def get_database_lockfile(self, repo = None):

526 if repo == None: 527 repo = self.Entropy.default_repository 528 return os.path.join(self.Entropy.get_local_database_dir(repo), 529 etpConst['etpdatabaselockfile'])

530

531 - def get_database_download_lockfile(self, repo = None):

532 if repo == None: 533 repo = self.Entropy.default_repository 534 return os.path.join(self.Entropy.get_local_database_dir(repo), 535 etpConst['etpdatabasedownloadlockfile'])

536

537 - def create_local_database_download_lockfile(self, repo = None):

538 if repo == None: 539 repo = self.Entropy.default_repository 540 lock_file = self.get_database_download_lockfile(repo) 541 f_lock = open(lock_file, "w") 542 f_lock.write("download locked") 543 f_lock.flush() 544 f_lock.close()

545

546 - def create_local_database_lockfile(self, repo = None):

547 if repo == None: 548 repo = self.Entropy.default_repository 549 lock_file = self.get_database_lockfile(repo) 550 f_lock = open(lock_file, "w") 551 f_lock.write("database locked") 552 f_lock.flush() 553 f_lock.close()

554

555 - def remove_local_database_lockfile(self, repo = None):

556 if repo == None: 557 repo = self.Entropy.default_repository 558 lock_file = self.get_database_lockfile(repo) 559 if os.path.isfile(lock_file): 560 os.remove(lock_file)

561

562 - def remove_local_database_download_lockfile(self, repo = None):

563 if repo == None: 564 repo = self.Entropy.default_repository 565 lock_file = self.get_database_download_lockfile(repo) 566 if os.path.isfile(lock_file): 567 os.remove(lock_file)

568

569 - def download_package(self, uri, pkg_relative_path, repo = None):

570 571 if repo == None: 572 repo = self.Entropy.default_repository 573 574 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 575 576 tries = 0 577 while tries < 5: 578 tries += 1 579 580 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 581 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 582 pkgfile = os.path.basename(pkg_relative_path) 583 584 self.Entropy.updateProgress( 585 "[repo:%s|%s|#%s] %s: %s" % ( 586 brown(repo), 587 darkgreen(crippled_uri), 588 brown(tries), 589 blue(_("connecting to download package")), 590 darkgreen(pkg_to_join_path), 591 ), 592 importance = 1, 593 type = "info", 594 header = darkgreen(" * "), 595 back = True 596 ) 597 598 try: 599 ftp = FtpInterface(uri, self.Entropy) 600 except ConnectionError: 601 self.entropyTools.print_traceback() 602 return False # issues 603 dirpath = os.path.join( 604 self.Entropy.get_remote_packages_relative_path(repo), 605 pkg_to_join_dirpath) 606 ftp.set_cwd(dirpath, dodir = True) 607 608 self.Entropy.updateProgress( 609 "[repo:%s|%s|#%s] %s: %s" % ( 610 brown(repo), 611 darkgreen(crippled_uri), 612 brown(tries), 613 blue(_("downloading package")), 614 darkgreen(pkg_to_join_path), 615 ), 616 importance = 1, 617 type = "info", 618 header = darkgreen(" * ") 619 ) 620 621 download_path = os.path.join( 622 self.Entropy.get_local_packages_directory(repo), 623 pkg_to_join_dirpath) 624 if (not os.path.isdir(download_path)) and \ 625 (not os.access(download_path, os.R_OK)): 626 os.makedirs(download_path) 627 rc_download = ftp.download_file(pkgfile, download_path) 628 if not rc_download: 629 self.Entropy.updateProgress( 630 "[repo:%s|%s|#%s] %s: %s %s" % ( 631 brown(repo), 632 darkgreen(crippled_uri), 633 brown(tries), 634 blue(_("package")), 635 darkgreen(pkg_to_join_path), 636 blue(_("does not exist")), 637 ), 638 importance = 1, 639 type = "error", 640 header = darkred(" !!! ") 641 ) 642 ftp.close() 643 return rc_download 644 645 dbconn = self.Entropy.open_server_repository(read_only = True, 646 no_upload = True, repo = repo) 647 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 648 if idpackage == -1: 649 self.Entropy.updateProgress( 650 "[repo:%s|%s|#%s] %s: %s %s" % ( 651 brown(repo), 652 darkgreen(crippled_uri), 653 brown(tries), 654 blue(_("package")), 655 darkgreen(pkgfile), 656 blue(_("is not listed in the repository !")), 657 ), 658 importance = 1, 659 type = "error", 660 header = darkred(" !!! ") 661 ) 662 ftp.close() 663 return False 664 665 storedmd5 = dbconn.retrieveDigest(idpackage) 666 self.Entropy.updateProgress( 667 "[repo:%s|%s|#%s] %s: %s" % ( 668 brown(repo), 669 darkgreen(crippled_uri), 670 brown(tries), 671 blue(_("verifying checksum of package")), 672 darkgreen(pkgfile), 673 ), 674 importance = 1, 675 type = "info", 676 header = darkgreen(" * "), 677 back = True 678 ) 679 680 pkg_path = os.path.join(download_path, pkgfile) 681 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 682 if md5check: 683 self.Entropy.updateProgress( 684 "[repo:%s|%s|#%s] %s: %s %s" % ( 685 brown(repo), 686 darkgreen(crippled_uri), 687 brown(tries), 688 blue(_("package")), 689 darkgreen(pkgfile), 690 blue(_("downloaded successfully")), 691 ), 692 importance = 1, 693 type = "info", 694 header = darkgreen(" * ") 695 ) 696 return True 697 else: 698 self.Entropy.updateProgress( 699 "[repo:%s|%s|#%s] %s: %s %s" % ( 700 brown(repo), 701 darkgreen(crippled_uri), 702 brown(tries), 703 blue(_("package")), 704 darkgreen(pkgfile), 705 blue(_("checksum does not match. re-downloading...")), 706 ), 707 importance = 1, 708 type = "warning", 709 header = darkred(" * ") 710 ) 711 if os.path.isfile(pkg_path): 712 os.remove(pkg_path) 713 714 continue 715 716 # if we get here it means the files hasn't been downloaded properly 717 self.Entropy.updateProgress( 718 "[repo:%s|%s|#%s] %s: %s %s" % ( 719 brown(repo), 720 darkgreen(crippled_uri), 721 brown(tries), 722 blue(_("package")), 723 darkgreen(pkgfile), 724 blue(_("seems broken. Consider to re-package it. Giving up!")), 725 ), 726 importance = 1, 727 type = "error", 728 header = darkred(" !!! ") 729 ) 730 return False

731 732

733 - def get_remote_databases_status(self, repo = None, mirrors = None):

734 735 if repo == None: 736 repo = self.Entropy.default_repository 737 738 if not mirrors: 739 mirrors = self.Entropy.get_remote_mirrors(repo) 740 741 data = [] 742 for uri in mirrors: 743 744 # let it raise an exception if connection is impossible 745 ftp = FtpInterface(uri, self.Entropy) 746 try: 747 my_path = os.path.join( 748 self.Entropy.get_remote_database_relative_path(repo), 749 self.SystemSettings['repositories']['branch']) 750 ftp.set_cwd(my_path, dodir = True) 751 except ftp.ftplib.error_perm: 752 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 753 self.Entropy.updateProgress( 754 "[repo:%s|%s] %s !" % ( 755 brown(repo), 756 darkgreen(crippled_uri), 757 blue(_("mirror has invalid directory structure")), 758 ), 759 importance = 1, 760 type = "warning", 761 header = darkred(" !!! ") 762 ) 763 ftp.close() 764 continue 765 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 766 db_format = sys_set['database_file_format'] 767 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 768 if cmethod == None: 769 raise InvalidDataType("InvalidDataType: %s." % ( 770 _("Wrong database compression method passed"), 771 ) 772 ) 773 compressedfile = etpConst[cmethod[2]] 774 775 revision = 0 776 rc1 = ftp.is_file_available(compressedfile) 777 revfilename = os.path.basename( 778 self.Entropy.get_local_database_revision_file(repo)) 779 rc2 = ftp.is_file_available(revfilename) 780 if rc1 and rc2: 781 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 782 revfilename) 783 dlcount = 5 784 while dlcount: 785 dled = ftp.download_file(revfilename, 786 etpConst['packagestmpdir'], True) 787 if dled: 788 break 789 dlcount -= 1 790 791 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 792 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 793 f_rev = open(revision_localtmppath,"r") 794 try: 795 revision = int(f_rev.readline().strip()) 796 except ValueError: 797 mytxt = _("mirror hasn't valid database revision file") 798 self.Entropy.updateProgress( 799 "[repo:%s|%s] %s: %s" % ( 800 brown(repo), 801 darkgreen(crippled_uri), 802 blue(mytxt), 803 bold(revision), 804 ), 805 importance = 1, 806 type = "error", 807 header = darkred(" !!! ") 808 ) 809 revision = 0 810 f_rev.close() 811 elif dlcount == 0: 812 self.Entropy.updateProgress( 813 "[repo:%s|%s] %s: %s" % ( 814 brown(repo), 815 darkgreen(crippled_uri), 816 blue(_("unable to download repository revision")), 817 bold(revision), 818 ), 819 importance = 1, 820 type = "error", 821 header = darkred(" !!! ") 822 ) 823 revision = 0 824 else: 825 self.Entropy.updateProgress( 826 "[repo:%s|%s] %s: %s" % ( 827 brown(repo), 828 darkgreen(crippled_uri), 829 blue(_("mirror doesn't have valid revision file")), 830 bold(revision), 831 ), 832 importance = 1, 833 type = "error", 834 header = darkred(" !!! ") 835 ) 836 revision = 0 837 if os.path.isfile(revision_localtmppath): 838 os.remove(revision_localtmppath) 839 840 info = [uri, revision] 841 data.append(info) 842 ftp.close() 843 844 return data

845

846 - def is_local_database_locked(self, repo = None):

847 local_repo = repo 848 if local_repo == None: 849 local_repo = self.Entropy.default_repository 850 lock_file = self.get_database_lockfile(local_repo) 851 return os.path.isfile(lock_file)

852

853 - def get_mirrors_lock(self, repo = None):

854 855 dbstatus = [] 856 for uri in self.Entropy.get_remote_mirrors(repo): 857 data = [uri, False, False] 858 ftp = FtpInterface(uri, self.Entropy) 859 try: 860 my_path = os.path.join( 861 self.Entropy.get_remote_database_relative_path(repo), 862 self.SystemSettings['repositories']['branch']) 863 ftp.set_cwd(my_path) 864 except ftp.ftplib.error_perm: 865 ftp.close() 866 continue 867 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 868 # upload locked 869 data[1] = True 870 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 871 # download locked 872 data[2] = True 873 ftp.close() 874 dbstatus.append(data) 875 return dbstatus

876

877 - def download_notice_board(self, repo = None):

878 879 if repo == None: 880 repo = self.Entropy.default_repository 881 mirrors = self.Entropy.get_remote_mirrors(repo) 882 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 883 mytmpdir = self.entropyTools.get_random_temp_file() 884 os.makedirs(mytmpdir) 885 886 self.Entropy.updateProgress( 887 "[repo:%s] %s %s" % ( 888 brown(repo), 889 blue(_("downloading notice board from mirrors to")), 890 red(rss_path), 891 ), 892 importance = 1, 893 type = "info", 894 header = blue(" @@ ") 895 ) 896 897 downloaded = False 898 for uri in mirrors: 899 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 900 downloader = self.FtpServerHandler( 901 FtpInterface, self.Entropy, [uri], 902 [rss_path], download = True, 903 local_basedir = mytmpdir, critical_files = [rss_path], 904 repo = repo 905 ) 906 errors, m_fine_uris, m_broken_uris = downloader.go() 907 if not errors: 908 self.Entropy.updateProgress( 909 "[repo:%s] %s: %s" % ( 910 brown(repo), 911 blue(_("notice board downloaded successfully from")), 912 red(crippled_uri), 913 ), 914 importance = 1, 915 type = "info", 916 header = blue(" @@ ") 917 ) 918 downloaded = True 919 break 920 921 if downloaded: 922 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 923 rss_path) 924 925 return downloaded

926

927 - def upload_notice_board(self, repo = None):

928 929 if repo == None: 930 repo = self.Entropy.default_repository 931 mirrors = self.Entropy.get_remote_mirrors(repo) 932 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 933 934 self.Entropy.updateProgress( 935 "[repo:%s] %s %s" % ( 936 brown(repo), 937 blue(_("uploading notice board from")), 938 red(rss_path), 939 ), 940 importance = 1, 941 type = "info", 942 header = blue(" @@ ") 943 ) 944 945 uploader = self.FtpServerHandler( 946 FtpInterface, 947 self.Entropy, 948 mirrors, 949 [rss_path], 950 critical_files = [rss_path], 951 repo = repo 952 ) 953 errors, m_fine_uris, m_broken_uris = uploader.go() 954 if errors: 955 m_broken_uris = sorted(m_broken_uris) 956 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 957 for x in m_broken_uris] 958 self.Entropy.updateProgress( 959 "[repo:%s] %s %s" % ( 960 brown(repo), 961 blue(_("notice board upload failed on")), 962 red(', '.join(m_broken_uris)), 963 ), 964 importance = 1, 965 type = "info", 966 header = blue(" @@ ") 967 ) 968 return False 969 self.Entropy.updateProgress( 970 "[repo:%s] %s" % ( 971 brown(repo), 972 blue(_("notice board upload success")), 973 ), 974 importance = 1, 975 type = "info", 976 header = blue(" @@ ") 977 ) 978 return True

979 980

981 - def update_notice_board(self, title, notice_text, link = None, repo = None):

982 983 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 984 rss_description = "Inform about important distribution activities." 985 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 986 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 987 if not link: 988 link = srv_set['rss']['website_url'] 989 990 self.download_notice_board(repo) 991 rss_main = RSS(rss_path, rss_title, rss_description, 992 maxentries = 20) 993 rss_main.add_item(title, link, description = notice_text) 994 rss_main.write_changes() 995 status = self.upload_notice_board(repo) 996 return status

997

998 - def read_notice_board(self, do_download = True, repo = None):

999 1000 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1001 if do_download: 1002 self.download_notice_board(repo) 1003 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1004 return None 1005 rss_main = RSS(rss_path, '', '') 1006 return rss_main.get_entries()

1007

1008 - def remove_from_notice_board(self, identifier, repo = None):

1009 1010 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1011 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 1012 rss_description = "Inform about important distribution activities." 1013 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1014 return 0 1015 rss_main = RSS(rss_path, rss_title, rss_description) 1016 data = rss_main.remove_entry(identifier) 1017 rss_main.write_changes() 1018 return data

1019

1020 - def update_rss_feed(self, repo = None):

1021 1022 product = self.SystemSettings['repositories']['product'] 1023 #db_dir = self.Entropy.get_local_database_dir(repo) 1024 rss_path = self.Entropy.get_local_database_rss_file(repo) 1025 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 1026 rss_dump_name = etpConst['rss-dump-name'] 1027 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 1028 1029 rss_title = u"%s Online Repository Status" % ( 1030 self.SystemSettings['system']['name'],) 1031 rss_description = \ 1032 u"Keep you updated on what's going on in the %s Repository." % ( 1033 self.SystemSettings['system']['name'],) 1034 1035 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1036 1037 rss_main = RSS(rss_path, rss_title, rss_description, 1038 maxentries = srv_set['rss']['max_entries']) 1039 # load dump 1040 db_actions = self.Cacher.pop(rss_dump_name) 1041 if db_actions: 1042 try: 1043 f_rev = open(db_revision_path) 1044 revision = f_rev.readline().strip() 1045 f_rev.close() 1046 except (IOError, OSError): 1047 revision = u"N/A" 1048 commitmessage = u'' 1049 if self.Entropy.rssMessages['commitmessage']: 1050 commitmessage = u' :: ' + \ 1051 self.Entropy.rssMessages['commitmessage'] 1052 1053 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 1054 product[0].upper() + product[1:] + " " + \ 1055 self.SystemSettings['repositories']['branch'] + \ 1056 " :: Revision: " + revision + commitmessage 1057 1058 link = srv_set['rss']['base_url'] 1059 # create description 1060 added_items = db_actions.get("added") 1061 1062 if added_items: 1063 for atom in sorted(added_items): 1064 mylink = link + "?search=" + atom.split("~")[0] + \ 1065 "&arch=" + etpConst['currentarch'] + "&product="+product 1066 description = atom + ": " + added_items[atom]['description'] 1067 rss_main.add_item(title = "Added/Updated" + title, 1068 link = mylink, description = description) 1069 removed_items = db_actions.get("removed") 1070 1071 if removed_items: 1072 for atom in sorted(removed_items): 1073 description = atom + ": " + \ 1074 removed_items[atom]['description'] 1075 rss_main.add_item(title = "Removed" + title, link = link, 1076 description = description) 1077 1078 light_items = db_actions.get('light') 1079 if light_items: 1080 rss_light = RSS(rss_light_path, rss_title, rss_description, 1081 maxentries = srv_set['rss']['light_max_entries']) 1082 for atom in sorted(light_items): 1083 mylink = link + "?search=" + atom.split("~")[0] + \ 1084 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1085 product 1086 description = light_items[atom]['description'] 1087 rss_light.add_item(title = "[" + revision + "] " + atom, 1088 link = mylink, description = description) 1089 rss_light.write_changes() 1090 1091 rss_main.write_changes() 1092 self.Entropy.rssMessages.clear() 1093 self.dumpTools.removeobj(rss_dump_name)

1094 1095

1096 - def dump_database_to_file(self, db_path, destination_path, opener, 1097 exclude_tables = None, repo = None):

1098 1099 if not exclude_tables: 1100 exclude_tables = [] 1101 1102 f_out = opener(destination_path, "wb") 1103 dbconn = self.Entropy.open_server_repository(db_path, 1104 just_reading = True, repo = repo, do_treeupdates = False) 1105 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1106 self.Entropy.close_server_database(dbconn) 1107 f_out.close()

1108

1109 - def create_file_checksum(self, file_path, checksum_path):

1110 mydigest = self.entropyTools.md5sum(file_path) 1111 f_ck = open(checksum_path, "w") 1112 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1113 f_ck.write(mystring) 1114 f_ck.flush() 1115 f_ck.close()

1116

1117 - def compress_file(self, file_path, destination_path, opener):

1118 f_out = opener(destination_path, "wb") 1119 f_in = open(file_path,"rb") 1120 data = f_in.read(8192) 1121 while data: 1122 f_out.write(data) 1123 data = f_in.read(8192) 1124 f_in.close() 1125 if hasattr(f_out,'flush'): 1126 f_out.flush() 1127 f_out.close()

1128

1129 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1130 disabled_eapis = None):

1131 1132 if disabled_eapis == None: 1133 disabled_eapis = [] 1134 1135 critical = [] 1136 extra_text_files = [] 1137 data = {} 1138 data['database_revision_file'] = \ 1139 self.Entropy.get_local_database_revision_file(repo) 1140 extra_text_files.append(data['database_revision_file']) 1141 critical.append(data['database_revision_file']) 1142 1143 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1144 if os.path.isfile(database_ts_file) or download: 1145 data['database_timestamp_file'] = database_ts_file 1146 if not download: 1147 critical.append(database_ts_file) 1148 1149 database_package_mask_file = \ 1150 self.Entropy.get_local_database_mask_file(repo) 1151 extra_text_files.append(database_package_mask_file) 1152 if os.path.isfile(database_package_mask_file) or download: 1153 data['database_package_mask_file'] = database_package_mask_file 1154 if not download: 1155 critical.append(data['database_package_mask_file']) 1156 1157 database_package_system_mask_file = \ 1158 self.Entropy.get_local_database_system_mask_file(repo) 1159 extra_text_files.append(database_package_system_mask_file) 1160 if os.path.isfile(database_package_system_mask_file) or download: 1161 data['database_package_system_mask_file'] = \ 1162 database_package_system_mask_file 1163 if not download: 1164 critical.append(data['database_package_system_mask_file']) 1165 1166 database_package_confl_tagged_file = \ 1167 self.Entropy.get_local_database_confl_tagged_file(repo) 1168 extra_text_files.append(database_package_confl_tagged_file) 1169 if os.path.isfile(database_package_confl_tagged_file) or download: 1170 data['database_package_confl_tagged_file'] = \ 1171 database_package_confl_tagged_file 1172 if not download: 1173 critical.append(data['database_package_confl_tagged_file']) 1174 1175 database_license_whitelist_file = \ 1176 self.Entropy.get_local_database_licensewhitelist_file(repo) 1177 extra_text_files.append(database_license_whitelist_file) 1178 if os.path.isfile(database_license_whitelist_file) or download: 1179 data['database_license_whitelist_file'] = \ 1180 database_license_whitelist_file 1181 if not download: 1182 critical.append(data['database_license_whitelist_file']) 1183 1184 exp_based_pkgs_removal_file = \ 1185 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1186 extra_text_files.append(exp_based_pkgs_removal_file) 1187 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1188 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1189 if not download: 1190 critical.append(data['exp_based_pkgs_removal_file']) 1191 1192 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1193 if os.path.isfile(database_rss_file) or download: 1194 data['database_rss_file'] = database_rss_file 1195 if not download: 1196 critical.append(data['database_rss_file']) 1197 database_rss_light_file = \ 1198 self.Entropy.get_local_database_rsslight_file(repo) 1199 1200 extra_text_files.append(database_rss_light_file) 1201 if os.path.isfile(database_rss_light_file) or download: 1202 data['database_rss_light_file'] = database_rss_light_file 1203 if not download: 1204 critical.append(data['database_rss_light_file']) 1205 1206 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1207 data['pkglist_file'] = pkglist_file 1208 if not download: 1209 critical.append(data['pkglist_file']) 1210 1211 # EAPI 2,3 1212 if not download: # we don't need to get the dump 1213 if 3 not in disabled_eapis: 1214 1215 data['metafiles_path'] = \ 1216 self.Entropy.get_local_database_compressed_metafiles_file( 1217 repo) 1218 critical.append(data['metafiles_path']) 1219 if 2 not in disabled_eapis: 1220 1221 data['dump_path'] = os.path.join( 1222 self.Entropy.get_local_database_dir(repo), 1223 etpConst[cmethod[3]]) 1224 critical.append(data['dump_path']) 1225 1226 data['dump_path_digest'] = os.path.join( 1227 self.Entropy.get_local_database_dir(repo), 1228 etpConst[cmethod[4]]) 1229 critical.append(data['dump_path_digest']) 1230 1231 data['dump_path_light'] = os.path.join( 1232 self.Entropy.get_local_database_dir(repo), 1233 etpConst[cmethod[5]]) 1234 critical.append(data['dump_path_light']) 1235 1236 data['dump_path_digest_light'] = os.path.join( 1237 self.Entropy.get_local_database_dir(repo), 1238 etpConst[cmethod[6]]) 1239 critical.append(data['dump_path_digest_light']) 1240 1241 data['database_path'] = \ 1242 self.Entropy.get_local_database_file(repo) 1243 critical.append(data['database_path']) 1244 1245 # EAPI 1 1246 if 1 not in disabled_eapis: 1247 1248 data['compressed_database_path'] = os.path.join( 1249 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1250 critical.append(data['compressed_database_path']) 1251 data['compressed_database_path_light'] = os.path.join( 1252 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1253 1254 data['database_path_digest'] = os.path.join( 1255 self.Entropy.get_local_database_dir(repo), 1256 etpConst['etpdatabasehashfile'] 1257 ) 1258 critical.append(data['database_path_digest']) 1259 1260 data['compressed_database_path_digest'] = os.path.join( 1261 self.Entropy.get_local_database_dir(repo), 1262 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1263 ) 1264 critical.append(data['compressed_database_path_digest']) 1265 1266 data['compressed_database_path_digest_light'] = os.path.join( 1267 self.Entropy.get_local_database_dir(repo), 1268 etpConst[cmethod[8]] 1269 ) 1270 critical.append(data['compressed_database_path_digest_light']) 1271 1272 1273 # SSL cert file, just for reference 1274 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1275 extra_text_files.append(ssl_ca_cert) 1276 if os.path.isfile(ssl_ca_cert): 1277 data['ssl_ca_cert_file'] = ssl_ca_cert 1278 if not download: 1279 critical.append(ssl_ca_cert) 1280 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1281 extra_text_files.append(ssl_server_cert) 1282 if os.path.isfile(ssl_server_cert): 1283 data['ssl_server_cert_file'] = ssl_server_cert 1284 if not download: 1285 critical.append(ssl_server_cert) 1286 1287 # Some information regarding how packages are built 1288 spm_files = [ 1289 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1290 (etpConst['spm']['global_package_keywords'], 1291 "global_package_keywords"), 1292 (etpConst['spm']['global_package_use'], "global_package_use"), 1293 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1294 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1295 ] 1296 for myfile, myname in spm_files: 1297 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1298 data[myname] = myfile 1299 extra_text_files.append(myfile) 1300 1301 make_profile = etpConst['spm']['global_make_profile'] 1302 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1303 mytmpdir = os.path.dirname(rnd_tmp_file) 1304 mytmpfile = os.path.join(mytmpdir, 1305 etpConst['spm']['global_make_profile_link_name']) 1306 extra_text_files.append(mytmpfile) 1307 1308 if os.path.islink(make_profile): 1309 mylink = os.readlink(make_profile) 1310 f_mkp = open(mytmpfile,"w") 1311 f_mkp.write(mylink) 1312 f_mkp.flush() 1313 f_mkp.close() 1314 data['global_make_profile'] = mytmpfile 1315 1316 return data, critical, extra_text_files

1317

1318 - def _show_package_sets_messages(self, repo):

1319 1320 self.Entropy.updateProgress( 1321 "[repo:%s] %s:" % ( 1322 brown(repo), 1323 blue(_("configured package sets")), 1324 ), 1325 importance = 0, 1326 type = "info", 1327 header = darkgreen(" * ") 1328 ) 1329 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1330 if not sets_data: 1331 self.Entropy.updateProgress( 1332 "%s" % (_("None configured"),), 1333 importance = 0, 1334 type = "info", 1335 header = brown(" # ") 1336 ) 1337 return 1338 for s_repo, s_name, s_sets in sets_data: 1339 self.Entropy.updateProgress( 1340 blue("%s" % (s_name,)), 1341 importance = 0, 1342 type = "info", 1343 header = brown(" # ") 1344 )

1345

1346 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1347 1348 self.Entropy.updateProgress( 1349 "[repo:%s|%s|%s:%s] %s" % ( 1350 brown(repo), 1351 darkgreen(crippled_uri), 1352 red("EAPI"), 1353 bold("3"), 1354 blue(_("preparing uncompressed database for the upload")), 1355 ), 1356 importance = 0, 1357 type = "info", 1358 header = darkgreen(" * ") 1359 ) 1360 self.Entropy.updateProgress( 1361 "%s: %s" % (_("database path"), blue(database_path),), 1362 importance = 0, 1363 type = "info", 1364 header = brown(" # ") 1365 )

1366

1367 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1368 upload_data, cmethod, repo):

1369 1370 if repo == None: 1371 repo = self.Entropy.default_repository 1372 1373 self.Entropy.updateProgress( 1374 "[repo:%s|%s|%s:%s] %s" % ( 1375 brown(repo), 1376 darkgreen(crippled_uri), 1377 red("EAPI"), 1378 bold("2"), 1379 blue(_("creating compressed database dump + checksum")), 1380 ), 1381 importance = 0, 1382 type = "info", 1383 header = darkgreen(" * ") 1384 ) 1385 self.Entropy.updateProgress( 1386 "%s: %s" % (_("database path"), blue(database_path),), 1387 importance = 0, 1388 type = "info", 1389 header = brown(" # ") 1390 ) 1391 self.Entropy.updateProgress( 1392 "%s: %s" % (_("dump"), blue(upload_data['dump_path']),), 1393 importance = 0, 1394 type = "info", 1395 header = brown(" # ") 1396 ) 1397 self.Entropy.updateProgress( 1398 "%s: %s" % ( 1399 _("dump checksum"), 1400 blue(upload_data['dump_path_digest']), 1401 ), 1402 importance = 0, 1403 type = "info", 1404 header = brown(" # ") 1405 ) 1406 self.Entropy.updateProgress( 1407 "%s: %s" % ( 1408 _("dump light"), 1409 blue(upload_data['dump_path_light']), 1410 ), 1411 importance = 0, 1412 type = "info", 1413 header = brown(" # ") 1414 ) 1415 self.Entropy.updateProgress( 1416 "%s: %s" % ( 1417 _("dump light checksum"), 1418 blue(upload_data['dump_path_digest_light']), 1419 ), 1420 importance = 0, 1421 type = "info", 1422 header = brown(" # ") 1423 ) 1424 1425 self.Entropy.updateProgress( 1426 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1427 importance = 0, 1428 type = "info", 1429 header = brown(" # ") 1430 )

1431

1432 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1433 upload_data, cmethod, repo):

1434 1435 self.Entropy.updateProgress( 1436 "[repo:%s|%s|%s:%s] %s" % ( 1437 brown(repo), 1438 darkgreen(crippled_uri), 1439 red("EAPI"), 1440 bold("1"), 1441 blue(_("compressing database + checksum")), 1442 ), 1443 importance = 0, 1444 type = "info", 1445 header = darkgreen(" * "), 1446 back = True 1447 ) 1448 self.Entropy.updateProgress( 1449 "%s: %s" % (_("database path"), blue(database_path),), 1450 importance = 0, 1451 type = "info", 1452 header = brown(" # ") 1453 ) 1454 self.Entropy.updateProgress( 1455 "%s: %s" % ( 1456 _("compressed database path"), 1457 blue(upload_data['compressed_database_path']), 1458 ), 1459 importance = 0, 1460 type = "info", 1461 header = brown(" # ") 1462 ) 1463 self.Entropy.updateProgress( 1464 "%s: %s" % ( 1465 _("database checksum"), 1466 blue(upload_data['database_path_digest']), 1467 ), 1468 importance = 0, 1469 type = "info", 1470 header = brown(" # ") 1471 ) 1472 self.Entropy.updateProgress( 1473 "%s: %s" % ( 1474 _("compressed checksum"), 1475 blue(upload_data['compressed_database_path_digest']), 1476 ), 1477 importance = 0, 1478 type = "info", 1479 header = brown(" # ") 1480 ) 1481 self.Entropy.updateProgress( 1482 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1483 importance = 0, 1484 type = "info", 1485 header = brown(" # ") 1486 )

1487

1488 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1489 1490 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1491 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1492 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1493 file_list if x not in found_file_list] 1494 metafile_not_found_file = \ 1495 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1496 1497 f_meta = open(metafile_not_found_file,"w") 1498 f_meta.writelines(not_found_file_list) 1499 f_meta.flush() 1500 f_meta.close() 1501 found_file_list.append(metafile_not_found_file) 1502 if os.path.isfile(compressed_dest_path): 1503 os.remove(compressed_dest_path) 1504 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1505

1506 - def create_mirror_directories(self, ftp_connection, path_to_create):

1507 bdir = "" 1508 for mydir in path_to_create.split("/"): 1509 bdir += "/"+mydir 1510 if not ftp_connection.is_file_available(bdir): 1511 try: 1512 ftp_connection.mkdir(bdir) 1513 except Exception, err: 1514 error = unicode(err) 1515 if (error.find("550") == -1) and \ 1516 (error.find("File exist") == -1): 1517 1518 mytxt = "%s %s, %s: %s" % ( 1519 _("cannot create mirror directory"), 1520 bdir, 1521 _("error"), 1522 err, 1523 ) 1524 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1525 mytxt,)) 1526 raise

1527

1528 - def mirror_lock_check(self, uri, repo = None):

1529 1530 if repo == None: 1531 repo = self.Entropy.default_repository 1532 1533 gave_up = False 1534 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1535 try: 1536 ftp = FtpInterface(uri, self.Entropy) 1537 except ConnectionError: 1538 self.entropyTools.print_traceback() 1539 return True # gave up 1540 my_path = os.path.join( 1541 self.Entropy.get_remote_database_relative_path(repo), 1542 self.SystemSettings['repositories']['branch']) 1543 ftp.set_cwd(my_path, dodir = True) 1544 1545 lock_file = self.get_database_lockfile(repo) 1546 lock_filename = os.path.basename(lock_file) 1547 if not os.path.isfile(lock_file) and \ 1548 ftp.is_file_available(lock_filename): 1549 1550 self.Entropy.updateProgress( 1551 "[repo:%s|%s|%s] %s, %s" % ( 1552 brown(str(repo)), 1553 darkgreen(crippled_uri), 1554 red(_("locking")), 1555 darkblue(_("mirror already locked")), 1556 blue(_("waiting up to 2 minutes before giving up")), 1557 ), 1558 importance = 1, 1559 type = "warning", 1560 header = brown(" * "), 1561 back = True 1562 ) 1563 unlocked = False 1564 count = 0 1565 while count < 120: 1566 count += 1 1567 time.sleep(1) 1568 if not ftp.is_file_available(os.path.basename(lock_file)): 1569 self.Entropy.updateProgress( 1570 red("[repo:%s|%s|%s] %s !" % ( 1571 repo, 1572 crippled_uri, 1573 _("locking"), 1574 _("mirror unlocked"), 1575 ) 1576 ), 1577 importance = 1, 1578 type = "info", 1579 header = darkgreen(" * ") 1580 ) 1581 unlocked = True 1582 break 1583 if not unlocked: 1584 gave_up = True 1585 1586 ftp.close() 1587 return gave_up

1588

1589 - def shrink_database_and_close(self, repo = None):

1590 dbconn = self.Entropy.open_server_repository(read_only = False, 1591 no_upload = True, repo = repo, indexing = False, 1592 do_treeupdates = False) 1593 dbconn.dropAllIndexes() 1594 dbconn.vacuum() 1595 dbconn.vacuum() 1596 dbconn.commitChanges() 1597 self.Entropy.close_server_database(dbconn)

1598

1599 - def update_repository_timestamp(self, repo = None):

1600 if repo == None: 1601 repo = self.Entropy.default_repository 1602 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1603 current_ts = self.Entropy.get_current_timestamp() 1604 ts_f = open(ts_file,"w") 1605 ts_f.write(current_ts) 1606 ts_f.flush() 1607 ts_f.close()

1608

1609 - def sync_database_treeupdates(self, repo = None):

1610 1611 if repo == None: 1612 repo = self.Entropy.default_repository 1613 dbconn = self.Entropy.open_server_repository(read_only = False, 1614 no_upload = True, repo = repo, do_treeupdates = False) 1615 # grab treeupdates from other databases and inject 1616 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1617 server_repos = srv_set['repositories'].keys() 1618 all_actions = set() 1619 for myrepo in server_repos: 1620 1621 # avoid __default__ 1622 if myrepo == etpConst['clientserverrepoid']: 1623 continue 1624 1625 mydbc = self.Entropy.open_server_repository(just_reading = True, 1626 repo = myrepo) 1627 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1628 for data in actions: 1629 all_actions.add(data) 1630 if not actions: 1631 continue 1632 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1633 try: 1634 # clear first 1635 dbconn.removeTreeUpdatesActions(repo) 1636 dbconn.insertTreeUpdatesActions(all_actions, repo) 1637 except dbapi2.Error, err: 1638 self.entropyTools.print_traceback() 1639 mytxt = "%s, %s: %s. %s" % ( 1640 _("Troubles with treeupdates"), 1641 _("error"), 1642 err, 1643 _("Bumping old data back"), 1644 ) 1645 self.Entropy.updateProgress( 1646 mytxt, 1647 importance = 1, 1648 type = "warning" 1649 ) 1650 # restore previous data 1651 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1652 1653 dbconn.commitChanges() 1654 self.Entropy.close_server_database(dbconn)

1655

1656 - def upload_database(self, uris, lock_check = False, pretend = False, 1657 repo = None):

1658 1659 if repo == None: 1660 repo = self.Entropy.default_repository 1661 1662 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1663 if srv_set['rss']['enabled']: 1664 self.update_rss_feed(repo = repo) 1665 1666 upload_errors = False 1667 broken_uris = set() 1668 fine_uris = set() 1669 1670 disabled_eapis = sorted(srv_set['disabled_eapis']) 1671 1672 for uri in uris: 1673 1674 db_format = srv_set['database_file_format'] 1675 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1676 if cmethod == None: 1677 raise InvalidDataType("InvalidDataType: %s." % ( 1678 _("wrong database compression method passed"), 1679 ) 1680 ) 1681 1682 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1683 database_path = self.Entropy.get_local_database_file(repo) 1684 1685 if disabled_eapis: 1686 self.Entropy.updateProgress( 1687 "[repo:%s|%s|%s] %s: %s" % ( 1688 blue(repo), 1689 red(crippled_uri), 1690 darkgreen(_("upload")), 1691 darkred(_("disabled EAPI")), 1692 bold(', '.join(disabled_eapis)), 1693 ), 1694 importance = 1, 1695 type = "warning", 1696 header = darkgreen(" * ") 1697 ) 1698 1699 # create/update timestamp file 1700 self.update_repository_timestamp(repo) 1701 # create pkglist service file 1702 self.Entropy.create_repository_pkglist(repo) 1703 1704 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1705 repo = repo, disabled_eapis = disabled_eapis) 1706 1707 if lock_check: 1708 given_up = self.mirror_lock_check(uri, repo = repo) 1709 if given_up: 1710 upload_errors = True 1711 broken_uris.add(uri) 1712 continue 1713 1714 self.lock_mirrors_for_download(True, [uri], repo = repo) 1715 1716 self.Entropy.updateProgress( 1717 "[repo:%s|%s|%s] %s" % ( 1718 blue(repo), 1719 red(crippled_uri), 1720 darkgreen(_("upload")), 1721 darkgreen(_("preparing to upload database to mirror")), 1722 ), 1723 importance = 1, 1724 type = "info", 1725 header = darkgreen(" * ") 1726 ) 1727 1728 # Package Sets info 1729 self._show_package_sets_messages(repo) 1730 self.sync_database_treeupdates(repo) 1731 self.Entropy.update_database_package_sets(repo) 1732 self.Entropy.close_server_databases() 1733 1734 # backup current database to avoid re-indexing 1735 old_dbpath = self.Entropy.get_local_database_file(repo) 1736 backup_dbpath = old_dbpath + ".up_backup" 1737 copy_back = False 1738 if not pretend: 1739 try: 1740 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1741 os.remove(backup_dbpath) 1742 shutil.copy2(old_dbpath, backup_dbpath) 1743 copy_back = True 1744 except shutil.Error: 1745 copy_back = False 1746 1747 self.shrink_database_and_close(repo) 1748 1749 # EAPI 3 1750 if 3 not in disabled_eapis: 1751 self._create_metafiles_file(upload_data['metafiles_path'], 1752 text_files, repo) 1753 self._show_eapi3_upload_messages(crippled_uri, database_path, 1754 repo) 1755 1756 # EAPI 2 1757 if 2 not in disabled_eapis: 1758 self._show_eapi2_upload_messages(crippled_uri, database_path, 1759 upload_data, cmethod, repo) 1760 1761 # create compressed dump + checksum 1762 self.dump_database_to_file(database_path, 1763 upload_data['dump_path'], cmethod[0], repo = repo) 1764 self.create_file_checksum(upload_data['dump_path'], 1765 upload_data['dump_path_digest']) 1766 1767 self.dump_database_to_file(database_path, 1768 upload_data['dump_path_light'], cmethod[0], 1769 exclude_tables = ["content"], repo = repo) 1770 self.create_file_checksum(upload_data['dump_path_light'], 1771 upload_data['dump_path_digest_light']) 1772 1773 1774 # EAPI 1 1775 if 1 not in disabled_eapis: 1776 self._show_eapi1_upload_messages(crippled_uri, database_path, 1777 upload_data, cmethod, repo) 1778 1779 # compress the database and create uncompressed 1780 # database checksum -- DEPRECATED 1781 self.compress_file(database_path, 1782 upload_data['compressed_database_path'], cmethod[0]) 1783 self.create_file_checksum(database_path, 1784 upload_data['database_path_digest']) 1785 1786 # create compressed database checksum 1787 self.create_file_checksum( 1788 upload_data['compressed_database_path'], 1789 upload_data['compressed_database_path_digest']) 1790 1791 # create light version of the compressed db 1792 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1793 temp_eapi1_dbfile = eapi1_dbfile+".light" 1794 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1795 # open and remove content table 1796 eapi1_tmp_dbconn = \ 1797 self.Entropy.ClientService.open_generic_database( 1798 temp_eapi1_dbfile, indexing_override = False, 1799 xcache = False) 1800 eapi1_tmp_dbconn.dropContent() 1801 eapi1_tmp_dbconn.commitChanges() 1802 eapi1_tmp_dbconn.vacuum() 1803 eapi1_tmp_dbconn.closeDB() 1804 1805 # compress 1806 self.compress_file(temp_eapi1_dbfile, 1807 upload_data['compressed_database_path_light'], cmethod[0]) 1808 # go away, we don't need you anymore 1809 os.remove(temp_eapi1_dbfile) 1810 # create compressed light database checksum 1811 self.create_file_checksum( 1812 upload_data['compressed_database_path_light'], 1813 upload_data['compressed_database_path_digest_light']) 1814 1815 if not pretend: 1816 # upload 1817 uploader = self.FtpServerHandler( 1818 FtpInterface, 1819 self.Entropy, 1820 [uri], 1821 [upload_data[x] for x in upload_data], 1822 critical_files = critical, 1823 repo = repo 1824 ) 1825 errors, m_fine_uris, m_broken_uris = uploader.go() 1826 if errors: 1827 my_broken_uris = sorted([ 1828 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1829 x[1]) for x in m_broken_uris]) 1830 self.Entropy.updateProgress( 1831 "[repo:%s|%s|%s] %s" % ( 1832 repo, 1833 crippled_uri, 1834 _("errors"), 1835 _("upload failed, not unlocking and continuing"), 1836 ), 1837 importance = 0, 1838 type = "error", 1839 header = darkred(" !!! ") 1840 ) 1841 # get reason 1842 reason = my_broken_uris[0][1] 1843 self.Entropy.updateProgress( 1844 blue("%s: %s" % (_("reason"), reason,)), 1845 importance = 0, 1846 type = "error", 1847 header = blue(" # ") 1848 ) 1849 upload_errors = True 1850 broken_uris |= m_broken_uris 1851 continue 1852 1853 # copy db back 1854 if copy_back and os.path.isfile(backup_dbpath): 1855 self.Entropy.close_server_databases() 1856 further_backup_dbpath = old_dbpath+".security_backup" 1857 if os.path.isfile(further_backup_dbpath): 1858 os.remove(further_backup_dbpath) 1859 shutil.copy2(old_dbpath, further_backup_dbpath) 1860 shutil.move(backup_dbpath, old_dbpath) 1861 1862 # unlock 1863 self.lock_mirrors_for_download(False, [uri], repo = repo) 1864 fine_uris |= m_fine_uris 1865 1866 if not fine_uris: 1867 upload_errors = True 1868 return upload_errors, broken_uris, fine_uris

1869 1870

1871 - def download_database(self, uris, lock_check = False, pretend = False, 1872 repo = None):

1873 1874 if repo == None: 1875 repo = self.Entropy.default_repository 1876 1877 download_errors = False 1878 broken_uris = set() 1879 fine_uris = set() 1880 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1881 disabled_eapis = sorted(srv_set['disabled_eapis']) 1882 1883 for uri in uris: 1884 1885 db_format = srv_set['database_file_format'] 1886 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1887 if cmethod == None: 1888 raise InvalidDataType("InvalidDataType: %s." % ( 1889 _("wrong database compression method passed"), 1890 ) 1891 ) 1892 1893 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1894 database_path = self.Entropy.get_local_database_file(repo) 1895 database_dir_path = os.path.dirname( 1896 self.Entropy.get_local_database_file(repo)) 1897 download_data, critical, text_files = self.get_files_to_sync( 1898 cmethod, download = True, 1899 repo = repo, disabled_eapis = disabled_eapis) 1900 mytmpdir = self.entropyTools.get_random_temp_file() 1901 os.makedirs(mytmpdir) 1902 1903 self.Entropy.updateProgress( 1904 "[repo:%s|%s|%s] %s" % ( 1905 brown(repo), 1906 darkgreen(crippled_uri), 1907 red(_("download")), 1908 blue(_("preparing to download database from mirror")), 1909 ), 1910 importance = 1, 1911 type = "info", 1912 header = darkgreen(" * ") 1913 ) 1914 files_to_sync = sorted(download_data.keys()) 1915 for myfile in files_to_sync: 1916 self.Entropy.updateProgress( 1917 "%s: %s" % ( 1918 blue(_("download path")), 1919 brown(unicode(download_data[myfile])), 1920 ), 1921 importance = 0, 1922 type = "info", 1923 header = brown(" # ") 1924 ) 1925 1926 if lock_check: 1927 given_up = self.mirror_lock_check(uri, repo = repo) 1928 if given_up: 1929 download_errors = True 1930 broken_uris.add(uri) 1931 continue 1932 1933 # avoid having others messing while we're downloading 1934 self.lock_mirrors(True, [uri], repo = repo) 1935 1936 if not pretend: 1937 # download 1938 downloader = self.FtpServerHandler( 1939 FtpInterface, self.Entropy, [uri], 1940 [download_data[x] for x in download_data], download = True, 1941 local_basedir = mytmpdir, critical_files = critical, 1942 repo = repo 1943 ) 1944 errors, m_fine_uris, m_broken_uris = downloader.go() 1945 if errors: 1946 my_broken_uris = sorted([ 1947 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1948 x[1]) for x in m_broken_uris]) 1949 self.Entropy.updateProgress( 1950 "[repo:%s|%s|%s] %s" % ( 1951 brown(repo), 1952 darkgreen(crippled_uri), 1953 red(_("errors")), 1954 blue(_("failed to download from mirror")), 1955 ), 1956 importance = 0, 1957 type = "error", 1958 header = darkred(" !!! ") 1959 ) 1960 # get reason 1961 reason = my_broken_uris[0][1] 1962 self.Entropy.updateProgress( 1963 blue("%s: %s" % (_("reason"), reason,)), 1964 importance = 0, 1965 type = "error", 1966 header = blue(" # ") 1967 ) 1968 download_errors = True 1969 broken_uris |= m_broken_uris 1970 self.lock_mirrors(False, [uri], repo = repo) 1971 continue 1972 1973 # all fine then, we need to move data from mytmpdir 1974 # to database_dir_path 1975 1976 # EAPI 1 -- unpack database 1977 if 1 not in disabled_eapis: 1978 compressed_db_filename = os.path.basename( 1979 download_data['compressed_database_path']) 1980 uncompressed_db_filename = os.path.basename(database_path) 1981 compressed_file = os.path.join(mytmpdir, 1982 compressed_db_filename) 1983 uncompressed_file = os.path.join(mytmpdir, 1984 uncompressed_db_filename) 1985 self.entropyTools.uncompress_file(compressed_file, 1986 uncompressed_file, cmethod[0]) 1987 1988 # now move 1989 for myfile in os.listdir(mytmpdir): 1990 fromfile = os.path.join(mytmpdir, myfile) 1991 tofile = os.path.join(database_dir_path, myfile) 1992 shutil.move(fromfile, tofile) 1993 self.Entropy.ClientService.setup_default_file_perms(tofile) 1994 1995 if os.path.isdir(mytmpdir): 1996 shutil.rmtree(mytmpdir) 1997 if os.path.isdir(mytmpdir): 1998 os.rmdir(mytmpdir) 1999 2000 2001 fine_uris.add(uri) 2002 self.lock_mirrors(False, [uri], repo = repo) 2003 2004 return download_errors, fine_uris, broken_uris

2005

2006 - def calculate_database_sync_queues(self, repo = None):

2007 2008 if repo == None: 2009 repo = self.Entropy.default_repository 2010 2011 remote_status = self.get_remote_databases_status(repo) 2012 local_revision = self.Entropy.get_local_database_revision(repo) 2013 upload_queue = [] 2014 download_latest = () 2015 2016 # all mirrors are empty ? I rule 2017 if not [x for x in remote_status if x[1]]: 2018 upload_queue = remote_status[:] 2019 else: 2020 highest_remote_revision = max([x[1] for x in remote_status]) 2021 2022 if local_revision < highest_remote_revision: 2023 for remote_item in remote_status: 2024 if remote_item[1] == highest_remote_revision: 2025 download_latest = x 2026 break 2027 2028 if download_latest: 2029 upload_queue = [x for x in remote_status if \ 2030 (x[1] < highest_remote_revision)] 2031 else: 2032 upload_queue = [x for x in remote_status if \ 2033 (x[1] < local_revision)] 2034 2035 return download_latest, upload_queue

2036

2037 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 2038 repo = None):

2039 2040 if repo == None: 2041 repo = self.Entropy.default_repository 2042 2043 while 1: 2044 2045 db_locked = False 2046 if self.is_local_database_locked(repo): 2047 db_locked = True 2048 2049 lock_data = self.get_mirrors_lock(repo) 2050 mirrors_locked = [x for x in lock_data if x[1]] 2051 2052 if not mirrors_locked and db_locked: 2053 # mirrors not locked remotely but only locally 2054 mylock_file = self.get_database_lockfile(repo) 2055 if os.access(mylock_file, os.W_OK | os.F_OK): 2056 os.remove(mylock_file) 2057 continue 2058 2059 break 2060 2061 if mirrors_locked and not db_locked: 2062 mytxt = "%s, %s %s" % ( 2063 _("Mirrors are locked, someone is working on the repository"), 2064 _("try again later"), 2065 "...", 2066 ) 2067 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 2068 2069 download_latest, upload_queue = self.calculate_database_sync_queues( 2070 repo) 2071 2072 if not download_latest and not upload_queue: 2073 self.Entropy.updateProgress( 2074 "[repo:%s|%s] %s" % ( 2075 brown(repo), 2076 red(_("sync")), # something short please 2077 blue(_("database already in sync")), 2078 ), 2079 importance = 1, 2080 type = "info", 2081 header = blue(" @@ ") 2082 ) 2083 return 0, set(), set() 2084 2085 if download_latest: 2086 download_uri = download_latest[0] 2087 download_errors, fine_uris, broken_uris = self.download_database( 2088 [download_uri], repo = repo) 2089 if download_errors: 2090 self.Entropy.updateProgress( 2091 "[repo:%s|%s] %s: %s" % ( 2092 brown(repo), 2093 red(_("sync")), 2094 blue(_("database sync failed")), 2095 red(_("download issues")), 2096 ), 2097 importance = 1, 2098 type = "error", 2099 header = darkred(" !!! ") 2100 ) 2101 return 1, fine_uris, broken_uris 2102 2103 if upload_queue and not no_upload: 2104 2105 # XXX QA checks, 2106 # please group them into entropy.qa 2107 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2108 if deps_not_found and not self.Entropy.community_repo: 2109 self.Entropy.updateProgress( 2110 "[repo:%s|%s] %s: %s" % ( 2111 brown(repo), 2112 red(_("sync")), 2113 blue(_("database sync forbidden")), 2114 red(_("dependencies_test() reported errors")), 2115 ), 2116 importance = 1, 2117 type = "error", 2118 header = darkred(" !!! ") 2119 ) 2120 return 3, set(), set() 2121 2122 problems = self.Entropy.check_config_file_updates() 2123 if problems: 2124 return 4, set(), set() 2125 2126 self.Entropy.updateProgress( 2127 "[repo:%s|%s] %s" % ( 2128 brown(repo), 2129 red(_("config files")), # something short please 2130 blue(_("no configuration files to commit. All fine.")), 2131 ), 2132 importance = 1, 2133 type = "info", 2134 header = blue(" @@ "), 2135 back = True 2136 ) 2137 2138 uris = [x[0] for x in upload_queue] 2139 errors, fine_uris, broken_uris = self.upload_database(uris, 2140 repo = repo) 2141 if errors: 2142 self.Entropy.updateProgress( 2143 "[repo:%s|%s] %s: %s" % ( 2144 brown(repo), 2145 red(_("sync")), 2146 blue(_("database sync failed")), 2147 red(_("upload issues")), 2148 ), 2149 importance = 1, 2150 type = "error", 2151 header = darkred(" !!! ") 2152 ) 2153 return 2, fine_uris, broken_uris 2154 2155 2156 self.Entropy.updateProgress( 2157 "[repo:%s|%s] %s" % ( 2158 brown(repo), 2159 red(_("sync")), 2160 blue(_("database sync completed successfully")), 2161 ), 2162 importance = 1, 2163 type = "info", 2164 header = darkgreen(" * ") 2165 ) 2166 2167 if unlock_mirrors: 2168 self.lock_mirrors(False, repo = repo) 2169 return 0, set(), set()

2170 2171

2172 - def calculate_local_upload_files(self, branch, repo = None):

2173 upload_files = 0 2174 upload_packages = set() 2175 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2176 branch) 2177 2178 # check if it exists 2179 if not os.path.isdir(upload_dir): 2180 return upload_files, upload_packages 2181 2182 pkg_ext = etpConst['packagesext'] 2183 pkg_md5_ext = etpConst['packagesmd5fileext'] 2184 for package in os.listdir(upload_dir): 2185 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2186 upload_packages.add(package) 2187 if package.endswith(pkg_ext): 2188 upload_files += 1 2189 2190 return upload_files, upload_packages

2191

2192 - def calculate_local_package_files(self, branch, repo = None):

2193 local_files = 0 2194 local_packages = set() 2195 packages_dir = os.path.join( 2196 self.Entropy.get_local_packages_directory(repo), branch) 2197 2198 if not os.path.isdir(packages_dir): 2199 os.makedirs(packages_dir) 2200 2201 pkg_ext = etpConst['packagesext'] 2202 pkg_md5_ext = etpConst['packagesmd5fileext'] 2203 for package in os.listdir(packages_dir): 2204 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2205 local_packages.add(package) 2206 if package.endswith(pkg_ext): 2207 local_files += 1 2208 2209 return local_files, local_packages

2210 2211

2212 - def _show_local_sync_stats(self, upload_files, local_files):

2213 self.Entropy.updateProgress( 2214 "%s:" % ( 2215 blue(_("Local statistics")), 2216 ), 2217 importance = 1, 2218 type = "info", 2219 header = red(" @@ ") 2220 ) 2221 self.Entropy.updateProgress( 2222 red("%s:\t\t%s %s" % ( 2223 blue(_("upload directory")), 2224 bold(str(upload_files)), 2225 red(_("files ready")), 2226 ) 2227 ), 2228 importance = 0, 2229 type = "info", 2230 header = red(" @@ ") 2231 ) 2232 self.Entropy.updateProgress( 2233 red("%s:\t\t%s %s" % ( 2234 blue(_("packages directory")), 2235 bold(str(local_files)), 2236 red(_("files ready")), 2237 ) 2238 ), 2239 importance = 0, 2240 type = "info", 2241 header = red(" @@ ") 2242 )

2243

2244 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2245 branch):

2246 2247 # show stats 2248 for package, size in upload: 2249 package = darkgreen(os.path.basename(package)) 2250 size = blue(self.entropyTools.bytes_into_human(size)) 2251 self.Entropy.updateProgress( 2252 "[branch:%s|%s] %s [%s]" % ( 2253 brown(branch), 2254 blue(_("upload")), 2255 darkgreen(package), 2256 size, 2257 ), 2258 importance = 0, 2259 type = "info", 2260 header = red(" # ") 2261 ) 2262 for package, size in download: 2263 package = darkred(os.path.basename(package)) 2264 size = blue(self.entropyTools.bytes_into_human(size)) 2265 self.Entropy.updateProgress( 2266 "[branch:%s|%s] %s [%s]" % ( 2267 brown(branch), 2268 darkred(_("download")), 2269 blue(package), 2270 size, 2271 ), 2272 importance = 0, 2273 type = "info", 2274 header = red(" # ") 2275 ) 2276 for package, size in copy: 2277 package = darkblue(os.path.basename(package)) 2278 size = blue(self.entropyTools.bytes_into_human(size)) 2279 self.Entropy.updateProgress( 2280 "[branch:%s|%s] %s [%s]" % ( 2281 brown(branch), 2282 darkgreen(_("copy")), 2283 brown(package), 2284 size, 2285 ), 2286 importance = 0, 2287 type = "info", 2288 header = red(" # ") 2289 ) 2290 for package, size in removal: 2291 package = brown(os.path.basename(package)) 2292 size = blue(self.entropyTools.bytes_into_human(size)) 2293 self.Entropy.updateProgress( 2294 "[branch:%s|%s] %s [%s]" % ( 2295 brown(branch), 2296 red(_("remove")), 2297 red(package), 2298 size, 2299 ), 2300 importance = 0, 2301 type = "info", 2302 header = red(" # ") 2303 ) 2304 2305 self.Entropy.updateProgress( 2306 "%s:\t\t\t%s" % ( 2307 blue(_("Packages to be removed")), 2308 darkred(str(len(removal))), 2309 ), 2310 importance = 0, 2311 type = "info", 2312 header = blue(" @@ ") 2313 ) 2314 self.Entropy.updateProgress( 2315 "%s:\t\t%s" % ( 2316 darkgreen(_("Packages to be moved locally")), 2317 darkgreen(str(len(copy))), 2318 ), 2319 importance = 0, 2320 type = "info", 2321 header = blue(" @@ ") 2322 ) 2323 self.Entropy.updateProgress( 2324 "%s:\t\t\t%s" % ( 2325 bold(_("Packages to be uploaded")), 2326 bold(str(len(upload))), 2327 ), 2328 importance = 0, 2329 type = "info", 2330 header = blue(" @@ ") 2331 ) 2332 2333 self.Entropy.updateProgress( 2334 "%s:\t\t\t%s" % ( 2335 darkred(_("Total removal size")), 2336 darkred( 2337 self.entropyTools.bytes_into_human(metainfo['removal']) 2338 ), 2339 ), 2340 importance = 0, 2341 type = "info", 2342 header = blue(" @@ ") 2343 ) 2344 2345 self.Entropy.updateProgress( 2346 "%s:\t\t\t%s" % ( 2347 blue(_("Total upload size")), 2348 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2349 ), 2350 importance = 0, 2351 type = "info", 2352 header = blue(" @@ ") 2353 ) 2354 self.Entropy.updateProgress( 2355 "%s:\t\t\t%s" % ( 2356 brown(_("Total download size")), 2357 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2358 ), 2359 importance = 0, 2360 type = "info", 2361 header = blue(" @@ ") 2362 )

2363 2364

2365 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2366 repo = None):

2367 2368 remote_files = 0 2369 close_conn = False 2370 remote_packages_data = {} 2371 2372 my_path = os.path.join( 2373 self.Entropy.get_remote_packages_relative_path(repo), branch) 2374 if ftp_connection == None: 2375 close_conn = True 2376 # let it raise an exception if things go bad 2377 ftp_connection = FtpInterface(uri, self.Entropy) 2378 ftp_connection.set_cwd(my_path, dodir = True) 2379 2380 remote_packages = ftp_connection.list_dir() 2381 remote_packages_info = ftp_connection.get_raw_list() 2382 if close_conn: 2383 ftp_connection.close() 2384 2385 for tbz2 in remote_packages: 2386 if tbz2.endswith(etpConst['packagesext']): 2387 remote_files += 1 2388 2389 for remote_package in remote_packages_info: 2390 remote_packages_data[remote_package.split()[8]] = \ 2391 int(remote_package.split()[4]) 2392 2393 return remote_files, remote_packages, remote_packages_data

2394

2395 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2396 2397 if repo == None: 2398 repo = self.Entropy.default_repository 2399 2400 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2401 upload_files, upload_packages = self.calculate_local_upload_files( 2402 branch, repo) 2403 local_files, local_packages = self.calculate_local_package_files(branch, 2404 repo) 2405 self._show_local_sync_stats(upload_files, local_files) 2406 2407 self.Entropy.updateProgress( 2408 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2409 importance = 1, 2410 type = "info", 2411 header = red(" @@ ") 2412 ) 2413 remote_files, remote_packages, remote_packages_data = \ 2414 self.calculate_remote_package_files(uri, branch, repo = repo) 2415 self.Entropy.updateProgress( 2416 "%s:\t\t\t%s %s" % ( 2417 blue(_("remote packages")), 2418 bold(str(remote_files)), 2419 red(_("files stored")), 2420 ), 2421 importance = 0, 2422 type = "info", 2423 header = red(" @@ ") 2424 ) 2425 2426 mytxt = blue("%s ...") % ( 2427 _("Calculating queues"), 2428 ) 2429 self.Entropy.updateProgress( 2430 mytxt, 2431 importance = 1, 2432 type = "info", 2433 header = red(" @@ ") 2434 ) 2435 2436 upload_queue, download_queue, removal_queue, fine_queue = \ 2437 self.calculate_sync_queues(upload_packages, local_packages, 2438 remote_packages, remote_packages_data, branch, repo) 2439 return upload_queue, download_queue, removal_queue, fine_queue, \ 2440 remote_packages_data

2441

2442 - def calculate_sync_queues( 2443 self, 2444 upload_packages, 2445 local_packages, 2446 remote_packages, 2447 remote_packages_data, 2448 branch, 2449 repo = None 2450 ):

2451 2452 upload_queue = set() 2453 download_queue = set() 2454 removal_queue = set() 2455 fine_queue = set() 2456 2457 for local_package in upload_packages: 2458 if local_package in remote_packages: 2459 2460 local_filepath = os.path.join( 2461 self.Entropy.get_local_upload_directory(repo), branch, 2462 local_package) 2463 2464 local_size = self.entropyTools.get_file_size(local_filepath) 2465 remote_size = remote_packages_data.get(local_package) 2466 if remote_size == None: 2467 remote_size = 0 2468 if (local_size != remote_size): 2469 # size does not match, adding to the upload queue 2470 upload_queue.add(local_package) 2471 else: 2472 # just move from upload to packages 2473 fine_queue.add(local_package) 2474 else: 2475 # always force upload of packages in uploaddir 2476 upload_queue.add(local_package) 2477 2478 # if a package is in the packages directory but not online, 2479 # we have to upload it we have local_packages and remote_packages 2480 for local_package in local_packages: 2481 if local_package in remote_packages: 2482 local_filepath = os.path.join( 2483 self.Entropy.get_local_packages_directory(repo), branch, 2484 local_package) 2485 local_size = self.entropyTools.get_file_size(local_filepath) 2486 remote_size = remote_packages_data.get(local_package) 2487 if remote_size == None: 2488 remote_size = 0 2489 if (local_size != remote_size) and (local_size != 0): 2490 # size does not match, adding to the upload queue 2491 if local_package not in fine_queue: 2492 upload_queue.add(local_package) 2493 else: 2494 # this means that the local package does not exist 2495 # so, we need to download it 2496 upload_queue.add(local_package) 2497 2498 # Fill download_queue and removal_queue 2499 for remote_package in remote_packages: 2500 if remote_package in local_packages: 2501 local_filepath = os.path.join( 2502 self.Entropy.get_local_packages_directory(repo), branch, 2503 remote_package) 2504 local_size = self.entropyTools.get_file_size(local_filepath) 2505 remote_size = remote_packages_data.get(remote_package) 2506 if remote_size == None: 2507 remote_size = 0 2508 if (local_size != remote_size) and (local_size != 0): 2509 # size does not match, remove first 2510 # do it only if the package has not been 2511 # added to the upload_queue 2512 if remote_package not in upload_queue: 2513 # remotePackage == localPackage 2514 # just remove something that differs 2515 # from the content of the mirror 2516 removal_queue.add(remote_package) 2517 # then add to the download queue 2518 download_queue.add(remote_package) 2519 else: 2520 # this means that the local package does not exist 2521 # so, we need to download it 2522 # ignore .tmp files 2523 if not remote_package.endswith(".tmp"): 2524 download_queue.add(remote_package) 2525 2526 # Collect packages that don't exist anymore in the database 2527 # so we can filter them out from the download queue 2528 dbconn = self.Entropy.open_server_repository(just_reading = True, 2529 repo = repo) 2530 db_files = dbconn.listAllDownloads(do_sort = False, 2531 full_path = True) 2532 db_files = set([os.path.basename(x) for x in db_files if \ 2533 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2534 2535 exclude = set() 2536 for myfile in download_queue: 2537 if myfile.endswith(etpConst['packagesext']): 2538 if myfile not in db_files: 2539 exclude.add(myfile) 2540 download_queue -= exclude 2541 2542 exclude = set() 2543 for myfile in upload_queue: 2544 if myfile.endswith(etpConst['packagesext']): 2545 if myfile not in db_files: 2546 exclude.add(myfile) 2547 upload_queue -= exclude 2548 2549 exclude = set() 2550 for myfile in download_queue: 2551 if myfile in upload_queue: 2552 exclude.add(myfile) 2553 download_queue -= exclude 2554 2555 return upload_queue, download_queue, removal_queue, fine_queue

2556 2557

2558 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2559 remote_packages_data, branch, repo):

2560 2561 metainfo = { 2562 'removal': 0, 2563 'download': 0, 2564 'upload': 0, 2565 } 2566 removal = [] 2567 download = [] 2568 do_copy = [] 2569 upload = [] 2570 2571 for item in removal_queue: 2572 if not item.endswith(etpConst['packagesext']): 2573 continue 2574 local_filepath = os.path.join( 2575 self.Entropy.get_local_packages_directory(repo), branch, item) 2576 size = self.entropyTools.get_file_size(local_filepath) 2577 metainfo['removal'] += size 2578 removal.append((local_filepath, size)) 2579 2580 for item in download_queue: 2581 if not item.endswith(etpConst['packagesext']): 2582 continue 2583 local_filepath = os.path.join( 2584 self.Entropy.get_local_upload_directory(repo), branch, item) 2585 if not os.path.isfile(local_filepath): 2586 size = remote_packages_data.get(item) 2587 if size == None: 2588 size = 0 2589 size = int(size) 2590 metainfo['removal'] += size 2591 download.append((local_filepath, size)) 2592 else: 2593 size = self.entropyTools.get_file_size(local_filepath) 2594 do_copy.append((local_filepath, size)) 2595 2596 for item in upload_queue: 2597 if not item.endswith(etpConst['packagesext']): 2598 continue 2599 local_filepath = os.path.join( 2600 self.Entropy.get_local_upload_directory(repo), branch, item) 2601 local_filepath_pkgs = os.path.join( 2602 self.Entropy.get_local_packages_directory(repo), branch, item) 2603 if os.path.isfile(local_filepath): 2604 size = self.entropyTools.get_file_size(local_filepath) 2605 upload.append((local_filepath, size)) 2606 else: 2607 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2608 upload.append((local_filepath_pkgs, size)) 2609 metainfo['upload'] += size 2610 2611 return upload, download, removal, do_copy, metainfo

2612 2613

2614 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2615 2616 if repo == None: 2617 repo = self.Entropy.default_repository 2618 2619 for itemdata in removal_queue: 2620 2621 remove_filename = itemdata[0] 2622 remove_filepath = os.path.join( 2623 self.Entropy.get_local_packages_directory(repo), branch, 2624 remove_filename) 2625 remove_filepath_hash = remove_filepath + \ 2626 etpConst['packagesmd5fileext'] 2627 self.Entropy.updateProgress( 2628 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2629 brown(repo), 2630 red("sync"), 2631 brown(branch), 2632 blue(_("removing package+hash")), 2633 darkgreen(remove_filename), 2634 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2635 ), 2636 importance = 0, 2637 type = "info", 2638 header = darkred(" * ") 2639 ) 2640 2641 if os.path.isfile(remove_filepath): 2642 os.remove(remove_filepath) 2643 if os.path.isfile(remove_filepath_hash): 2644 os.remove(remove_filepath_hash) 2645 2646 self.Entropy.updateProgress( 2647 "[repo:%s|%s|%s] %s" % ( 2648 brown(repo), 2649 red(_("sync")), 2650 brown(branch), 2651 blue(_("removal complete")), 2652 ), 2653 importance = 0, 2654 type = "info", 2655 header = darkred(" * ") 2656 )

2657 2658

2659 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2660 2661 if repo == None: 2662 repo = self.Entropy.default_repository 2663 2664 for itemdata in copy_queue: 2665 2666 from_file = itemdata[0] 2667 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2668 to_file = os.path.join( 2669 self.Entropy.get_local_packages_directory(repo), branch, 2670 os.path.basename(from_file)) 2671 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2672 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2673 self.Entropy.updateProgress( 2674 "[repo:%s|%s|%s] %s: %s" % ( 2675 brown(repo), 2676 red("sync"), 2677 brown(branch), 2678 blue(_("copying file+hash to repository")), 2679 darkgreen(from_file), 2680 ), 2681 importance = 0, 2682 type = "info", 2683 header = darkred(" * ") 2684 ) 2685 2686 if not os.path.isdir(os.path.dirname(to_file)): 2687 os.makedirs(os.path.dirname(to_file)) 2688 2689 shutil.copy2(from_file, to_file) 2690 if not os.path.isfile(from_file_hash): 2691 self.create_file_checksum(from_file, from_file_hash) 2692 shutil.copy2(from_file_hash, to_file_hash) 2693 2694 # clear expiration file 2695 if os.path.isfile(expiration_file): 2696 os.remove(expiration_file)

2697 2698

2699 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2700 2701 if repo == None: 2702 repo = self.Entropy.default_repository 2703 2704 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2705 myqueue = [] 2706 for itemdata in upload_queue: 2707 upload_item = itemdata[0] 2708 hash_file = upload_item + etpConst['packagesmd5fileext'] 2709 if not os.path.isfile(hash_file): 2710 self.entropyTools.create_md5_file(upload_item) 2711 myqueue.append(hash_file) 2712 myqueue.append(upload_item) 2713 2714 ftp_basedir = os.path.join( 2715 self.Entropy.get_remote_packages_relative_path(repo), branch) 2716 uploader = self.FtpServerHandler(FtpInterface, 2717 self.Entropy, [uri], myqueue, critical_files = myqueue, 2718 use_handlers = True, ftp_basedir = ftp_basedir, 2719 handlers_data = {'branch': branch }, repo = repo) 2720 errors, m_fine_uris, m_broken_uris = uploader.go() 2721 if errors: 2722 my_broken_uris = [ 2723 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2724 x in m_broken_uris] 2725 reason = my_broken_uris[0][1] 2726 self.Entropy.updateProgress( 2727 "[branch:%s] %s: %s, %s: %s" % ( 2728 brown(branch), 2729 blue(_("upload errors")), 2730 red(crippled_uri), 2731 blue(_("reason")), 2732 darkgreen(unicode(reason)), 2733 ), 2734 importance = 1, 2735 type = "error", 2736 header = darkred(" !!! ") 2737 ) 2738 return errors, m_fine_uris, m_broken_uris 2739 2740 self.Entropy.updateProgress( 2741 "[branch:%s] %s: %s" % ( 2742 brown(branch), 2743 blue(_("upload completed successfully")), 2744 red(crippled_uri), 2745 ), 2746 importance = 1, 2747 type = "info", 2748 header = blue(" @@ ") 2749 ) 2750 return errors, m_fine_uris, m_broken_uris

2751 2752

2753 - def _sync_run_download_queue(self, uri, download_queue, branch, 2754 repo = None):

2755 2756 if repo == None: 2757 repo = self.Entropy.default_repository 2758 2759 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2760 myqueue = [] 2761 for package in download_queue: 2762 hash_file = package + etpConst['packagesmd5fileext'] 2763 myqueue.append(package) 2764 myqueue.append(hash_file) 2765 2766 ftp_basedir = os.path.join( 2767 self.Entropy.get_remote_packages_relative_path(repo), branch) 2768 local_basedir = os.path.join( 2769 self.Entropy.get_local_packages_directory(repo), branch) 2770 downloader = self.FtpServerHandler( 2771 FtpInterface, self.Entropy, [uri], myqueue, 2772 critical_files = myqueue, use_handlers = True, 2773 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2774 handlers_data = {'branch': branch }, download = True, repo = repo) 2775 2776 errors, m_fine_uris, m_broken_uris = downloader.go() 2777 if errors: 2778 my_broken_uris = [ 2779 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2780 for x, y in m_broken_uris] 2781 reason = my_broken_uris[0][1] 2782 self.Entropy.updateProgress( 2783 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2784 brown(repo), 2785 red(_("sync")), 2786 brown(branch), 2787 blue(_("download errors")), 2788 darkgreen(crippled_uri), 2789 blue(_("reason")), 2790 reason, 2791 ), 2792 importance = 1, 2793 type = "error", 2794 header = darkred(" !!! ") 2795 ) 2796 return errors, m_fine_uris, m_broken_uris 2797 2798 self.Entropy.updateProgress( 2799 "[repo:%s|%s|%s] %s: %s" % ( 2800 brown(repo), 2801 red(_("sync")), 2802 brown(branch), 2803 blue(_("download completed successfully")), 2804 darkgreen(crippled_uri), 2805 ), 2806 importance = 1, 2807 type = "info", 2808 header = darkgreen(" * ") 2809 ) 2810 return errors, m_fine_uris, m_broken_uris

2811

2812 - def run_package_files_qa_checks(self, packages_list, repo = None):

2813 2814 if repo == None: 2815 repo = self.Entropy.default_repository 2816 2817 my_qa = self.Entropy.QA() 2818 2819 qa_total = len(packages_list) 2820 qa_count = 0 2821 qa_some_faulty = [] 2822 2823 for upload_package in packages_list: 2824 qa_count += 1 2825 2826 self.Entropy.updateProgress( 2827 "%s: %s" % ( 2828 purple(_("QA checking package file")), 2829 darkgreen(os.path.basename(upload_package)), 2830 ), 2831 importance = 0, 2832 type = "info", 2833 header = purple(" @@ "), 2834 back = True, 2835 count = (qa_count, qa_total,) 2836 ) 2837 2838 result = my_qa.entropy_package_checks(upload_package) 2839 if not result: 2840 # call wolfman-911 2841 qa_some_faulty.append(os.path.basename(upload_package)) 2842 2843 if qa_some_faulty: 2844 2845 for qa_faulty_pkg in qa_some_faulty: 2846 self.Entropy.updateProgress( 2847 "[repo:%s|branch:%s] %s: %s" % ( 2848 brown(repo), 2849 self.SystemSettings['repositories']['branch'], 2850 red(_("faulty package file, please fix")), 2851 blue(os.path.basename(qa_faulty_pkg)), 2852 ), 2853 importance = 1, 2854 type = "error", 2855 header = darkred(" @@ ") 2856 ) 2857 raise EntropyPackageException( 2858 'EntropyPackageException: cannot continue')

2859 2860

2861 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2862 repo = None):

2863 2864 if repo == None: 2865 repo = self.Entropy.default_repository 2866 2867 self.Entropy.updateProgress( 2868 "[repo:%s|%s] %s" % ( 2869 repo, 2870 red(_("sync")), 2871 darkgreen(_("starting packages sync")), 2872 ), 2873 importance = 1, 2874 type = "info", 2875 header = red(" @@ "), 2876 back = True 2877 ) 2878 2879 2880 successfull_mirrors = set() 2881 broken_mirrors = set() 2882 check_data = () 2883 upload_queue_qa_checked = set() 2884 mirrors_tainted = False 2885 mirror_errors = False 2886 mirrors_errors = False 2887 2888 for uri in self.Entropy.get_remote_mirrors(repo): 2889 2890 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2891 mirror_errors = False 2892 2893 self.Entropy.updateProgress( 2894 "[repo:%s|%s|branch:%s] %s: %s" % ( 2895 repo, 2896 red(_("sync")), 2897 brown(self.SystemSettings['repositories']['branch']), 2898 blue(_("packages sync")), 2899 bold(crippled_uri), 2900 ), 2901 importance = 1, 2902 type = "info", 2903 header = red(" @@ ") 2904 ) 2905 2906 try: 2907 upload_queue, download_queue, removal_queue, fine_queue, \ 2908 remote_packages_data = self.calculate_packages_to_sync(uri, 2909 self.SystemSettings['repositories']['branch'], repo) 2910 except self.socket.error, err: 2911 self.Entropy.updateProgress( 2912 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2913 repo, 2914 red(_("sync")), 2915 self.SystemSettings['repositories']['branch'], 2916 darkred(_("socket error")), 2917 err, 2918 darkred(_("on")), 2919 crippled_uri, 2920 ), 2921 importance = 1, 2922 type = "error", 2923 header = darkgreen(" * ") 2924 ) 2925 continue 2926 2927 if (not upload_queue) and (not download_queue) and \ 2928 (not removal_queue): 2929 self.Entropy.updateProgress( 2930 "[repo:%s|%s|branch:%s] %s: %s" % ( 2931 repo, 2932 red(_("sync")), 2933 self.SystemSettings['repositories']['branch'], 2934 darkgreen(_("nothing to do on")), 2935 crippled_uri, 2936 ), 2937 importance = 1, 2938 type = "info", 2939 header = darkgreen(" * ") 2940 ) 2941 successfull_mirrors.add(uri) 2942 continue 2943 2944 self.Entropy.updateProgress( 2945 "%s:" % (blue(_("Expanding queues")),), 2946 importance = 1, 2947 type = "info", 2948 header = red(" ** ") 2949 ) 2950 2951 upload, download, removal, copy, metainfo = self.expand_queues( 2952 upload_queue, 2953 download_queue, 2954 removal_queue, 2955 remote_packages_data, 2956 self.SystemSettings['repositories']['branch'], 2957 repo 2958 ) 2959 del upload_queue, download_queue, removal_queue, \ 2960 remote_packages_data 2961 self._show_sync_queues(upload, download, removal, copy, metainfo, 2962 self.SystemSettings['repositories']['branch']) 2963 2964 if not len(upload)+len(download)+len(removal)+len(copy): 2965 2966 self.Entropy.updateProgress( 2967 "[repo:%s|%s|branch:%s] %s %s" % ( 2968 self.Entropy.default_repository, 2969 red(_("sync")), 2970 self.SystemSettings['repositories']['branch'], 2971 blue(_("nothing to sync for")), 2972 crippled_uri, 2973 ), 2974 importance = 1, 2975 type = "info", 2976 header = darkgreen(" @@ ") 2977 ) 2978 2979 successfull_mirrors.add(uri) 2980 continue 2981 2982 if pretend: 2983 successfull_mirrors.add(uri) 2984 continue 2985 2986 if ask: 2987 rc_sync = self.Entropy.askQuestion( 2988 _("Would you like to run the steps above ?")) 2989 if rc_sync == "No": 2990 continue 2991 2992 try: 2993 2994 # QA checks 2995 qa_package_files = [x[0] for x in upload if x[0] \ 2996 not in upload_queue_qa_checked] 2997 upload_queue_qa_checked |= set(qa_package_files) 2998 2999 self.run_package_files_qa_checks(qa_package_files, repo = repo) 3000 3001 if removal: 3002 self._sync_run_removal_queue(removal, 3003 self.SystemSettings['repositories']['branch'], repo) 3004 3005 if copy: 3006 self._sync_run_copy_queue(copy, 3007 self.SystemSettings['repositories']['branch'], repo) 3008 3009 if upload or download: 3010 mirrors_tainted = True 3011 3012 if upload: 3013 d_errors, m_fine_uris, \ 3014 m_broken_uris = self._sync_run_upload_queue( 3015 uri, upload, 3016 self.SystemSettings['repositories']['branch'], repo) 3017 3018 if d_errors: 3019 mirror_errors = True 3020 3021 if download: 3022 my_downlist = [x[0] for x in download] 3023 d_errors, m_fine_uris, \ 3024 m_broken_uris = self._sync_run_download_queue( 3025 uri, my_downlist, 3026 self.SystemSettings['repositories']['branch'], repo) 3027 3028 if d_errors: 3029 mirror_errors = True 3030 if not mirror_errors: 3031 successfull_mirrors.add(uri) 3032 else: 3033 mirrors_errors = True 3034 3035 except KeyboardInterrupt: 3036 self.Entropy.updateProgress( 3037 "[repo:%s|%s|branch:%s] %s" % ( 3038 repo, 3039 red(_("sync")), 3040 self.SystemSettings['repositories']['branch'], 3041 darkgreen(_("keyboard interrupt !")), 3042 ), 3043 importance = 1, 3044 type = "info", 3045 header = darkgreen(" * ") 3046 ) 3047 continue 3048 3049 except EntropyPackageException: 3050 3051 mirrors_errors = True 3052 broken_mirrors.add(uri) 3053 successfull_mirrors.clear() 3054 # so that people will realize this is a very bad thing 3055 self.Entropy.updateProgress( 3056 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3057 repo, 3058 red(_("sync")), 3059 self.SystemSettings['repositories']['branch'], 3060 darkred(_("you must package them again")), 3061 Exception, 3062 _("error"), 3063 err, 3064 ), 3065 importance = 1, 3066 type = "error", 3067 header = darkred(" !!! ") 3068 ) 3069 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3070 broken_mirrors, check_data 3071 3072 except Exception, err: 3073 3074 self.entropyTools.print_traceback() 3075 mirrors_errors = True 3076 broken_mirrors.add(uri) 3077 self.Entropy.updateProgress( 3078 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3079 repo, 3080 red(_("sync")), 3081 self.SystemSettings['repositories']['branch'], 3082 darkred(_("exception caught")), 3083 Exception, 3084 _("error"), 3085 err, 3086 ), 3087 importance = 1, 3088 type = "error", 3089 header = darkred(" !!! ") 3090 ) 3091 3092 exc_txt = self.Entropy.entropyTools.print_exception( 3093 returndata = True) 3094 for line in exc_txt: 3095 self.Entropy.updateProgress( 3096 unicode(line), 3097 importance = 1, 3098 type = "error", 3099 header = darkred(": ") 3100 ) 3101 3102 if len(successfull_mirrors) > 0: 3103 self.Entropy.updateProgress( 3104 "[repo:%s|%s|branch:%s] %s" % ( 3105 repo, 3106 red(_("sync")), 3107 self.SystemSettings['repositories']['branch'], 3108 darkred( 3109 _("at least one mirror synced properly!")), 3110 ), 3111 importance = 1, 3112 type = "error", 3113 header = darkred(" !!! ") 3114 ) 3115 continue 3116 3117 # if at least one server has been synced successfully, move files 3118 if (len(successfull_mirrors) > 0) and not pretend: 3119 self.remove_expiration_files( 3120 self.SystemSettings['repositories']['branch'], repo) 3121 3122 if packages_check: 3123 check_data = self.Entropy.verify_local_packages([], ask = ask, 3124 repo = repo) 3125 3126 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3127 broken_mirrors, check_data

3128

3129 - def remove_expiration_files(self, branch, repo = None):

3130 3131 if repo == None: 3132 repo = self.Entropy.default_repository 3133 3134 branch_dir = os.path.join( 3135 self.Entropy.get_local_upload_directory(repo), branch) 3136 3137 # check if it exists 3138 if not os.path.isdir(branch_dir): 3139 return None 3140 3141 branchcontent = os.listdir(branch_dir) 3142 for xfile in branchcontent: 3143 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3144 branch, xfile) 3145 destdir = os.path.join( 3146 self.Entropy.get_local_packages_directory(repo), branch) 3147 if not os.path.isdir(destdir): 3148 os.makedirs(destdir) 3149 dest = os.path.join(destdir, xfile) 3150 shutil.move(source, dest) 3151 # clear expiration file 3152 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3153 if os.path.isfile(dest_expiration): 3154 os.remove(dest_expiration)

3155 3156

3157 - def is_package_expired(self, package_file, branch, repo = None):

3158 3159 pkg_path = os.path.join( 3160 self.Entropy.get_local_packages_directory(repo), branch, 3161 package_file) 3162 pkg_path += etpConst['packagesexpirationfileext'] 3163 if not os.path.isfile(pkg_path): 3164 return False 3165 3166 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3167 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3168 days = srv_set['packages_expiration_days'] 3169 delta = int(days)*24*3600 3170 currmtime = time.time() 3171 file_delta = currmtime - mtime 3172 3173 if file_delta > delta: 3174 return True 3175 return False

3176

3177 - def create_expiration_file(self, package_file, branch, repo = None, 3178 gentle = False):

3179 3180 pkg_path = os.path.join( 3181 self.Entropy.get_local_packages_directory(repo), branch, 3182 package_file) 3183 pkg_path += etpConst['packagesexpirationfileext'] 3184 if gentle and os.path.isfile(pkg_path): 3185 return 3186 f_exp = open(pkg_path,"w") 3187 f_exp.flush() 3188 f_exp.close()

3189 3190

3191 - def collect_expiring_packages(self, branch, repo = None):

3192 3193 dbconn = self.Entropy.open_server_repository(just_reading = True, 3194 repo = repo) 3195 database_bins = dbconn.listAllDownloads(do_sort = False, 3196 full_path = True) 3197 bins_dir = os.path.join( 3198 self.Entropy.get_local_packages_directory(repo), branch) 3199 repo_bins = set() 3200 3201 if os.path.isdir(bins_dir): 3202 repo_bins = os.listdir(bins_dir) 3203 repo_bins = set([ 3204 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3205 in repo_bins if x.endswith(etpConst['packagesext'])]) 3206 repo_bins -= database_bins 3207 3208 return set([os.path.basename(x) for x in repo_bins])

3209 3210

3211 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3212 3213 if repo == None: 3214 repo = self.Entropy.default_repository 3215 3216 self.Entropy.updateProgress( 3217 "[repo:%s|%s|branch:%s] %s" % ( 3218 brown(repo), 3219 red(_("tidy")), 3220 blue(self.SystemSettings['repositories']['branch']), 3221 blue(_("collecting expired packages")), 3222 ), 3223 importance = 1, 3224 type = "info", 3225 header = red(" @@ ") 3226 ) 3227 3228 branch_data = {} 3229 errors = False 3230 branch_data['errors'] = False 3231 branch = self.SystemSettings['repositories']['branch'] 3232 3233 self.Entropy.updateProgress( 3234 "[branch:%s] %s" % ( 3235 brown(branch), 3236 blue(_("collecting expired packages in the selected branches")), 3237 ), 3238 importance = 1, 3239 type = "info", 3240 header = blue(" @@ ") 3241 ) 3242 3243 # collect removed packages 3244 expiring_packages = self.collect_expiring_packages(branch, repo) 3245 if expiring_packages: 3246 3247 # filter expired packages used by other branches 3248 # this is done for the sake of consistency 3249 # --- read packages.db.pkglist, make sure your repository 3250 # has been ported to latest Entropy 3251 3252 branch_pkglist_data = self.read_remote_file_in_branches( 3253 etpConst['etpdatabasepkglist'], repo = repo, 3254 excluded_branches = [branch]) 3255 # format data 3256 for key, val in branch_pkglist_data.items(): 3257 branch_pkglist_data[key] = val.split("\n") 3258 3259 3260 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3261 branch) 3262 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3263 expiring_packages]) 3264 3265 for other_branch in branch_pkglist_data: 3266 branch_pkglist = set(branch_pkglist_data[other_branch]) 3267 my_expiring_pkgs -= branch_pkglist 3268 3269 # fallback to normality, set new expiring packages var 3270 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3271 3272 removal = [] 3273 for package in expiring_packages: 3274 expired = self.is_package_expired(package, branch, repo) 3275 if expired: 3276 removal.append(package) 3277 else: 3278 self.create_expiration_file(package, branch, repo, 3279 gentle = True) 3280 3281 # fill returning data 3282 branch_data['removal'] = removal[:] 3283 3284 if not removal: 3285 self.Entropy.updateProgress( 3286 "[branch:%s] %s" % ( 3287 brown(branch), 3288 blue(_("nothing to remove on this branch")), 3289 ), 3290 importance = 1, 3291 type = "info", 3292 header = blue(" @@ ") 3293 ) 3294 return errors, branch_data 3295 else: 3296 self.Entropy.updateProgress( 3297 "[branch:%s] %s:" % ( 3298 brown(branch), 3299 blue(_("these are the expired packages")), 3300 ), 3301 importance = 1, 3302 type = "info", 3303 header = blue(" @@ ") 3304 ) 3305 for package in removal: 3306 self.Entropy.updateProgress( 3307 "[branch:%s] %s: %s" % ( 3308 brown(branch), 3309 blue(_("remove")), 3310 darkgreen(package), 3311 ), 3312 importance = 1, 3313 type = "info", 3314 header = brown(" # ") 3315 ) 3316 3317 if pretend: 3318 return errors, branch_data 3319 3320 if ask: 3321 rc_question = self.Entropy.askQuestion( 3322 _("Would you like to continue ?")) 3323 if rc_question == "No": 3324 return errors, branch_data 3325 3326 myqueue = [] 3327 for package in removal: 3328 myqueue.append(package+etpConst['packagesmd5fileext']) 3329 myqueue.append(package) 3330 3331 ftp_basedir = os.path.join( 3332 self.Entropy.get_remote_packages_relative_path(repo), branch) 3333 for uri in self.Entropy.get_remote_mirrors(repo): 3334 3335 self.Entropy.updateProgress( 3336 "[branch:%s] %s..." % ( 3337 brown(branch), 3338 blue(_("removing packages remotely")), 3339 ), 3340 importance = 1, 3341 type = "info", 3342 header = blue(" @@ ") 3343 ) 3344 3345 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3346 destroyer = self.FtpServerHandler( 3347 FtpInterface, 3348 self.Entropy, 3349 [uri], 3350 myqueue, 3351 critical_files = [], 3352 ftp_basedir = ftp_basedir, 3353 remove = True, 3354 repo = repo 3355 ) 3356 errors, m_fine_uris, m_broken_uris = destroyer.go() 3357 if errors: 3358 my_broken_uris = [ 3359 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3360 for x in m_broken_uris] 3361 3362 reason = my_broken_uris[0][1] 3363 self.Entropy.updateProgress( 3364 "[branch:%s] %s: %s, %s: %s" % ( 3365 brown(branch), 3366 blue(_("remove errors")), 3367 red(crippled_uri), 3368 blue(_("reason")), 3369 reason, 3370 ), 3371 importance = 1, 3372 type = "warning", 3373 header = brown(" !!! ") 3374 ) 3375 branch_data['errors'] = True 3376 errors = True 3377 3378 self.Entropy.updateProgress( 3379 "[branch:%s] %s..." % ( 3380 brown(branch), 3381 blue(_("removing packages locally")), 3382 ), 3383 importance = 1, 3384 type = "info", 3385 header = blue(" @@ ") 3386 ) 3387 3388 branch_data['removed'] = set() 3389 for package in removal: 3390 package_path = os.path.join( 3391 self.Entropy.get_local_packages_directory(repo), 3392 branch, package) 3393 package_path_hash = package_path + \ 3394 etpConst['packagesmd5fileext'] 3395 package_path_expired = package_path + \ 3396 etpConst['packagesexpirationfileext'] 3397 3398 my_rm_list = (package_path_hash, package_path, 3399 package_path_expired) 3400 for myfile in my_rm_list: 3401 if os.path.isfile(myfile): 3402 self.Entropy.updateProgress( 3403 "[branch:%s] %s: %s" % ( 3404 brown(branch), 3405 blue(_("removing")), 3406 darkgreen(myfile), 3407 ), 3408 importance = 1, 3409 type = "info", 3410 header = brown(" @@ ") 3411 ) 3412 os.remove(myfile) 3413 branch_data['removed'].add(myfile) 3414 3415 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors