entropy.server.interfaces.mirrors

28 29 import socket 30 import entropy.dump as dumpTools 31 import entropy.tools as entropyTools

32 - def __init__(self, ServerInstance, repo = None):

33 34 from entropy.server.interfaces.main import Server as MainServer 35 if not isinstance(ServerInstance, MainServer): 36 mytxt = _("entropy.server.interfaces.main.Server needed") 37 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 38 39 self.Entropy = ServerInstance 40 from entropy.transceivers import FtpServerHandler 41 self.FtpServerHandler = FtpServerHandler 42 from entropy.cache import EntropyCacher 43 self.Cacher = EntropyCacher() 44 self.sys_settings_plugin_id = \ 45 etpConst['system_settings_plugins_ids']['server_plugin'] 46 self.SystemSettings = self.Entropy.SystemSettings 47 48 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 49 self.Entropy.updateProgress( 50 mytxt, 51 importance = 2, 52 type = "info", 53 header = red(" @@ ") 54 ) 55 mytxt = _("mirror") 56 for mirror in self.Entropy.get_remote_mirrors(repo): 57 mirror = self.entropyTools.hide_ftp_password(mirror) 58 self.Entropy.updateProgress( 59 blue("%s: %s") % (mytxt, darkgreen(mirror),), 60 importance = 0, 61 type = "info", 62 header = brown(" # ") 63 )

64

65 - def get_remote_branches(self, repo = None):

66 """ 67 Returns a list of remotely available branches for the provided 68 repository identifier. 69 70 @keyword repo: repository identifier 71 @type repo: string 72 @return: list of valid, available remote branches 73 @rtype: set 74 """ 75 76 if repo == None: 77 repo = self.Entropy.default_repository 78 79 remote_branches = set() 80 # this is used to validate Entropy repository path 81 ts_file = etpConst['etpdatabasetimestampfile'] 82 mirrors = self.Entropy.get_remote_mirrors(repo) 83 for uri in mirrors: 84 85 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 86 87 self.Entropy.updateProgress( 88 "[repo:%s] %s: %s" % ( 89 brown(repo), 90 blue(_("listing branches in mirror")), 91 darkgreen(crippled_uri), 92 ), 93 importance = 1, 94 type = "info", 95 header = brown(" @@ ") 96 ) 97 98 ftp = FtpInterface(uri, self.Entropy, verbose = False) 99 try: 100 101 branches_path = self.Entropy.get_remote_database_relative_path( 102 repo) 103 try: 104 ftp.set_cwd(branches_path) 105 except FtpError: 106 continue # no branches in this mirror 107 108 branches = ftp.list_dir() 109 for branch in branches: 110 mypath = os.path.join("/", branches_path, branch) 111 try: 112 ftp.set_cwd(mypath) 113 except FtpError: 114 # not a dir 115 continue 116 if ftp.is_file_available(ts_file): 117 remote_branches.add(branch) 118 119 finally: 120 ftp.close() 121 122 return remote_branches

123

124 - def read_remote_file_in_branches(self, filename, repo = None, 125 excluded_branches = None):

126 """ 127 Reads a file remotely located in all the available branches. 128 129 @param filename: name of the file that should be located inside 130 repository database directory 131 @type filename: string 132 @keyword repo: repository identifier 133 @type repo: string 134 @keyword excluded_branches: list of branch identifiers excluded or None 135 @type excluded_branches: list or None 136 @return: dictionary with branches as key and raw file content as value: 137 {'4': 'abcd\n', '5': 'defg\n'} 138 @rtype: dict 139 """ 140 if repo == None: 141 repo = self.Entropy.default_repository 142 if excluded_branches == None: 143 excluded_branches = [] 144 145 branch_data = {} 146 mirrors = self.Entropy.get_remote_mirrors(repo) 147 for uri in mirrors: 148 149 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 150 151 self.Entropy.updateProgress( 152 "[repo:%s] %s: %s => %s" % ( 153 brown(repo), 154 blue(_("looking for file in mirror")), 155 darkgreen(crippled_uri), 156 filename, 157 ), 158 importance = 1, 159 type = "info", 160 header = brown(" @@ ") 161 ) 162 163 ftp = FtpInterface(uri, self.Entropy, verbose = False) 164 try: 165 166 branches_path = self.Entropy.get_remote_database_relative_path( 167 repo) 168 try: 169 ftp.set_cwd(branches_path) 170 except FtpError: 171 continue # no branches in this mirror 172 173 branches = ftp.list_dir() 174 for branch in branches: 175 176 # is branch excluded ? 177 if branch in excluded_branches: 178 continue 179 180 if branch_data.get(branch) != None: 181 # already read 182 continue 183 184 mypath = os.path.join("/", branches_path, branch) 185 try: 186 ftp.set_cwd(mypath) 187 except FtpError: 188 # not a dir 189 continue 190 191 if not ftp.is_file_available(filename): 192 # nothing to do, no file here 193 continue 194 195 tmp_dir = tempfile.mkdtemp() 196 tries = 4 197 success = False 198 while tries: 199 downloaded = ftp.download_file(filename, tmp_dir) 200 if not downloaded: 201 tries -= 1 202 continue # argh! 203 success = True 204 break 205 206 down_path = os.path.join(tmp_dir, filename) 207 if success and os.access(down_path, os.F_OK): 208 down_f = open(down_path) 209 branch_data[branch] = down_f.read() 210 down_f.close() 211 shutil.rmtree(tmp_dir) 212 213 finally: 214 ftp.close() 215 216 return branch_data

217

218 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

219 220 if repo == None: 221 repo = self.Entropy.default_repository 222 223 if not mirrors: 224 mirrors = self.Entropy.get_remote_mirrors(repo) 225 226 issues = False 227 for uri in mirrors: 228 229 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 230 231 lock_text = _("unlocking") 232 if lock: 233 lock_text = _("locking") 234 self.Entropy.updateProgress( 235 "[repo:%s|%s] %s %s" % ( 236 brown(repo), 237 darkgreen(crippled_uri), 238 bold(lock_text), 239 blue("%s...") % (_("mirror"),), 240 ), 241 importance = 1, 242 type = "info", 243 header = brown(" * "), 244 back = True 245 ) 246 247 try: 248 ftp = FtpInterface(uri, self.Entropy) 249 except ConnectionError: 250 self.entropyTools.print_traceback() 251 return True # issues 252 my_path = os.path.join( 253 self.Entropy.get_remote_database_relative_path(repo), 254 self.SystemSettings['repositories']['branch']) 255 ftp.set_cwd(my_path, dodir = True) 256 257 lock_file = etpConst['etpdatabaselockfile'] 258 if lock and ftp.is_file_available(lock_file): 259 self.Entropy.updateProgress( 260 "[repo:%s|%s] %s" % ( 261 brown(repo), 262 darkgreen(crippled_uri), 263 blue(_("mirror already locked")), 264 ), 265 importance = 1, 266 type = "info", 267 header = darkgreen(" * ") 268 ) 269 ftp.close() 270 continue 271 elif not lock and not ftp.is_file_available(lock_file): 272 self.Entropy.updateProgress( 273 "[repo:%s|%s] %s" % ( 274 brown(repo), 275 darkgreen(crippled_uri), 276 blue(_("mirror already unlocked")), 277 ), 278 importance = 1, 279 type = "info", 280 header = darkgreen(" * ") 281 ) 282 ftp.close() 283 continue 284 285 if lock: 286 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 287 else: 288 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 289 ftp.close() 290 if not rc_lock: 291 issues = True 292 293 if not issues: 294 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 295 if os.path.isfile(db_taint_file): 296 os.remove(db_taint_file) 297 298 return issues

299 300

301 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 302 repo = None):

303 # this functions makes entropy clients to not download anything 304 # from the chosen mirrors. it is used to avoid clients to 305 # download databases while we're uploading a new one 306 307 if mirrors == None: 308 mirrors = [] 309 310 if repo == None: 311 repo = self.Entropy.default_repository 312 313 if not mirrors: 314 mirrors = self.Entropy.get_remote_mirrors(repo) 315 316 issues = False 317 for uri in mirrors: 318 319 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 320 321 lock_text = _("unlocking") 322 if lock: 323 lock_text = _("locking") 324 self.Entropy.updateProgress( 325 "[repo:%s|%s] %s %s..." % ( 326 blue(repo), 327 red(crippled_uri), 328 bold(lock_text), 329 blue(_("mirror for download")), 330 ), 331 importance = 1, 332 type = "info", 333 header = red(" @@ "), 334 back = True 335 ) 336 337 try: 338 ftp = FtpInterface(uri, self.Entropy) 339 except ConnectionError: 340 self.entropyTools.print_traceback() 341 return True # issues 342 my_path = os.path.join( 343 self.Entropy.get_remote_database_relative_path(repo), 344 self.SystemSettings['repositories']['branch']) 345 ftp.set_cwd(my_path, dodir = True) 346 347 lock_file = etpConst['etpdatabasedownloadlockfile'] 348 349 if lock and ftp.is_file_available(lock_file): 350 self.Entropy.updateProgress( 351 "[repo:%s|%s] %s" % ( 352 blue(repo), 353 red(crippled_uri), 354 blue(_("mirror already locked for download")), 355 ), 356 importance = 1, 357 type = "info", 358 header = red(" @@ ") 359 ) 360 ftp.close() 361 continue 362 363 elif not lock and not ftp.is_file_available(lock_file): 364 self.Entropy.updateProgress( 365 "[repo:%s|%s] %s" % ( 366 blue(repo), 367 red(crippled_uri), 368 blue(_("mirror already unlocked for download")), 369 ), 370 importance = 1, 371 type = "info", 372 header = red(" @@ ") 373 ) 374 ftp.close() 375 continue 376 377 if lock: 378 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 379 repo = repo) 380 else: 381 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 382 repo = repo) 383 ftp.close() 384 if not rc_lock: 385 issues = True 386 387 return issues

388

389 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 390 repo = None):

391 392 if repo == None: 393 repo = self.Entropy.default_repository 394 395 my_path = os.path.join( 396 self.Entropy.get_remote_database_relative_path(repo), 397 self.SystemSettings['repositories']['branch']) 398 if not ftp_connection: 399 try: 400 ftp_connection = FtpInterface(uri, self.Entropy) 401 except ConnectionError: 402 self.entropyTools.print_traceback() 403 return False # issues 404 ftp_connection.set_cwd(my_path, dodir = True) 405 else: 406 mycwd = ftp_connection.get_cwd() 407 if mycwd != my_path: 408 ftp_connection.set_basedir() 409 ftp_connection.set_cwd(my_path, dodir = True) 410 411 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 412 lock_string = u'' 413 414 if dblock: 415 self.create_local_database_lockfile(repo) 416 lock_file = self.get_database_lockfile(repo) 417 else: 418 # locking/unlocking mirror1 for download 419 lock_string = _('for download') 420 self.create_local_database_download_lockfile(repo) 421 lock_file = self.get_database_download_lockfile(repo) 422 423 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 424 if rc_upload: 425 self.Entropy.updateProgress( 426 "[repo:%s|%s] %s %s" % ( 427 blue(repo), 428 red(crippled_uri), 429 blue(_("mirror successfully locked")), 430 blue(lock_string), 431 ), 432 importance = 1, 433 type = "info", 434 header = red(" @@ ") 435 ) 436 else: 437 self.Entropy.updateProgress( 438 "[repo:%s|%s] %s: %s - %s %s" % ( 439 blue(repo), 440 red(crippled_uri), 441 blue("lock error"), 442 rc_upload, 443 blue(_("mirror not locked")), 444 blue(lock_string), 445 ), 446 importance = 1, 447 type = "error", 448 header = darkred(" * ") 449 ) 450 self.remove_local_database_lockfile(repo) 451 452 return rc_upload

453 454

455 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

456 457 if repo == None: 458 repo = self.Entropy.default_repository 459 460 my_path = os.path.join( 461 self.Entropy.get_remote_database_relative_path(repo), 462 self.SystemSettings['repositories']['branch']) 463 if not ftp_connection: 464 try: 465 ftp_connection = FtpInterface(uri, self.Entropy) 466 except ConnectionError: 467 self.entropyTools.print_traceback() 468 return False # issues 469 ftp_connection.set_cwd(my_path) 470 else: 471 mycwd = ftp_connection.get_cwd() 472 if mycwd != my_path: 473 ftp_connection.set_basedir() 474 ftp_connection.set_cwd(my_path) 475 476 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 477 478 if dblock: 479 dbfile = etpConst['etpdatabaselockfile'] 480 else: 481 dbfile = etpConst['etpdatabasedownloadlockfile'] 482 rc_delete = ftp_connection.delete_file(dbfile) 483 if rc_delete: 484 self.Entropy.updateProgress( 485 "[repo:%s|%s] %s" % ( 486 blue(repo), 487 red(crippled_uri), 488 blue(_("mirror successfully unlocked")), 489 ), 490 importance = 1, 491 type = "info", 492 header = darkgreen(" * ") 493 ) 494 if dblock: 495 self.remove_local_database_lockfile(repo) 496 else: 497 self.remove_local_database_download_lockfile(repo) 498 else: 499 self.Entropy.updateProgress( 500 "[repo:%s|%s] %s: %s - %s" % ( 501 blue(repo), 502 red(crippled_uri), 503 blue(_("unlock error")), 504 rc_delete, 505 blue(_("mirror not unlocked")), 506 ), 507 importance = 1, 508 type = "error", 509 header = darkred(" * ") 510 ) 511 512 return rc_delete

513

514 - def get_database_lockfile(self, repo = None):

515 if repo == None: 516 repo = self.Entropy.default_repository 517 return os.path.join(self.Entropy.get_local_database_dir(repo), 518 etpConst['etpdatabaselockfile'])

519

520 - def get_database_download_lockfile(self, repo = None):

521 if repo == None: 522 repo = self.Entropy.default_repository 523 return os.path.join(self.Entropy.get_local_database_dir(repo), 524 etpConst['etpdatabasedownloadlockfile'])

525

526 - def create_local_database_download_lockfile(self, repo = None):

527 if repo == None: 528 repo = self.Entropy.default_repository 529 lock_file = self.get_database_download_lockfile(repo) 530 f_lock = open(lock_file, "w") 531 f_lock.write("download locked") 532 f_lock.flush() 533 f_lock.close()

534

535 - def create_local_database_lockfile(self, repo = None):

536 if repo == None: 537 repo = self.Entropy.default_repository 538 lock_file = self.get_database_lockfile(repo) 539 f_lock = open(lock_file, "w") 540 f_lock.write("database locked") 541 f_lock.flush() 542 f_lock.close()

543

544 - def remove_local_database_lockfile(self, repo = None):

545 if repo == None: 546 repo = self.Entropy.default_repository 547 lock_file = self.get_database_lockfile(repo) 548 if os.path.isfile(lock_file): 549 os.remove(lock_file)

550

551 - def remove_local_database_download_lockfile(self, repo = None):

552 if repo == None: 553 repo = self.Entropy.default_repository 554 lock_file = self.get_database_download_lockfile(repo) 555 if os.path.isfile(lock_file): 556 os.remove(lock_file)

557

558 - def download_package(self, uri, pkg_relative_path, repo = None):

559 560 if repo == None: 561 repo = self.Entropy.default_repository 562 563 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 564 565 tries = 0 566 567 try: 568 ftp = FtpInterface(uri, self.Entropy) 569 except ConnectionError: 570 self.entropyTools.print_traceback() 571 return False # issues 572 573 while tries < 5: 574 tries += 1 575 576 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 577 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 578 pkgfile = os.path.basename(pkg_relative_path) 579 580 self.Entropy.updateProgress( 581 "[repo:%s|%s|#%s] %s: %s" % ( 582 brown(repo), 583 darkgreen(crippled_uri), 584 brown(str(tries)), 585 blue(_("connecting to download package")), 586 darkgreen(pkg_to_join_path), 587 ), 588 importance = 1, 589 type = "info", 590 header = darkgreen(" * "), 591 back = True 592 ) 593 594 dirpath = os.path.join( 595 self.Entropy.get_remote_packages_relative_path(repo), 596 pkg_to_join_dirpath) 597 ftp.set_cwd(dirpath, dodir = True) 598 599 self.Entropy.updateProgress( 600 "[repo:%s|%s|#%s] %s: %s" % ( 601 brown(repo), 602 darkgreen(crippled_uri), 603 brown(str(tries)), 604 blue(_("downloading package")), 605 darkgreen(pkg_to_join_path), 606 ), 607 importance = 1, 608 type = "info", 609 header = darkgreen(" * ") 610 ) 611 612 download_path = os.path.join( 613 self.Entropy.get_local_packages_directory(repo), 614 pkg_to_join_dirpath) 615 if (not os.path.isdir(download_path)) and \ 616 (not os.access(download_path, os.R_OK)): 617 os.makedirs(download_path) 618 rc_download = ftp.download_file(pkgfile, download_path) 619 if not rc_download: 620 self.Entropy.updateProgress( 621 "[repo:%s|%s|#%s] %s: %s %s" % ( 622 brown(repo), 623 darkgreen(crippled_uri), 624 brown(str(tries)), 625 blue(_("package")), 626 darkgreen(pkg_to_join_path), 627 blue(_("does not exist")), 628 ), 629 importance = 1, 630 type = "error", 631 header = darkred(" !!! ") 632 ) 633 ftp.close() 634 return rc_download 635 636 dbconn = self.Entropy.open_server_repository(read_only = True, 637 no_upload = True, repo = repo) 638 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 639 if idpackage == -1: 640 self.Entropy.updateProgress( 641 "[repo:%s|%s|#%s] %s: %s %s" % ( 642 brown(repo), 643 darkgreen(crippled_uri), 644 brown(str(tries)), 645 blue(_("package")), 646 darkgreen(pkgfile), 647 blue(_("is not listed in the repository !")), 648 ), 649 importance = 1, 650 type = "error", 651 header = darkred(" !!! ") 652 ) 653 ftp.close() 654 return False 655 656 storedmd5 = dbconn.retrieveDigest(idpackage) 657 self.Entropy.updateProgress( 658 "[repo:%s|%s|#%s] %s: %s" % ( 659 brown(repo), 660 darkgreen(crippled_uri), 661 brown(str(tries)), 662 blue(_("verifying checksum of package")), 663 darkgreen(pkgfile), 664 ), 665 importance = 1, 666 type = "info", 667 header = darkgreen(" * "), 668 back = True 669 ) 670 671 pkg_path = os.path.join(download_path, pkgfile) 672 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 673 if md5check: 674 self.Entropy.updateProgress( 675 "[repo:%s|%s|#%s] %s: %s %s" % ( 676 brown(repo), 677 darkgreen(crippled_uri), 678 brown(str(tries)), 679 blue(_("package")), 680 darkgreen(pkgfile), 681 blue(_("downloaded successfully")), 682 ), 683 importance = 1, 684 type = "info", 685 header = darkgreen(" * ") 686 ) 687 ftp.close() 688 return True 689 else: 690 self.Entropy.updateProgress( 691 "[repo:%s|%s|#%s] %s: %s %s" % ( 692 brown(repo), 693 darkgreen(crippled_uri), 694 brown(str(tries)), 695 blue(_("package")), 696 darkgreen(pkgfile), 697 blue(_("checksum does not match. re-downloading...")), 698 ), 699 importance = 1, 700 type = "warning", 701 header = darkred(" * ") 702 ) 703 if os.path.isfile(pkg_path): 704 os.remove(pkg_path) 705 706 continue 707 708 # if we get here it means the files hasn't been downloaded properly 709 self.Entropy.updateProgress( 710 "[repo:%s|%s|#%s] %s: %s %s" % ( 711 brown(repo), 712 darkgreen(crippled_uri), 713 brown(str(tries)), 714 blue(_("package")), 715 darkgreen(pkgfile), 716 blue(_("seems broken. Consider to re-package it. Giving up!")), 717 ), 718 importance = 1, 719 type = "error", 720 header = darkred(" !!! ") 721 ) 722 ftp.close() 723 return False

724 725

726 - def get_remote_databases_status(self, repo = None, mirrors = None):

727 728 if repo == None: 729 repo = self.Entropy.default_repository 730 731 if not mirrors: 732 mirrors = self.Entropy.get_remote_mirrors(repo) 733 734 data = [] 735 for uri in mirrors: 736 737 # let it raise an exception if connection is impossible 738 ftp = FtpInterface(uri, self.Entropy) 739 try: 740 my_path = os.path.join( 741 self.Entropy.get_remote_database_relative_path(repo), 742 self.SystemSettings['repositories']['branch']) 743 ftp.set_cwd(my_path, dodir = True) 744 except ftp.ftplib.error_perm: 745 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 746 self.Entropy.updateProgress( 747 "[repo:%s|%s] %s !" % ( 748 brown(repo), 749 darkgreen(crippled_uri), 750 blue(_("mirror has invalid directory structure")), 751 ), 752 importance = 1, 753 type = "warning", 754 header = darkred(" !!! ") 755 ) 756 ftp.close() 757 continue 758 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 759 db_format = sys_set['database_file_format'] 760 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 761 if cmethod == None: 762 raise InvalidDataType("InvalidDataType: %s." % ( 763 _("Wrong database compression method passed"), 764 ) 765 ) 766 compressedfile = etpConst[cmethod[2]] 767 768 revision = 0 769 rc1 = ftp.is_file_available(compressedfile) 770 revfilename = os.path.basename( 771 self.Entropy.get_local_database_revision_file(repo)) 772 rc2 = ftp.is_file_available(revfilename) 773 if rc1 and rc2: 774 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 775 revfilename) 776 dlcount = 5 777 while dlcount: 778 dled = ftp.download_file(revfilename, 779 etpConst['packagestmpdir'], True) 780 if dled: 781 break 782 dlcount -= 1 783 784 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 785 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 786 f_rev = open(revision_localtmppath,"r") 787 try: 788 revision = int(f_rev.readline().strip()) 789 except ValueError: 790 mytxt = _("mirror hasn't valid database revision file") 791 self.Entropy.updateProgress( 792 "[repo:%s|%s] %s: %s" % ( 793 brown(repo), 794 darkgreen(crippled_uri), 795 blue(mytxt), 796 bold(revision), 797 ), 798 importance = 1, 799 type = "error", 800 header = darkred(" !!! ") 801 ) 802 revision = 0 803 f_rev.close() 804 elif dlcount == 0: 805 self.Entropy.updateProgress( 806 "[repo:%s|%s] %s: %s" % ( 807 brown(repo), 808 darkgreen(crippled_uri), 809 blue(_("unable to download repository revision")), 810 bold(revision), 811 ), 812 importance = 1, 813 type = "error", 814 header = darkred(" !!! ") 815 ) 816 revision = 0 817 else: 818 self.Entropy.updateProgress( 819 "[repo:%s|%s] %s: %s" % ( 820 brown(repo), 821 darkgreen(crippled_uri), 822 blue(_("mirror doesn't have valid revision file")), 823 bold(revision), 824 ), 825 importance = 1, 826 type = "error", 827 header = darkred(" !!! ") 828 ) 829 revision = 0 830 if os.path.isfile(revision_localtmppath): 831 os.remove(revision_localtmppath) 832 833 info = [uri, revision] 834 data.append(info) 835 ftp.close() 836 837 return data

838

839 - def is_local_database_locked(self, repo = None):

840 local_repo = repo 841 if local_repo == None: 842 local_repo = self.Entropy.default_repository 843 lock_file = self.get_database_lockfile(local_repo) 844 return os.path.isfile(lock_file)

845

846 - def get_mirrors_lock(self, repo = None):

847 848 dbstatus = [] 849 for uri in self.Entropy.get_remote_mirrors(repo): 850 data = [uri, False, False] 851 ftp = FtpInterface(uri, self.Entropy) 852 try: 853 my_path = os.path.join( 854 self.Entropy.get_remote_database_relative_path(repo), 855 self.SystemSettings['repositories']['branch']) 856 ftp.set_cwd(my_path) 857 except ftp.ftplib.error_perm: 858 ftp.close() 859 continue 860 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 861 # upload locked 862 data[1] = True 863 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 864 # download locked 865 data[2] = True 866 ftp.close() 867 dbstatus.append(data) 868 return dbstatus

869

870 - def download_notice_board(self, repo = None):

871 872 if repo == None: 873 repo = self.Entropy.default_repository 874 mirrors = self.Entropy.get_remote_mirrors(repo) 875 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 876 mytmpdir = self.entropyTools.get_random_temp_file() 877 os.makedirs(mytmpdir) 878 879 self.Entropy.updateProgress( 880 "[repo:%s] %s %s" % ( 881 brown(repo), 882 blue(_("downloading notice board from mirrors to")), 883 red(rss_path), 884 ), 885 importance = 1, 886 type = "info", 887 header = blue(" @@ ") 888 ) 889 890 downloaded = False 891 for uri in mirrors: 892 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 893 downloader = self.FtpServerHandler( 894 FtpInterface, self.Entropy, [uri], 895 [rss_path], download = True, 896 local_basedir = mytmpdir, critical_files = [rss_path], 897 repo = repo 898 ) 899 errors, m_fine_uris, m_broken_uris = downloader.go() 900 if not errors: 901 self.Entropy.updateProgress( 902 "[repo:%s] %s: %s" % ( 903 brown(repo), 904 blue(_("notice board downloaded successfully from")), 905 red(crippled_uri), 906 ), 907 importance = 1, 908 type = "info", 909 header = blue(" @@ ") 910 ) 911 downloaded = True 912 break 913 914 if downloaded: 915 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 916 rss_path) 917 918 return downloaded

919

920 - def upload_notice_board(self, repo = None):

921 922 if repo == None: 923 repo = self.Entropy.default_repository 924 mirrors = self.Entropy.get_remote_mirrors(repo) 925 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 926 927 self.Entropy.updateProgress( 928 "[repo:%s] %s %s" % ( 929 brown(repo), 930 blue(_("uploading notice board from")), 931 red(rss_path), 932 ), 933 importance = 1, 934 type = "info", 935 header = blue(" @@ ") 936 ) 937 938 uploader = self.FtpServerHandler( 939 FtpInterface, 940 self.Entropy, 941 mirrors, 942 [rss_path], 943 critical_files = [rss_path], 944 repo = repo 945 ) 946 errors, m_fine_uris, m_broken_uris = uploader.go() 947 if errors: 948 m_broken_uris = sorted(m_broken_uris) 949 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 950 for x in m_broken_uris] 951 self.Entropy.updateProgress( 952 "[repo:%s] %s %s" % ( 953 brown(repo), 954 blue(_("notice board upload failed on")), 955 red(', '.join(m_broken_uris)), 956 ), 957 importance = 1, 958 type = "info", 959 header = blue(" @@ ") 960 ) 961 return False 962 self.Entropy.updateProgress( 963 "[repo:%s] %s" % ( 964 brown(repo), 965 blue(_("notice board upload success")), 966 ), 967 importance = 1, 968 type = "info", 969 header = blue(" @@ ") 970 ) 971 return True

972 973

974 - def update_notice_board(self, title, notice_text, link = None, repo = None):

975 976 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 977 rss_description = "Inform about important distribution activities." 978 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 979 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 980 if not link: 981 link = srv_set['rss']['website_url'] 982 983 self.download_notice_board(repo) 984 rss_main = RSS(rss_path, rss_title, rss_description, 985 maxentries = 20) 986 rss_main.add_item(title, link, description = notice_text) 987 rss_main.write_changes() 988 status = self.upload_notice_board(repo) 989 return status

990

991 - def read_notice_board(self, do_download = True, repo = None):

992 993 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 994 if do_download: 995 self.download_notice_board(repo) 996 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 997 return None 998 rss_main = RSS(rss_path, '', '') 999 return rss_main.get_entries()

1000

1001 - def remove_from_notice_board(self, identifier, repo = None):

1002 1003 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 1004 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 1005 rss_description = "Inform about important distribution activities." 1006 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 1007 return 0 1008 rss_main = RSS(rss_path, rss_title, rss_description) 1009 data = rss_main.remove_entry(identifier) 1010 rss_main.write_changes() 1011 return data

1012

1013 - def update_rss_feed(self, repo = None):

1014 1015 if repo == None: 1016 repo = self.Entropy.default_repository 1017 1018 product = self.SystemSettings['repositories']['product'] 1019 #db_dir = self.Entropy.get_local_database_dir(repo) 1020 rss_path = self.Entropy.get_local_database_rss_file(repo) 1021 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 1022 rss_dump_name = repo + etpConst['rss-dump-name'] 1023 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 1024 1025 rss_title = u"%s Online Repository Status" % ( 1026 self.SystemSettings['system']['name'],) 1027 rss_description = \ 1028 u"Keep you updated on what's going on in the %s Repository." % ( 1029 self.SystemSettings['system']['name'],) 1030 1031 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1032 1033 rss_main = RSS(rss_path, rss_title, rss_description, 1034 maxentries = srv_set['rss']['max_entries']) 1035 # load dump 1036 db_actions = self.Cacher.pop(rss_dump_name) 1037 if db_actions: 1038 try: 1039 f_rev = open(db_revision_path) 1040 revision = f_rev.readline().strip() 1041 f_rev.close() 1042 except (IOError, OSError): 1043 revision = u"N/A" 1044 commitmessage = u'' 1045 if self.Entropy.rssMessages['commitmessage']: 1046 commitmessage = u' :: ' + \ 1047 self.Entropy.rssMessages['commitmessage'] 1048 1049 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 1050 product[0].upper() + product[1:] + " " + \ 1051 self.SystemSettings['repositories']['branch'] + \ 1052 " :: Revision: " + revision + commitmessage 1053 1054 link = srv_set['rss']['base_url'] 1055 # create description 1056 added_items = db_actions.get("added") 1057 1058 if added_items: 1059 for atom in sorted(added_items): 1060 mylink = link + "?search=" + atom.split("~")[0] + \ 1061 "&arch=" + etpConst['currentarch'] + "&product="+product 1062 description = atom + ": " + added_items[atom]['description'] 1063 rss_main.add_item(title = "Added/Updated" + title, 1064 link = mylink, description = description) 1065 removed_items = db_actions.get("removed") 1066 1067 if removed_items: 1068 for atom in sorted(removed_items): 1069 description = atom + ": " + \ 1070 removed_items[atom]['description'] 1071 rss_main.add_item(title = "Removed" + title, link = link, 1072 description = description) 1073 1074 light_items = db_actions.get('light') 1075 if light_items: 1076 rss_light = RSS(rss_light_path, rss_title, rss_description, 1077 maxentries = srv_set['rss']['light_max_entries']) 1078 for atom in sorted(light_items): 1079 mylink = link + "?search=" + atom.split("~")[0] + \ 1080 "&arch=" + etpConst['currentarch'] + "&product=" + \ 1081 product 1082 description = light_items[atom]['description'] 1083 rss_light.add_item(title = "[" + revision + "] " + atom, 1084 link = mylink, description = description) 1085 rss_light.write_changes() 1086 1087 rss_main.write_changes() 1088 self.Entropy.rssMessages.clear() 1089 self.dumpTools.removeobj(rss_dump_name)

1090 1091

1092 - def dump_database_to_file(self, db_path, destination_path, opener, 1093 exclude_tables = None, repo = None):

1094 1095 if not exclude_tables: 1096 exclude_tables = [] 1097 1098 f_out = opener(destination_path, "wb") 1099 dbconn = self.Entropy.open_server_repository(db_path, 1100 just_reading = True, repo = repo, do_treeupdates = False) 1101 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 1102 self.Entropy.close_server_database(dbconn) 1103 f_out.close()

1104

1105 - def create_file_checksum(self, file_path, checksum_path):

1106 mydigest = self.entropyTools.md5sum(file_path) 1107 f_ck = open(checksum_path, "w") 1108 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 1109 f_ck.write(mystring) 1110 f_ck.flush() 1111 f_ck.close()

1112

1113 - def compress_file(self, file_path, destination_path, opener):

1114 f_out = opener(destination_path, "wb") 1115 f_in = open(file_path,"rb") 1116 data = f_in.read(8192) 1117 while data: 1118 f_out.write(data) 1119 data = f_in.read(8192) 1120 f_in.close() 1121 if hasattr(f_out,'flush'): 1122 f_out.flush() 1123 f_out.close()

1124

1125 - def get_files_to_sync(self, cmethod, download = False, repo = None, 1126 disabled_eapis = None):

1127 1128 if disabled_eapis == None: 1129 disabled_eapis = [] 1130 1131 critical = [] 1132 extra_text_files = [] 1133 data = {} 1134 data['database_revision_file'] = \ 1135 self.Entropy.get_local_database_revision_file(repo) 1136 extra_text_files.append(data['database_revision_file']) 1137 critical.append(data['database_revision_file']) 1138 1139 1140 # branch migration support scripts 1141 post_branch_mig_file = self.Entropy.get_local_post_branch_mig_script( 1142 repo) 1143 if os.path.isfile(post_branch_mig_file) or download: 1144 data['database_post_branch_hop_script'] = post_branch_mig_file 1145 extra_text_files.append(data['database_post_branch_hop_script']) 1146 1147 post_branch_upg_file = self.Entropy.get_local_post_branch_upg_script( 1148 repo) 1149 if os.path.isfile(post_branch_upg_file) or download: 1150 data['database_post_branch_upgrade_script'] = post_branch_upg_file 1151 extra_text_files.append(data['database_post_branch_upgrade_script']) 1152 1153 1154 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1155 if os.path.isfile(database_ts_file) or download: 1156 data['database_timestamp_file'] = database_ts_file 1157 if not download: 1158 critical.append(database_ts_file) 1159 1160 database_package_mask_file = \ 1161 self.Entropy.get_local_database_mask_file(repo) 1162 extra_text_files.append(database_package_mask_file) 1163 if os.path.isfile(database_package_mask_file) or download: 1164 data['database_package_mask_file'] = database_package_mask_file 1165 if not download: 1166 critical.append(data['database_package_mask_file']) 1167 1168 database_package_system_mask_file = \ 1169 self.Entropy.get_local_database_system_mask_file(repo) 1170 extra_text_files.append(database_package_system_mask_file) 1171 if os.path.isfile(database_package_system_mask_file) or download: 1172 data['database_package_system_mask_file'] = \ 1173 database_package_system_mask_file 1174 if not download: 1175 critical.append(data['database_package_system_mask_file']) 1176 1177 database_package_confl_tagged_file = \ 1178 self.Entropy.get_local_database_confl_tagged_file(repo) 1179 extra_text_files.append(database_package_confl_tagged_file) 1180 if os.path.isfile(database_package_confl_tagged_file) or download: 1181 data['database_package_confl_tagged_file'] = \ 1182 database_package_confl_tagged_file 1183 if not download: 1184 critical.append(data['database_package_confl_tagged_file']) 1185 1186 database_license_whitelist_file = \ 1187 self.Entropy.get_local_database_licensewhitelist_file(repo) 1188 extra_text_files.append(database_license_whitelist_file) 1189 if os.path.isfile(database_license_whitelist_file) or download: 1190 data['database_license_whitelist_file'] = \ 1191 database_license_whitelist_file 1192 if not download: 1193 critical.append(data['database_license_whitelist_file']) 1194 1195 exp_based_pkgs_removal_file = \ 1196 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1197 extra_text_files.append(exp_based_pkgs_removal_file) 1198 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1199 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1200 if not download: 1201 critical.append(data['exp_based_pkgs_removal_file']) 1202 1203 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1204 if os.path.isfile(database_rss_file) or download: 1205 data['database_rss_file'] = database_rss_file 1206 if not download: 1207 critical.append(data['database_rss_file']) 1208 database_rss_light_file = \ 1209 self.Entropy.get_local_database_rsslight_file(repo) 1210 1211 extra_text_files.append(database_rss_light_file) 1212 if os.path.isfile(database_rss_light_file) or download: 1213 data['database_rss_light_file'] = database_rss_light_file 1214 if not download: 1215 critical.append(data['database_rss_light_file']) 1216 1217 pkglist_file = self.Entropy.get_local_pkglist_file(repo) 1218 data['pkglist_file'] = pkglist_file 1219 if not download: 1220 critical.append(data['pkglist_file']) 1221 1222 critical_updates_file = self.Entropy.get_local_critical_updates_file( 1223 repo) 1224 if os.path.isfile(critical_updates_file) or download: 1225 data['critical_updates_file'] = critical_updates_file 1226 extra_text_files.append(data['critical_updates_file']) 1227 if not download: 1228 critical.append(data['critical_updates_file']) 1229 1230 keywords_file = self.Entropy.get_local_database_keywords_file( 1231 repo) 1232 if os.path.isfile(keywords_file) or download: 1233 data['keywords_file'] = keywords_file 1234 extra_text_files.append(data['keywords_file']) 1235 if not download: 1236 critical.append(data['keywords_file']) 1237 1238 # EAPI 2,3 1239 if not download: # we don't need to get the dump 1240 1241 if 3 not in disabled_eapis: 1242 1243 data['metafiles_path'] = \ 1244 self.Entropy.get_local_database_compressed_metafiles_file( 1245 repo) 1246 critical.append(data['metafiles_path']) 1247 1248 if 2 not in disabled_eapis: 1249 1250 data['dump_path_light'] = os.path.join( 1251 self.Entropy.get_local_database_dir(repo), 1252 etpConst[cmethod[5]]) 1253 critical.append(data['dump_path_light']) 1254 1255 data['dump_path_digest_light'] = os.path.join( 1256 self.Entropy.get_local_database_dir(repo), 1257 etpConst[cmethod[6]]) 1258 critical.append(data['dump_path_digest_light']) 1259 1260 # EAPI 1 1261 if 1 not in disabled_eapis: 1262 1263 data['compressed_database_path'] = os.path.join( 1264 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1265 critical.append(data['compressed_database_path']) 1266 data['compressed_database_path_light'] = os.path.join( 1267 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[7]]) 1268 1269 data['database_path_digest'] = os.path.join( 1270 self.Entropy.get_local_database_dir(repo), 1271 etpConst['etpdatabasehashfile'] 1272 ) 1273 critical.append(data['database_path_digest']) 1274 1275 data['compressed_database_path_digest'] = os.path.join( 1276 self.Entropy.get_local_database_dir(repo), 1277 etpConst[cmethod[2]] + etpConst['packagesmd5fileext'] 1278 ) 1279 critical.append(data['compressed_database_path_digest']) 1280 1281 data['compressed_database_path_digest_light'] = os.path.join( 1282 self.Entropy.get_local_database_dir(repo), 1283 etpConst[cmethod[8]] 1284 ) 1285 critical.append(data['compressed_database_path_digest_light']) 1286 1287 1288 # SSL cert file, just for reference 1289 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1290 extra_text_files.append(ssl_ca_cert) 1291 if os.path.isfile(ssl_ca_cert): 1292 data['ssl_ca_cert_file'] = ssl_ca_cert 1293 if not download: 1294 critical.append(ssl_ca_cert) 1295 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1296 extra_text_files.append(ssl_server_cert) 1297 if os.path.isfile(ssl_server_cert): 1298 data['ssl_server_cert_file'] = ssl_server_cert 1299 if not download: 1300 critical.append(ssl_server_cert) 1301 1302 # Some information regarding how packages are built 1303 spm_files = [ 1304 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1305 (etpConst['spm']['global_package_keywords'], 1306 "global_package_keywords"), 1307 (etpConst['spm']['global_package_use'], "global_package_use"), 1308 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1309 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1310 ] 1311 for myfile, myname in spm_files: 1312 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1313 data[myname] = myfile 1314 extra_text_files.append(myfile) 1315 1316 make_profile = etpConst['spm']['global_make_profile'] 1317 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1318 mytmpdir = os.path.dirname(rnd_tmp_file) 1319 mytmpfile = os.path.join(mytmpdir, 1320 etpConst['spm']['global_make_profile_link_name']) 1321 extra_text_files.append(mytmpfile) 1322 1323 if os.path.islink(make_profile): 1324 mylink = os.readlink(make_profile) 1325 f_mkp = open(mytmpfile,"w") 1326 f_mkp.write(mylink) 1327 f_mkp.flush() 1328 f_mkp.close() 1329 data['global_make_profile'] = mytmpfile 1330 1331 return data, critical, extra_text_files

1332

1333 - def _show_package_sets_messages(self, repo):

1334 1335 self.Entropy.updateProgress( 1336 "[repo:%s] %s:" % ( 1337 brown(repo), 1338 blue(_("configured package sets")), 1339 ), 1340 importance = 0, 1341 type = "info", 1342 header = darkgreen(" * ") 1343 ) 1344 sets_data = self.Entropy.package_set_list(matchRepo = [repo]) 1345 if not sets_data: 1346 self.Entropy.updateProgress( 1347 "%s" % (_("None configured"),), 1348 importance = 0, 1349 type = "info", 1350 header = brown(" # ") 1351 ) 1352 return 1353 for s_repo, s_name, s_sets in sets_data: 1354 self.Entropy.updateProgress( 1355 blue("%s" % (s_name,)), 1356 importance = 0, 1357 type = "info", 1358 header = brown(" # ") 1359 )

1360

1361 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1362 1363 self.Entropy.updateProgress( 1364 "[repo:%s|%s|%s:%s] %s" % ( 1365 brown(repo), 1366 darkgreen(crippled_uri), 1367 red("EAPI"), 1368 bold("3"), 1369 blue(_("preparing uncompressed database for the upload")), 1370 ), 1371 importance = 0, 1372 type = "info", 1373 header = darkgreen(" * ") 1374 ) 1375 self.Entropy.updateProgress( 1376 "%s: %s" % (_("database path"), blue(database_path),), 1377 importance = 0, 1378 type = "info", 1379 header = brown(" # ") 1380 )

1381

1382 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1383 upload_data, cmethod, repo):

1384 1385 if repo == None: 1386 repo = self.Entropy.default_repository 1387 1388 self.Entropy.updateProgress( 1389 "[repo:%s|%s|%s:%s] %s" % ( 1390 brown(repo), 1391 darkgreen(crippled_uri), 1392 red("EAPI"), 1393 bold("2"), 1394 blue(_("creating compressed database dump + checksum")), 1395 ), 1396 importance = 0, 1397 type = "info", 1398 header = darkgreen(" * ") 1399 ) 1400 self.Entropy.updateProgress( 1401 "%s: %s" % (_("database path"), blue(database_path),), 1402 importance = 0, 1403 type = "info", 1404 header = brown(" # ") 1405 ) 1406 self.Entropy.updateProgress( 1407 "%s: %s" % ( 1408 _("dump light"), 1409 blue(upload_data['dump_path_light']), 1410 ), 1411 importance = 0, 1412 type = "info", 1413 header = brown(" # ") 1414 ) 1415 self.Entropy.updateProgress( 1416 "%s: %s" % ( 1417 _("dump light checksum"), 1418 blue(upload_data['dump_path_digest_light']), 1419 ), 1420 importance = 0, 1421 type = "info", 1422 header = brown(" # ") 1423 ) 1424 1425 self.Entropy.updateProgress( 1426 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1427 importance = 0, 1428 type = "info", 1429 header = brown(" # ") 1430 )

1431

1432 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1433 upload_data, cmethod, repo):

1434 1435 self.Entropy.updateProgress( 1436 "[repo:%s|%s|%s:%s] %s" % ( 1437 brown(repo), 1438 darkgreen(crippled_uri), 1439 red("EAPI"), 1440 bold("1"), 1441 blue(_("compressing database + checksum")), 1442 ), 1443 importance = 0, 1444 type = "info", 1445 header = darkgreen(" * "), 1446 back = True 1447 ) 1448 self.Entropy.updateProgress( 1449 "%s: %s" % (_("database path"), blue(database_path),), 1450 importance = 0, 1451 type = "info", 1452 header = brown(" # ") 1453 ) 1454 self.Entropy.updateProgress( 1455 "%s: %s" % ( 1456 _("compressed database path"), 1457 blue(upload_data['compressed_database_path']), 1458 ), 1459 importance = 0, 1460 type = "info", 1461 header = brown(" # ") 1462 ) 1463 self.Entropy.updateProgress( 1464 "%s: %s" % ( 1465 _("database checksum"), 1466 blue(upload_data['database_path_digest']), 1467 ), 1468 importance = 0, 1469 type = "info", 1470 header = brown(" # ") 1471 ) 1472 self.Entropy.updateProgress( 1473 "%s: %s" % ( 1474 _("compressed checksum"), 1475 blue(upload_data['compressed_database_path_digest']), 1476 ), 1477 importance = 0, 1478 type = "info", 1479 header = brown(" # ") 1480 ) 1481 self.Entropy.updateProgress( 1482 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1483 importance = 0, 1484 type = "info", 1485 header = brown(" # ") 1486 )

1487

1488 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1489 1490 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1491 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1492 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1493 file_list if x not in found_file_list] 1494 metafile_not_found_file = \ 1495 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1496 1497 f_meta = open(metafile_not_found_file,"w") 1498 f_meta.writelines(not_found_file_list) 1499 f_meta.flush() 1500 f_meta.close() 1501 found_file_list.append(metafile_not_found_file) 1502 if os.path.isfile(compressed_dest_path): 1503 os.remove(compressed_dest_path) 1504 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1505

1506 - def create_mirror_directories(self, ftp_connection, path_to_create):

1507 bdir = "" 1508 for mydir in path_to_create.split("/"): 1509 bdir += "/"+mydir 1510 if not ftp_connection.is_file_available(bdir): 1511 try: 1512 ftp_connection.mkdir(bdir) 1513 except Exception, err: 1514 error = unicode(err) 1515 if (error.find("550") == -1) and \ 1516 (error.find("File exist") == -1): 1517 1518 mytxt = "%s %s, %s: %s" % ( 1519 _("cannot create mirror directory"), 1520 bdir, 1521 _("error"), 1522 err, 1523 ) 1524 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1525 mytxt,)) 1526 raise

1527

1528 - def mirror_lock_check(self, uri, repo = None):

1529 1530 if repo == None: 1531 repo = self.Entropy.default_repository 1532 1533 gave_up = False 1534 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1535 try: 1536 ftp = FtpInterface(uri, self.Entropy) 1537 except ConnectionError: 1538 self.entropyTools.print_traceback() 1539 return True # gave up 1540 my_path = os.path.join( 1541 self.Entropy.get_remote_database_relative_path(repo), 1542 self.SystemSettings['repositories']['branch']) 1543 ftp.set_cwd(my_path, dodir = True) 1544 1545 lock_file = self.get_database_lockfile(repo) 1546 lock_filename = os.path.basename(lock_file) 1547 if not os.path.isfile(lock_file) and \ 1548 ftp.is_file_available(lock_filename): 1549 1550 self.Entropy.updateProgress( 1551 "[repo:%s|%s|%s] %s, %s" % ( 1552 brown(str(repo)), 1553 darkgreen(crippled_uri), 1554 red(_("locking")), 1555 darkblue(_("mirror already locked")), 1556 blue(_("waiting up to 2 minutes before giving up")), 1557 ), 1558 importance = 1, 1559 type = "warning", 1560 header = brown(" * "), 1561 back = True 1562 ) 1563 unlocked = False 1564 count = 0 1565 while count < 120: 1566 count += 1 1567 time.sleep(1) 1568 if not ftp.is_file_available(os.path.basename(lock_file)): 1569 self.Entropy.updateProgress( 1570 red("[repo:%s|%s|%s] %s !" % ( 1571 repo, 1572 crippled_uri, 1573 _("locking"), 1574 _("mirror unlocked"), 1575 ) 1576 ), 1577 importance = 1, 1578 type = "info", 1579 header = darkgreen(" * ") 1580 ) 1581 unlocked = True 1582 break 1583 if not unlocked: 1584 gave_up = True 1585 1586 ftp.close() 1587 return gave_up

1588

1589 - def shrink_database_and_close(self, repo = None):

1590 dbconn = self.Entropy.open_server_repository(read_only = False, 1591 no_upload = True, repo = repo, indexing = False, 1592 do_treeupdates = False) 1593 dbconn.dropAllIndexes() 1594 dbconn.vacuum() 1595 dbconn.vacuum() 1596 dbconn.commitChanges() 1597 self.Entropy.close_server_database(dbconn)

1598

1599 - def update_repository_timestamp(self, repo = None):

1600 if repo == None: 1601 repo = self.Entropy.default_repository 1602 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1603 current_ts = self.Entropy.get_current_timestamp() 1604 ts_f = open(ts_file,"w") 1605 ts_f.write(current_ts) 1606 ts_f.flush() 1607 ts_f.close()

1608

1609 - def sync_database_treeupdates(self, repo = None):

1610 1611 if repo == None: 1612 repo = self.Entropy.default_repository 1613 dbconn = self.Entropy.open_server_repository(read_only = False, 1614 no_upload = True, repo = repo, do_treeupdates = False) 1615 # grab treeupdates from other databases and inject 1616 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1617 server_repos = srv_set['repositories'].keys() 1618 all_actions = set() 1619 for myrepo in server_repos: 1620 1621 # avoid __default__ 1622 if myrepo == etpConst['clientserverrepoid']: 1623 continue 1624 1625 mydbc = self.Entropy.open_server_repository(just_reading = True, 1626 repo = myrepo) 1627 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1628 for data in actions: 1629 all_actions.add(data) 1630 if not actions: 1631 continue 1632 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1633 try: 1634 # clear first 1635 dbconn.removeTreeUpdatesActions(repo) 1636 dbconn.insertTreeUpdatesActions(all_actions, repo) 1637 except dbapi2.Error, err: 1638 self.entropyTools.print_traceback() 1639 mytxt = "%s, %s: %s. %s" % ( 1640 _("Troubles with treeupdates"), 1641 _("error"), 1642 err, 1643 _("Bumping old data back"), 1644 ) 1645 self.Entropy.updateProgress( 1646 mytxt, 1647 importance = 1, 1648 type = "warning" 1649 ) 1650 # restore previous data 1651 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1652 1653 dbconn.commitChanges() 1654 self.Entropy.close_server_database(dbconn)

1655

1656 - def upload_database(self, uris, lock_check = False, pretend = False, 1657 repo = None):

1658 1659 if repo == None: 1660 repo = self.Entropy.default_repository 1661 1662 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1663 if srv_set['rss']['enabled']: 1664 self.update_rss_feed(repo = repo) 1665 1666 upload_errors = False 1667 broken_uris = set() 1668 fine_uris = set() 1669 1670 disabled_eapis = sorted(srv_set['disabled_eapis']) 1671 1672 for uri in uris: 1673 1674 db_format = srv_set['database_file_format'] 1675 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1676 if cmethod == None: 1677 raise InvalidDataType("InvalidDataType: %s." % ( 1678 _("wrong database compression method passed"), 1679 ) 1680 ) 1681 1682 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1683 database_path = self.Entropy.get_local_database_file(repo) 1684 1685 if disabled_eapis: 1686 self.Entropy.updateProgress( 1687 "[repo:%s|%s|%s] %s: %s" % ( 1688 blue(repo), 1689 red(crippled_uri), 1690 darkgreen(_("upload")), 1691 darkred(_("disabled EAPI")), 1692 bold(', '.join(disabled_eapis)), 1693 ), 1694 importance = 1, 1695 type = "warning", 1696 header = darkgreen(" * ") 1697 ) 1698 1699 # create/update timestamp file 1700 self.update_repository_timestamp(repo) 1701 # create pkglist service file 1702 self.Entropy.create_repository_pkglist(repo) 1703 1704 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1705 repo = repo, disabled_eapis = disabled_eapis) 1706 1707 if lock_check: 1708 given_up = self.mirror_lock_check(uri, repo = repo) 1709 if given_up: 1710 upload_errors = True 1711 broken_uris.add(uri) 1712 continue 1713 1714 self.lock_mirrors_for_download(True, [uri], repo = repo) 1715 1716 self.Entropy.updateProgress( 1717 "[repo:%s|%s|%s] %s" % ( 1718 blue(repo), 1719 red(crippled_uri), 1720 darkgreen(_("upload")), 1721 darkgreen(_("preparing to upload database to mirror")), 1722 ), 1723 importance = 1, 1724 type = "info", 1725 header = darkgreen(" * ") 1726 ) 1727 1728 # Package Sets info 1729 self._show_package_sets_messages(repo) 1730 self.sync_database_treeupdates(repo) 1731 self.Entropy.update_database_package_sets(repo) 1732 self.Entropy.close_server_databases() 1733 1734 # backup current database to avoid re-indexing 1735 old_dbpath = self.Entropy.get_local_database_file(repo) 1736 backup_dbpath = old_dbpath + ".up_backup" 1737 copy_back = False 1738 if not pretend: 1739 try: 1740 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1741 os.remove(backup_dbpath) 1742 shutil.copy2(old_dbpath, backup_dbpath) 1743 copy_back = True 1744 except shutil.Error: 1745 copy_back = False 1746 1747 self.shrink_database_and_close(repo) 1748 1749 # EAPI 3 1750 if 3 not in disabled_eapis: 1751 self._create_metafiles_file(upload_data['metafiles_path'], 1752 text_files, repo) 1753 self._show_eapi3_upload_messages(crippled_uri, database_path, 1754 repo) 1755 1756 # EAPI 2 1757 if 2 not in disabled_eapis: 1758 self._show_eapi2_upload_messages(crippled_uri, database_path, 1759 upload_data, cmethod, repo) 1760 1761 # create compressed dump + checksum 1762 self.dump_database_to_file(database_path, 1763 upload_data['dump_path_light'], cmethod[0], 1764 exclude_tables = ["content"], repo = repo) 1765 self.create_file_checksum(upload_data['dump_path_light'], 1766 upload_data['dump_path_digest_light']) 1767 1768 1769 # EAPI 1 1770 if 1 not in disabled_eapis: 1771 self._show_eapi1_upload_messages(crippled_uri, database_path, 1772 upload_data, cmethod, repo) 1773 1774 # compress the database and create uncompressed 1775 # database checksum -- DEPRECATED 1776 self.compress_file(database_path, 1777 upload_data['compressed_database_path'], cmethod[0]) 1778 self.create_file_checksum(database_path, 1779 upload_data['database_path_digest']) 1780 1781 # create compressed database checksum 1782 self.create_file_checksum( 1783 upload_data['compressed_database_path'], 1784 upload_data['compressed_database_path_digest']) 1785 1786 # create light version of the compressed db 1787 eapi1_dbfile = self.Entropy.get_local_database_file(repo) 1788 temp_eapi1_dbfile = eapi1_dbfile+".light" 1789 shutil.copy2(eapi1_dbfile, temp_eapi1_dbfile) 1790 # open and remove content table 1791 eapi1_tmp_dbconn = \ 1792 self.Entropy.ClientService.open_generic_database( 1793 temp_eapi1_dbfile, indexing_override = False, 1794 xcache = False) 1795 eapi1_tmp_dbconn.dropContent() 1796 eapi1_tmp_dbconn.commitChanges() 1797 eapi1_tmp_dbconn.vacuum() 1798 eapi1_tmp_dbconn.closeDB() 1799 1800 # compress 1801 self.compress_file(temp_eapi1_dbfile, 1802 upload_data['compressed_database_path_light'], cmethod[0]) 1803 # go away, we don't need you anymore 1804 os.remove(temp_eapi1_dbfile) 1805 # create compressed light database checksum 1806 self.create_file_checksum( 1807 upload_data['compressed_database_path_light'], 1808 upload_data['compressed_database_path_digest_light']) 1809 1810 if not pretend: 1811 # upload 1812 uploader = self.FtpServerHandler( 1813 FtpInterface, 1814 self.Entropy, 1815 [uri], 1816 [upload_data[x] for x in upload_data], 1817 critical_files = critical, 1818 repo = repo 1819 ) 1820 errors, m_fine_uris, m_broken_uris = uploader.go() 1821 if errors: 1822 my_broken_uris = sorted([ 1823 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1824 x[1]) for x in m_broken_uris]) 1825 self.Entropy.updateProgress( 1826 "[repo:%s|%s|%s] %s" % ( 1827 repo, 1828 crippled_uri, 1829 _("errors"), 1830 _("upload failed, not unlocking and continuing"), 1831 ), 1832 importance = 0, 1833 type = "error", 1834 header = darkred(" !!! ") 1835 ) 1836 # get reason 1837 reason = my_broken_uris[0][1] 1838 self.Entropy.updateProgress( 1839 blue("%s: %s" % (_("reason"), reason,)), 1840 importance = 0, 1841 type = "error", 1842 header = blue(" # ") 1843 ) 1844 upload_errors = True 1845 broken_uris |= m_broken_uris 1846 continue 1847 1848 # copy db back 1849 if copy_back and os.path.isfile(backup_dbpath): 1850 self.Entropy.close_server_databases() 1851 further_backup_dbpath = old_dbpath+".security_backup" 1852 if os.path.isfile(further_backup_dbpath): 1853 os.remove(further_backup_dbpath) 1854 shutil.copy2(old_dbpath, further_backup_dbpath) 1855 shutil.move(backup_dbpath, old_dbpath) 1856 1857 # unlock 1858 self.lock_mirrors_for_download(False, [uri], repo = repo) 1859 fine_uris |= m_fine_uris 1860 1861 if not fine_uris: 1862 upload_errors = True 1863 return upload_errors, broken_uris, fine_uris

1864 1865

1866 - def download_database(self, uris, lock_check = False, pretend = False, 1867 repo = None):

1868 1869 if repo == None: 1870 repo = self.Entropy.default_repository 1871 1872 download_errors = False 1873 broken_uris = set() 1874 fine_uris = set() 1875 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1876 disabled_eapis = sorted(srv_set['disabled_eapis']) 1877 1878 for uri in uris: 1879 1880 db_format = srv_set['database_file_format'] 1881 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1882 if cmethod == None: 1883 raise InvalidDataType("InvalidDataType: %s." % ( 1884 _("wrong database compression method passed"), 1885 ) 1886 ) 1887 1888 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1889 database_path = self.Entropy.get_local_database_file(repo) 1890 database_dir_path = os.path.dirname( 1891 self.Entropy.get_local_database_file(repo)) 1892 download_data, critical, text_files = self.get_files_to_sync( 1893 cmethod, download = True, 1894 repo = repo, disabled_eapis = disabled_eapis) 1895 mytmpdir = self.entropyTools.get_random_temp_file() 1896 os.makedirs(mytmpdir) 1897 1898 self.Entropy.updateProgress( 1899 "[repo:%s|%s|%s] %s" % ( 1900 brown(repo), 1901 darkgreen(crippled_uri), 1902 red(_("download")), 1903 blue(_("preparing to download database from mirror")), 1904 ), 1905 importance = 1, 1906 type = "info", 1907 header = darkgreen(" * ") 1908 ) 1909 files_to_sync = sorted(download_data.keys()) 1910 for myfile in files_to_sync: 1911 self.Entropy.updateProgress( 1912 "%s: %s" % ( 1913 blue(_("download path")), 1914 brown(unicode(download_data[myfile])), 1915 ), 1916 importance = 0, 1917 type = "info", 1918 header = brown(" # ") 1919 ) 1920 1921 if lock_check: 1922 given_up = self.mirror_lock_check(uri, repo = repo) 1923 if given_up: 1924 download_errors = True 1925 broken_uris.add(uri) 1926 continue 1927 1928 # avoid having others messing while we're downloading 1929 self.lock_mirrors(True, [uri], repo = repo) 1930 1931 if not pretend: 1932 # download 1933 downloader = self.FtpServerHandler( 1934 FtpInterface, self.Entropy, [uri], 1935 [download_data[x] for x in download_data], download = True, 1936 local_basedir = mytmpdir, critical_files = critical, 1937 repo = repo 1938 ) 1939 errors, m_fine_uris, m_broken_uris = downloader.go() 1940 if errors: 1941 my_broken_uris = sorted([ 1942 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1943 x[1]) for x in m_broken_uris]) 1944 self.Entropy.updateProgress( 1945 "[repo:%s|%s|%s] %s" % ( 1946 brown(repo), 1947 darkgreen(crippled_uri), 1948 red(_("errors")), 1949 blue(_("failed to download from mirror")), 1950 ), 1951 importance = 0, 1952 type = "error", 1953 header = darkred(" !!! ") 1954 ) 1955 # get reason 1956 reason = my_broken_uris[0][1] 1957 self.Entropy.updateProgress( 1958 blue("%s: %s" % (_("reason"), reason,)), 1959 importance = 0, 1960 type = "error", 1961 header = blue(" # ") 1962 ) 1963 download_errors = True 1964 broken_uris |= m_broken_uris 1965 self.lock_mirrors(False, [uri], repo = repo) 1966 continue 1967 1968 # all fine then, we need to move data from mytmpdir 1969 # to database_dir_path 1970 1971 # EAPI 1 -- unpack database 1972 if 1 not in disabled_eapis: 1973 compressed_db_filename = os.path.basename( 1974 download_data['compressed_database_path']) 1975 uncompressed_db_filename = os.path.basename(database_path) 1976 compressed_file = os.path.join(mytmpdir, 1977 compressed_db_filename) 1978 uncompressed_file = os.path.join(mytmpdir, 1979 uncompressed_db_filename) 1980 self.entropyTools.uncompress_file(compressed_file, 1981 uncompressed_file, cmethod[0]) 1982 1983 # now move 1984 for myfile in os.listdir(mytmpdir): 1985 fromfile = os.path.join(mytmpdir, myfile) 1986 tofile = os.path.join(database_dir_path, myfile) 1987 shutil.move(fromfile, tofile) 1988 self.Entropy.ClientService.setup_default_file_perms(tofile) 1989 1990 if os.path.isdir(mytmpdir): 1991 shutil.rmtree(mytmpdir) 1992 if os.path.isdir(mytmpdir): 1993 os.rmdir(mytmpdir) 1994 1995 1996 fine_uris.add(uri) 1997 self.lock_mirrors(False, [uri], repo = repo) 1998 1999 return download_errors, fine_uris, broken_uris

2000

2001 - def calculate_database_sync_queues(self, repo = None):

2002 2003 if repo == None: 2004 repo = self.Entropy.default_repository 2005 2006 remote_status = self.get_remote_databases_status(repo) 2007 local_revision = self.Entropy.get_local_database_revision(repo) 2008 upload_queue = [] 2009 download_latest = () 2010 2011 # all mirrors are empty ? I rule 2012 if not [x for x in remote_status if x[1]]: 2013 upload_queue = remote_status[:] 2014 else: 2015 highest_remote_revision = max([x[1] for x in remote_status]) 2016 2017 if local_revision < highest_remote_revision: 2018 for remote_item in remote_status: 2019 if remote_item[1] == highest_remote_revision: 2020 download_latest = x 2021 break 2022 2023 if download_latest: 2024 upload_queue = [x for x in remote_status if \ 2025 (x[1] < highest_remote_revision)] 2026 else: 2027 upload_queue = [x for x in remote_status if \ 2028 (x[1] < local_revision)] 2029 2030 return download_latest, upload_queue

2031

2032 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 2033 repo = None):

2034 2035 if repo == None: 2036 repo = self.Entropy.default_repository 2037 2038 while 1: 2039 2040 db_locked = False 2041 if self.is_local_database_locked(repo): 2042 db_locked = True 2043 2044 lock_data = self.get_mirrors_lock(repo) 2045 mirrors_locked = [x for x in lock_data if x[1]] 2046 2047 if not mirrors_locked and db_locked: 2048 # mirrors not locked remotely but only locally 2049 mylock_file = self.get_database_lockfile(repo) 2050 if os.access(mylock_file, os.W_OK | os.F_OK): 2051 os.remove(mylock_file) 2052 continue 2053 2054 break 2055 2056 if mirrors_locked and not db_locked: 2057 mytxt = "%s, %s %s" % ( 2058 _("Mirrors are locked, someone is working on the repository"), 2059 _("try again later"), 2060 "...", 2061 ) 2062 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 2063 2064 download_latest, upload_queue = self.calculate_database_sync_queues( 2065 repo) 2066 2067 if not download_latest and not upload_queue: 2068 self.Entropy.updateProgress( 2069 "[repo:%s|%s] %s" % ( 2070 brown(repo), 2071 red(_("sync")), # something short please 2072 blue(_("database already in sync")), 2073 ), 2074 importance = 1, 2075 type = "info", 2076 header = blue(" @@ ") 2077 ) 2078 return 0, set(), set() 2079 2080 if download_latest: 2081 download_uri = download_latest[0] 2082 download_errors, fine_uris, broken_uris = self.download_database( 2083 [download_uri], repo = repo) 2084 if download_errors: 2085 self.Entropy.updateProgress( 2086 "[repo:%s|%s] %s: %s" % ( 2087 brown(repo), 2088 red(_("sync")), 2089 blue(_("database sync failed")), 2090 red(_("download issues")), 2091 ), 2092 importance = 1, 2093 type = "error", 2094 header = darkred(" !!! ") 2095 ) 2096 return 1, fine_uris, broken_uris 2097 2098 if upload_queue and not no_upload: 2099 2100 # XXX QA checks, 2101 # please group them into entropy.qa 2102 deps_not_found = self.Entropy.dependencies_test(repo = repo) 2103 if deps_not_found and not self.Entropy.community_repo: 2104 self.Entropy.updateProgress( 2105 "[repo:%s|%s] %s: %s" % ( 2106 brown(repo), 2107 red(_("sync")), 2108 blue(_("database sync forbidden")), 2109 red(_("dependencies_test() reported errors")), 2110 ), 2111 importance = 1, 2112 type = "error", 2113 header = darkred(" !!! ") 2114 ) 2115 return 3, set(), set() 2116 2117 problems = self.Entropy.check_config_file_updates() 2118 if problems: 2119 return 4, set(), set() 2120 2121 self.Entropy.updateProgress( 2122 "[repo:%s|%s] %s" % ( 2123 brown(repo), 2124 red(_("config files")), # something short please 2125 blue(_("no configuration files to commit. All fine.")), 2126 ), 2127 importance = 1, 2128 type = "info", 2129 header = blue(" @@ "), 2130 back = True 2131 ) 2132 2133 uris = [x[0] for x in upload_queue] 2134 errors, fine_uris, broken_uris = self.upload_database(uris, 2135 repo = repo) 2136 if errors: 2137 self.Entropy.updateProgress( 2138 "[repo:%s|%s] %s: %s" % ( 2139 brown(repo), 2140 red(_("sync")), 2141 blue(_("database sync failed")), 2142 red(_("upload issues")), 2143 ), 2144 importance = 1, 2145 type = "error", 2146 header = darkred(" !!! ") 2147 ) 2148 return 2, fine_uris, broken_uris 2149 2150 2151 self.Entropy.updateProgress( 2152 "[repo:%s|%s] %s" % ( 2153 brown(repo), 2154 red(_("sync")), 2155 blue(_("database sync completed successfully")), 2156 ), 2157 importance = 1, 2158 type = "info", 2159 header = darkgreen(" * ") 2160 ) 2161 2162 if unlock_mirrors: 2163 self.lock_mirrors(False, repo = repo) 2164 return 0, set(), set()

2165 2166

2167 - def calculate_local_upload_files(self, branch, repo = None):

2168 upload_files = 0 2169 upload_packages = set() 2170 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 2171 branch) 2172 2173 # check if it exists 2174 if not os.path.isdir(upload_dir): 2175 return upload_files, upload_packages 2176 2177 pkg_ext = etpConst['packagesext'] 2178 pkg_md5_ext = etpConst['packagesmd5fileext'] 2179 for package in os.listdir(upload_dir): 2180 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2181 upload_packages.add(package) 2182 if package.endswith(pkg_ext): 2183 upload_files += 1 2184 2185 return upload_files, upload_packages

2186

2187 - def calculate_local_package_files(self, branch, repo = None):

2188 local_files = 0 2189 local_packages = set() 2190 packages_dir = os.path.join( 2191 self.Entropy.get_local_packages_directory(repo), branch) 2192 2193 if not os.path.isdir(packages_dir): 2194 os.makedirs(packages_dir) 2195 2196 pkg_ext = etpConst['packagesext'] 2197 pkg_md5_ext = etpConst['packagesmd5fileext'] 2198 for package in os.listdir(packages_dir): 2199 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 2200 local_packages.add(package) 2201 if package.endswith(pkg_ext): 2202 local_files += 1 2203 2204 return local_files, local_packages

2205 2206

2207 - def _show_local_sync_stats(self, upload_files, local_files):

2208 self.Entropy.updateProgress( 2209 "%s:" % ( 2210 blue(_("Local statistics")), 2211 ), 2212 importance = 1, 2213 type = "info", 2214 header = red(" @@ ") 2215 ) 2216 self.Entropy.updateProgress( 2217 red("%s:\t\t%s %s" % ( 2218 blue(_("upload directory")), 2219 bold(str(upload_files)), 2220 red(_("files ready")), 2221 ) 2222 ), 2223 importance = 0, 2224 type = "info", 2225 header = red(" @@ ") 2226 ) 2227 self.Entropy.updateProgress( 2228 red("%s:\t\t%s %s" % ( 2229 blue(_("packages directory")), 2230 bold(str(local_files)), 2231 red(_("files ready")), 2232 ) 2233 ), 2234 importance = 0, 2235 type = "info", 2236 header = red(" @@ ") 2237 )

2238

2239 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2240 branch):

2241 2242 # show stats 2243 for package, size in upload: 2244 package = darkgreen(os.path.basename(package)) 2245 size = blue(self.entropyTools.bytes_into_human(size)) 2246 self.Entropy.updateProgress( 2247 "[branch:%s|%s] %s [%s]" % ( 2248 brown(branch), 2249 blue(_("upload")), 2250 darkgreen(package), 2251 size, 2252 ), 2253 importance = 0, 2254 type = "info", 2255 header = red(" # ") 2256 ) 2257 for package, size in download: 2258 package = darkred(os.path.basename(package)) 2259 size = blue(self.entropyTools.bytes_into_human(size)) 2260 self.Entropy.updateProgress( 2261 "[branch:%s|%s] %s [%s]" % ( 2262 brown(branch), 2263 darkred(_("download")), 2264 blue(package), 2265 size, 2266 ), 2267 importance = 0, 2268 type = "info", 2269 header = red(" # ") 2270 ) 2271 for package, size in copy: 2272 package = darkblue(os.path.basename(package)) 2273 size = blue(self.entropyTools.bytes_into_human(size)) 2274 self.Entropy.updateProgress( 2275 "[branch:%s|%s] %s [%s]" % ( 2276 brown(branch), 2277 darkgreen(_("copy")), 2278 brown(package), 2279 size, 2280 ), 2281 importance = 0, 2282 type = "info", 2283 header = red(" # ") 2284 ) 2285 for package, size in removal: 2286 package = brown(os.path.basename(package)) 2287 size = blue(self.entropyTools.bytes_into_human(size)) 2288 self.Entropy.updateProgress( 2289 "[branch:%s|%s] %s [%s]" % ( 2290 brown(branch), 2291 red(_("remove")), 2292 red(package), 2293 size, 2294 ), 2295 importance = 0, 2296 type = "info", 2297 header = red(" # ") 2298 ) 2299 2300 self.Entropy.updateProgress( 2301 "%s:\t\t\t%s" % ( 2302 blue(_("Packages to be removed")), 2303 darkred(str(len(removal))), 2304 ), 2305 importance = 0, 2306 type = "info", 2307 header = blue(" @@ ") 2308 ) 2309 self.Entropy.updateProgress( 2310 "%s:\t\t%s" % ( 2311 darkgreen(_("Packages to be moved locally")), 2312 darkgreen(str(len(copy))), 2313 ), 2314 importance = 0, 2315 type = "info", 2316 header = blue(" @@ ") 2317 ) 2318 self.Entropy.updateProgress( 2319 "%s:\t\t\t%s" % ( 2320 bold(_("Packages to be uploaded")), 2321 bold(str(len(upload))), 2322 ), 2323 importance = 0, 2324 type = "info", 2325 header = blue(" @@ ") 2326 ) 2327 2328 self.Entropy.updateProgress( 2329 "%s:\t\t\t%s" % ( 2330 darkred(_("Total removal size")), 2331 darkred( 2332 self.entropyTools.bytes_into_human(metainfo['removal']) 2333 ), 2334 ), 2335 importance = 0, 2336 type = "info", 2337 header = blue(" @@ ") 2338 ) 2339 2340 self.Entropy.updateProgress( 2341 "%s:\t\t\t%s" % ( 2342 blue(_("Total upload size")), 2343 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2344 ), 2345 importance = 0, 2346 type = "info", 2347 header = blue(" @@ ") 2348 ) 2349 self.Entropy.updateProgress( 2350 "%s:\t\t\t%s" % ( 2351 brown(_("Total download size")), 2352 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2353 ), 2354 importance = 0, 2355 type = "info", 2356 header = blue(" @@ ") 2357 )

2358 2359

2360 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2361 repo = None):

2362 2363 remote_files = 0 2364 close_conn = False 2365 remote_packages_data = {} 2366 2367 my_path = os.path.join( 2368 self.Entropy.get_remote_packages_relative_path(repo), branch) 2369 if ftp_connection == None: 2370 close_conn = True 2371 # let it raise an exception if things go bad 2372 ftp_connection = FtpInterface(uri, self.Entropy) 2373 ftp_connection.set_cwd(my_path, dodir = True) 2374 2375 remote_packages = ftp_connection.list_dir() 2376 remote_packages_info = ftp_connection.get_raw_list() 2377 if close_conn: 2378 ftp_connection.close() 2379 2380 for tbz2 in remote_packages: 2381 if tbz2.endswith(etpConst['packagesext']): 2382 remote_files += 1 2383 2384 for remote_package in remote_packages_info: 2385 remote_packages_data[remote_package.split()[8]] = \ 2386 int(remote_package.split()[4]) 2387 2388 return remote_files, remote_packages, remote_packages_data

2389

2390 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2391 2392 if repo == None: 2393 repo = self.Entropy.default_repository 2394 2395 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2396 upload_files, upload_packages = self.calculate_local_upload_files( 2397 branch, repo) 2398 local_files, local_packages = self.calculate_local_package_files(branch, 2399 repo) 2400 self._show_local_sync_stats(upload_files, local_files) 2401 2402 self.Entropy.updateProgress( 2403 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2404 importance = 1, 2405 type = "info", 2406 header = red(" @@ ") 2407 ) 2408 remote_files, remote_packages, remote_packages_data = \ 2409 self.calculate_remote_package_files(uri, branch, repo = repo) 2410 self.Entropy.updateProgress( 2411 "%s:\t\t\t%s %s" % ( 2412 blue(_("remote packages")), 2413 bold(str(remote_files)), 2414 red(_("files stored")), 2415 ), 2416 importance = 0, 2417 type = "info", 2418 header = red(" @@ ") 2419 ) 2420 2421 mytxt = blue("%s ...") % ( 2422 _("Calculating queues"), 2423 ) 2424 self.Entropy.updateProgress( 2425 mytxt, 2426 importance = 1, 2427 type = "info", 2428 header = red(" @@ ") 2429 ) 2430 2431 upload_queue, download_queue, removal_queue, fine_queue = \ 2432 self.calculate_sync_queues(upload_packages, local_packages, 2433 remote_packages, remote_packages_data, branch, repo) 2434 return upload_queue, download_queue, removal_queue, fine_queue, \ 2435 remote_packages_data

2436

2437 - def calculate_sync_queues( 2438 self, 2439 upload_packages, 2440 local_packages, 2441 remote_packages, 2442 remote_packages_data, 2443 branch, 2444 repo = None 2445 ):

2446 2447 upload_queue = set() 2448 download_queue = set() 2449 removal_queue = set() 2450 fine_queue = set() 2451 2452 for local_package in upload_packages: 2453 if local_package in remote_packages: 2454 2455 local_filepath = os.path.join( 2456 self.Entropy.get_local_upload_directory(repo), branch, 2457 local_package) 2458 2459 local_size = self.entropyTools.get_file_size(local_filepath) 2460 remote_size = remote_packages_data.get(local_package) 2461 if remote_size == None: 2462 remote_size = 0 2463 if (local_size != remote_size): 2464 # size does not match, adding to the upload queue 2465 upload_queue.add(local_package) 2466 else: 2467 # just move from upload to packages 2468 fine_queue.add(local_package) 2469 else: 2470 # always force upload of packages in uploaddir 2471 upload_queue.add(local_package) 2472 2473 # if a package is in the packages directory but not online, 2474 # we have to upload it we have local_packages and remote_packages 2475 for local_package in local_packages: 2476 if local_package in remote_packages: 2477 local_filepath = os.path.join( 2478 self.Entropy.get_local_packages_directory(repo), branch, 2479 local_package) 2480 local_size = self.entropyTools.get_file_size(local_filepath) 2481 remote_size = remote_packages_data.get(local_package) 2482 if remote_size == None: 2483 remote_size = 0 2484 if (local_size != remote_size) and (local_size != 0): 2485 # size does not match, adding to the upload queue 2486 if local_package not in fine_queue: 2487 upload_queue.add(local_package) 2488 else: 2489 # this means that the local package does not exist 2490 # so, we need to download it 2491 upload_queue.add(local_package) 2492 2493 # Fill download_queue and removal_queue 2494 for remote_package in remote_packages: 2495 if remote_package in local_packages: 2496 local_filepath = os.path.join( 2497 self.Entropy.get_local_packages_directory(repo), branch, 2498 remote_package) 2499 local_size = self.entropyTools.get_file_size(local_filepath) 2500 remote_size = remote_packages_data.get(remote_package) 2501 if remote_size == None: 2502 remote_size = 0 2503 if (local_size != remote_size) and (local_size != 0): 2504 # size does not match, remove first 2505 # do it only if the package has not been 2506 # added to the upload_queue 2507 if remote_package not in upload_queue: 2508 # remotePackage == localPackage 2509 # just remove something that differs 2510 # from the content of the mirror 2511 removal_queue.add(remote_package) 2512 # then add to the download queue 2513 download_queue.add(remote_package) 2514 else: 2515 # this means that the local package does not exist 2516 # so, we need to download it 2517 # ignore .tmp files 2518 if not remote_package.endswith(".tmp"): 2519 download_queue.add(remote_package) 2520 2521 # Collect packages that don't exist anymore in the database 2522 # so we can filter them out from the download queue 2523 dbconn = self.Entropy.open_server_repository(just_reading = True, 2524 repo = repo) 2525 db_files = dbconn.listAllDownloads(do_sort = False, 2526 full_path = True) 2527 db_files = set([os.path.basename(x) for x in db_files if \ 2528 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2529 2530 exclude = set() 2531 for myfile in download_queue: 2532 if myfile.endswith(etpConst['packagesext']): 2533 if myfile not in db_files: 2534 exclude.add(myfile) 2535 download_queue -= exclude 2536 2537 exclude = set() 2538 for myfile in upload_queue: 2539 if myfile.endswith(etpConst['packagesext']): 2540 if myfile not in db_files: 2541 exclude.add(myfile) 2542 upload_queue -= exclude 2543 2544 exclude = set() 2545 for myfile in download_queue: 2546 if myfile in upload_queue: 2547 exclude.add(myfile) 2548 download_queue -= exclude 2549 2550 return upload_queue, download_queue, removal_queue, fine_queue

2551 2552

2553 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2554 remote_packages_data, branch, repo):

2555 2556 metainfo = { 2557 'removal': 0, 2558 'download': 0, 2559 'upload': 0, 2560 } 2561 removal = [] 2562 download = [] 2563 do_copy = [] 2564 upload = [] 2565 2566 for item in removal_queue: 2567 if not item.endswith(etpConst['packagesext']): 2568 continue 2569 local_filepath = os.path.join( 2570 self.Entropy.get_local_packages_directory(repo), branch, item) 2571 size = self.entropyTools.get_file_size(local_filepath) 2572 metainfo['removal'] += size 2573 removal.append((local_filepath, size)) 2574 2575 for item in download_queue: 2576 if not item.endswith(etpConst['packagesext']): 2577 continue 2578 local_filepath = os.path.join( 2579 self.Entropy.get_local_upload_directory(repo), branch, item) 2580 if not os.path.isfile(local_filepath): 2581 size = remote_packages_data.get(item) 2582 if size == None: 2583 size = 0 2584 size = int(size) 2585 metainfo['removal'] += size 2586 download.append((local_filepath, size)) 2587 else: 2588 size = self.entropyTools.get_file_size(local_filepath) 2589 do_copy.append((local_filepath, size)) 2590 2591 for item in upload_queue: 2592 if not item.endswith(etpConst['packagesext']): 2593 continue 2594 local_filepath = os.path.join( 2595 self.Entropy.get_local_upload_directory(repo), branch, item) 2596 local_filepath_pkgs = os.path.join( 2597 self.Entropy.get_local_packages_directory(repo), branch, item) 2598 if os.path.isfile(local_filepath): 2599 size = self.entropyTools.get_file_size(local_filepath) 2600 upload.append((local_filepath, size)) 2601 else: 2602 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2603 upload.append((local_filepath_pkgs, size)) 2604 metainfo['upload'] += size 2605 2606 return upload, download, removal, do_copy, metainfo

2607 2608

2609 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2610 2611 if repo == None: 2612 repo = self.Entropy.default_repository 2613 2614 for itemdata in removal_queue: 2615 2616 remove_filename = itemdata[0] 2617 remove_filepath = os.path.join( 2618 self.Entropy.get_local_packages_directory(repo), branch, 2619 remove_filename) 2620 remove_filepath_hash = remove_filepath + \ 2621 etpConst['packagesmd5fileext'] 2622 self.Entropy.updateProgress( 2623 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2624 brown(repo), 2625 red("sync"), 2626 brown(branch), 2627 blue(_("removing package+hash")), 2628 darkgreen(remove_filename), 2629 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2630 ), 2631 importance = 0, 2632 type = "info", 2633 header = darkred(" * ") 2634 ) 2635 2636 if os.path.isfile(remove_filepath): 2637 os.remove(remove_filepath) 2638 if os.path.isfile(remove_filepath_hash): 2639 os.remove(remove_filepath_hash) 2640 2641 self.Entropy.updateProgress( 2642 "[repo:%s|%s|%s] %s" % ( 2643 brown(repo), 2644 red(_("sync")), 2645 brown(branch), 2646 blue(_("removal complete")), 2647 ), 2648 importance = 0, 2649 type = "info", 2650 header = darkred(" * ") 2651 )

2652 2653

2654 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2655 2656 if repo == None: 2657 repo = self.Entropy.default_repository 2658 2659 for itemdata in copy_queue: 2660 2661 from_file = itemdata[0] 2662 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2663 to_file = os.path.join( 2664 self.Entropy.get_local_packages_directory(repo), branch, 2665 os.path.basename(from_file)) 2666 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2667 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2668 self.Entropy.updateProgress( 2669 "[repo:%s|%s|%s] %s: %s" % ( 2670 brown(repo), 2671 red("sync"), 2672 brown(branch), 2673 blue(_("copying file+hash to repository")), 2674 darkgreen(from_file), 2675 ), 2676 importance = 0, 2677 type = "info", 2678 header = darkred(" * ") 2679 ) 2680 2681 if not os.path.isdir(os.path.dirname(to_file)): 2682 os.makedirs(os.path.dirname(to_file)) 2683 2684 shutil.copy2(from_file, to_file) 2685 if not os.path.isfile(from_file_hash): 2686 self.create_file_checksum(from_file, from_file_hash) 2687 shutil.copy2(from_file_hash, to_file_hash) 2688 2689 # clear expiration file 2690 if os.path.isfile(expiration_file): 2691 os.remove(expiration_file)

2692 2693

2694 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2695 2696 if repo == None: 2697 repo = self.Entropy.default_repository 2698 2699 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2700 myqueue = [] 2701 for itemdata in upload_queue: 2702 upload_item = itemdata[0] 2703 hash_file = upload_item + etpConst['packagesmd5fileext'] 2704 if not os.path.isfile(hash_file): 2705 self.entropyTools.create_md5_file(upload_item) 2706 myqueue.append(hash_file) 2707 myqueue.append(upload_item) 2708 2709 ftp_basedir = os.path.join( 2710 self.Entropy.get_remote_packages_relative_path(repo), branch) 2711 uploader = self.FtpServerHandler(FtpInterface, 2712 self.Entropy, [uri], myqueue, critical_files = myqueue, 2713 use_handlers = True, ftp_basedir = ftp_basedir, 2714 handlers_data = {'branch': branch }, repo = repo) 2715 errors, m_fine_uris, m_broken_uris = uploader.go() 2716 if errors: 2717 my_broken_uris = [ 2718 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2719 x in m_broken_uris] 2720 reason = my_broken_uris[0][1] 2721 self.Entropy.updateProgress( 2722 "[branch:%s] %s: %s, %s: %s" % ( 2723 brown(branch), 2724 blue(_("upload errors")), 2725 red(crippled_uri), 2726 blue(_("reason")), 2727 darkgreen(unicode(reason)), 2728 ), 2729 importance = 1, 2730 type = "error", 2731 header = darkred(" !!! ") 2732 ) 2733 return errors, m_fine_uris, m_broken_uris 2734 2735 self.Entropy.updateProgress( 2736 "[branch:%s] %s: %s" % ( 2737 brown(branch), 2738 blue(_("upload completed successfully")), 2739 red(crippled_uri), 2740 ), 2741 importance = 1, 2742 type = "info", 2743 header = blue(" @@ ") 2744 ) 2745 return errors, m_fine_uris, m_broken_uris

2746 2747

2748 - def _sync_run_download_queue(self, uri, download_queue, branch, 2749 repo = None):

2750 2751 if repo == None: 2752 repo = self.Entropy.default_repository 2753 2754 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2755 myqueue = [] 2756 for package in download_queue: 2757 hash_file = package + etpConst['packagesmd5fileext'] 2758 myqueue.append(package) 2759 myqueue.append(hash_file) 2760 2761 ftp_basedir = os.path.join( 2762 self.Entropy.get_remote_packages_relative_path(repo), branch) 2763 local_basedir = os.path.join( 2764 self.Entropy.get_local_packages_directory(repo), branch) 2765 downloader = self.FtpServerHandler( 2766 FtpInterface, self.Entropy, [uri], myqueue, 2767 critical_files = myqueue, use_handlers = True, 2768 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2769 handlers_data = {'branch': branch }, download = True, repo = repo) 2770 2771 errors, m_fine_uris, m_broken_uris = downloader.go() 2772 if errors: 2773 my_broken_uris = [ 2774 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2775 for x, y in m_broken_uris] 2776 reason = my_broken_uris[0][1] 2777 self.Entropy.updateProgress( 2778 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2779 brown(repo), 2780 red(_("sync")), 2781 brown(branch), 2782 blue(_("download errors")), 2783 darkgreen(crippled_uri), 2784 blue(_("reason")), 2785 reason, 2786 ), 2787 importance = 1, 2788 type = "error", 2789 header = darkred(" !!! ") 2790 ) 2791 return errors, m_fine_uris, m_broken_uris 2792 2793 self.Entropy.updateProgress( 2794 "[repo:%s|%s|%s] %s: %s" % ( 2795 brown(repo), 2796 red(_("sync")), 2797 brown(branch), 2798 blue(_("download completed successfully")), 2799 darkgreen(crippled_uri), 2800 ), 2801 importance = 1, 2802 type = "info", 2803 header = darkgreen(" * ") 2804 ) 2805 return errors, m_fine_uris, m_broken_uris

2806

2807 - def run_package_files_qa_checks(self, packages_list, repo = None):

2808 2809 if repo == None: 2810 repo = self.Entropy.default_repository 2811 2812 my_qa = self.Entropy.QA() 2813 2814 qa_total = len(packages_list) 2815 qa_count = 0 2816 qa_some_faulty = [] 2817 2818 for upload_package in packages_list: 2819 qa_count += 1 2820 2821 self.Entropy.updateProgress( 2822 "%s: %s" % ( 2823 purple(_("QA checking package file")), 2824 darkgreen(os.path.basename(upload_package)), 2825 ), 2826 importance = 0, 2827 type = "info", 2828 header = purple(" @@ "), 2829 back = True, 2830 count = (qa_count, qa_total,) 2831 ) 2832 2833 result = my_qa.entropy_package_checks(upload_package) 2834 if not result: 2835 # call wolfman-911 2836 qa_some_faulty.append(os.path.basename(upload_package)) 2837 2838 if qa_some_faulty: 2839 2840 for qa_faulty_pkg in qa_some_faulty: 2841 self.Entropy.updateProgress( 2842 "[repo:%s|branch:%s] %s: %s" % ( 2843 brown(repo), 2844 self.SystemSettings['repositories']['branch'], 2845 red(_("faulty package file, please fix")), 2846 blue(os.path.basename(qa_faulty_pkg)), 2847 ), 2848 importance = 1, 2849 type = "error", 2850 header = darkred(" @@ ") 2851 ) 2852 raise EntropyPackageException( 2853 'EntropyPackageException: cannot continue')

2854 2855

2856 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2857 repo = None):

2858 2859 if repo == None: 2860 repo = self.Entropy.default_repository 2861 2862 self.Entropy.updateProgress( 2863 "[repo:%s|%s] %s" % ( 2864 repo, 2865 red(_("sync")), 2866 darkgreen(_("starting packages sync")), 2867 ), 2868 importance = 1, 2869 type = "info", 2870 header = red(" @@ "), 2871 back = True 2872 ) 2873 2874 2875 successfull_mirrors = set() 2876 broken_mirrors = set() 2877 check_data = () 2878 upload_queue_qa_checked = set() 2879 mirrors_tainted = False 2880 mirror_errors = False 2881 mirrors_errors = False 2882 2883 for uri in self.Entropy.get_remote_mirrors(repo): 2884 2885 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2886 mirror_errors = False 2887 2888 self.Entropy.updateProgress( 2889 "[repo:%s|%s|branch:%s] %s: %s" % ( 2890 repo, 2891 red(_("sync")), 2892 brown(self.SystemSettings['repositories']['branch']), 2893 blue(_("packages sync")), 2894 bold(crippled_uri), 2895 ), 2896 importance = 1, 2897 type = "info", 2898 header = red(" @@ ") 2899 ) 2900 2901 try: 2902 upload_queue, download_queue, removal_queue, fine_queue, \ 2903 remote_packages_data = self.calculate_packages_to_sync(uri, 2904 self.SystemSettings['repositories']['branch'], repo) 2905 except self.socket.error, err: 2906 self.Entropy.updateProgress( 2907 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2908 repo, 2909 red(_("sync")), 2910 self.SystemSettings['repositories']['branch'], 2911 darkred(_("socket error")), 2912 err, 2913 darkred(_("on")), 2914 crippled_uri, 2915 ), 2916 importance = 1, 2917 type = "error", 2918 header = darkgreen(" * ") 2919 ) 2920 continue 2921 2922 if (not upload_queue) and (not download_queue) and \ 2923 (not removal_queue): 2924 self.Entropy.updateProgress( 2925 "[repo:%s|%s|branch:%s] %s: %s" % ( 2926 repo, 2927 red(_("sync")), 2928 self.SystemSettings['repositories']['branch'], 2929 darkgreen(_("nothing to do on")), 2930 crippled_uri, 2931 ), 2932 importance = 1, 2933 type = "info", 2934 header = darkgreen(" * ") 2935 ) 2936 successfull_mirrors.add(uri) 2937 continue 2938 2939 self.Entropy.updateProgress( 2940 "%s:" % (blue(_("Expanding queues")),), 2941 importance = 1, 2942 type = "info", 2943 header = red(" ** ") 2944 ) 2945 2946 upload, download, removal, copy, metainfo = self.expand_queues( 2947 upload_queue, 2948 download_queue, 2949 removal_queue, 2950 remote_packages_data, 2951 self.SystemSettings['repositories']['branch'], 2952 repo 2953 ) 2954 del upload_queue, download_queue, removal_queue, \ 2955 remote_packages_data 2956 self._show_sync_queues(upload, download, removal, copy, metainfo, 2957 self.SystemSettings['repositories']['branch']) 2958 2959 if not len(upload)+len(download)+len(removal)+len(copy): 2960 2961 self.Entropy.updateProgress( 2962 "[repo:%s|%s|branch:%s] %s %s" % ( 2963 self.Entropy.default_repository, 2964 red(_("sync")), 2965 self.SystemSettings['repositories']['branch'], 2966 blue(_("nothing to sync for")), 2967 crippled_uri, 2968 ), 2969 importance = 1, 2970 type = "info", 2971 header = darkgreen(" @@ ") 2972 ) 2973 2974 successfull_mirrors.add(uri) 2975 continue 2976 2977 if pretend: 2978 successfull_mirrors.add(uri) 2979 continue 2980 2981 if ask: 2982 rc_sync = self.Entropy.askQuestion( 2983 _("Would you like to run the steps above ?")) 2984 if rc_sync == "No": 2985 continue 2986 2987 try: 2988 2989 # QA checks 2990 qa_package_files = [x[0] for x in upload if x[0] \ 2991 not in upload_queue_qa_checked] 2992 upload_queue_qa_checked |= set(qa_package_files) 2993 2994 self.run_package_files_qa_checks(qa_package_files, repo = repo) 2995 2996 if removal: 2997 self._sync_run_removal_queue(removal, 2998 self.SystemSettings['repositories']['branch'], repo) 2999 3000 if copy: 3001 self._sync_run_copy_queue(copy, 3002 self.SystemSettings['repositories']['branch'], repo) 3003 3004 if upload or download: 3005 mirrors_tainted = True 3006 3007 if upload: 3008 d_errors, m_fine_uris, \ 3009 m_broken_uris = self._sync_run_upload_queue( 3010 uri, upload, 3011 self.SystemSettings['repositories']['branch'], repo) 3012 3013 if d_errors: 3014 mirror_errors = True 3015 3016 if download: 3017 my_downlist = [x[0] for x in download] 3018 d_errors, m_fine_uris, \ 3019 m_broken_uris = self._sync_run_download_queue( 3020 uri, my_downlist, 3021 self.SystemSettings['repositories']['branch'], repo) 3022 3023 if d_errors: 3024 mirror_errors = True 3025 if not mirror_errors: 3026 successfull_mirrors.add(uri) 3027 else: 3028 mirrors_errors = True 3029 3030 except KeyboardInterrupt: 3031 self.Entropy.updateProgress( 3032 "[repo:%s|%s|branch:%s] %s" % ( 3033 repo, 3034 red(_("sync")), 3035 self.SystemSettings['repositories']['branch'], 3036 darkgreen(_("keyboard interrupt !")), 3037 ), 3038 importance = 1, 3039 type = "info", 3040 header = darkgreen(" * ") 3041 ) 3042 continue 3043 3044 except EntropyPackageException, err: 3045 3046 mirrors_errors = True 3047 broken_mirrors.add(uri) 3048 successfull_mirrors.clear() 3049 # so that people will realize this is a very bad thing 3050 self.Entropy.updateProgress( 3051 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3052 repo, 3053 red(_("sync")), 3054 self.SystemSettings['repositories']['branch'], 3055 darkred(_("you must package them again")), 3056 Exception, 3057 _("error"), 3058 err, 3059 ), 3060 importance = 1, 3061 type = "error", 3062 header = darkred(" !!! ") 3063 ) 3064 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3065 broken_mirrors, check_data 3066 3067 except Exception, err: 3068 3069 self.entropyTools.print_traceback() 3070 mirrors_errors = True 3071 broken_mirrors.add(uri) 3072 self.Entropy.updateProgress( 3073 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 3074 repo, 3075 red(_("sync")), 3076 self.SystemSettings['repositories']['branch'], 3077 darkred(_("exception caught")), 3078 Exception, 3079 _("error"), 3080 err, 3081 ), 3082 importance = 1, 3083 type = "error", 3084 header = darkred(" !!! ") 3085 ) 3086 3087 exc_txt = self.Entropy.entropyTools.print_exception( 3088 returndata = True) 3089 for line in exc_txt: 3090 self.Entropy.updateProgress( 3091 unicode(line), 3092 importance = 1, 3093 type = "error", 3094 header = darkred(": ") 3095 ) 3096 3097 if len(successfull_mirrors) > 0: 3098 self.Entropy.updateProgress( 3099 "[repo:%s|%s|branch:%s] %s" % ( 3100 repo, 3101 red(_("sync")), 3102 self.SystemSettings['repositories']['branch'], 3103 darkred( 3104 _("at least one mirror synced properly!")), 3105 ), 3106 importance = 1, 3107 type = "error", 3108 header = darkred(" !!! ") 3109 ) 3110 continue 3111 3112 # if at least one server has been synced successfully, move files 3113 if (len(successfull_mirrors) > 0) and not pretend: 3114 self.remove_expiration_files( 3115 self.SystemSettings['repositories']['branch'], repo) 3116 3117 if packages_check: 3118 check_data = self.Entropy.verify_local_packages([], ask = ask, 3119 repo = repo) 3120 3121 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 3122 broken_mirrors, check_data

3123

3124 - def remove_expiration_files(self, branch, repo = None):

3125 3126 if repo == None: 3127 repo = self.Entropy.default_repository 3128 3129 branch_dir = os.path.join( 3130 self.Entropy.get_local_upload_directory(repo), branch) 3131 3132 # check if it exists 3133 if not os.path.isdir(branch_dir): 3134 return None 3135 3136 branchcontent = os.listdir(branch_dir) 3137 for xfile in branchcontent: 3138 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 3139 branch, xfile) 3140 destdir = os.path.join( 3141 self.Entropy.get_local_packages_directory(repo), branch) 3142 if not os.path.isdir(destdir): 3143 os.makedirs(destdir) 3144 dest = os.path.join(destdir, xfile) 3145 shutil.move(source, dest) 3146 # clear expiration file 3147 dest_expiration = dest + etpConst['packagesexpirationfileext'] 3148 if os.path.isfile(dest_expiration): 3149 os.remove(dest_expiration)

3150 3151

3152 - def is_package_expired(self, package_file, branch, repo = None):

3153 3154 pkg_path = os.path.join( 3155 self.Entropy.get_local_packages_directory(repo), branch, 3156 package_file) 3157 pkg_path += etpConst['packagesexpirationfileext'] 3158 if not os.path.isfile(pkg_path): 3159 return False 3160 3161 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 3162 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 3163 days = srv_set['packages_expiration_days'] 3164 delta = int(days)*24*3600 3165 currmtime = time.time() 3166 file_delta = currmtime - mtime 3167 3168 if file_delta > delta: 3169 return True 3170 return False

3171

3172 - def create_expiration_file(self, package_file, branch, repo = None, 3173 gentle = False):

3174 3175 pkg_path = os.path.join( 3176 self.Entropy.get_local_packages_directory(repo), branch, 3177 package_file) 3178 pkg_path += etpConst['packagesexpirationfileext'] 3179 if gentle and os.path.isfile(pkg_path): 3180 return 3181 f_exp = open(pkg_path,"w") 3182 f_exp.flush() 3183 f_exp.close()

3184 3185

3186 - def collect_expiring_packages(self, branch, repo = None):

3187 3188 dbconn = self.Entropy.open_server_repository(just_reading = True, 3189 repo = repo) 3190 database_bins = dbconn.listAllDownloads(do_sort = False, 3191 full_path = True) 3192 bins_dir = os.path.join( 3193 self.Entropy.get_local_packages_directory(repo), branch) 3194 repo_bins = set() 3195 3196 if os.path.isdir(bins_dir): 3197 repo_bins = os.listdir(bins_dir) 3198 repo_bins = set([ 3199 os.path.join('packages', etpSys['arch'], branch, x) for x \ 3200 in repo_bins if x.endswith(etpConst['packagesext'])]) 3201 repo_bins -= database_bins 3202 3203 return set([os.path.basename(x) for x in repo_bins])

3204 3205

3206 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

3207 3208 if repo == None: 3209 repo = self.Entropy.default_repository 3210 3211 self.Entropy.updateProgress( 3212 "[repo:%s|%s|branch:%s] %s" % ( 3213 brown(repo), 3214 red(_("tidy")), 3215 blue(self.SystemSettings['repositories']['branch']), 3216 blue(_("collecting expired packages")), 3217 ), 3218 importance = 1, 3219 type = "info", 3220 header = red(" @@ ") 3221 ) 3222 3223 branch_data = {} 3224 errors = False 3225 branch_data['errors'] = False 3226 branch = self.SystemSettings['repositories']['branch'] 3227 3228 self.Entropy.updateProgress( 3229 "[branch:%s] %s" % ( 3230 brown(branch), 3231 blue(_("collecting expired packages in the selected branches")), 3232 ), 3233 importance = 1, 3234 type = "info", 3235 header = blue(" @@ ") 3236 ) 3237 3238 # collect removed packages 3239 expiring_packages = self.collect_expiring_packages(branch, repo) 3240 if expiring_packages: 3241 3242 # filter expired packages used by other branches 3243 # this is done for the sake of consistency 3244 # --- read packages.db.pkglist, make sure your repository 3245 # has been ported to latest Entropy 3246 3247 branch_pkglist_data = self.read_remote_file_in_branches( 3248 etpConst['etpdatabasepkglist'], repo = repo, 3249 excluded_branches = [branch]) 3250 # format data 3251 for key, val in branch_pkglist_data.items(): 3252 branch_pkglist_data[key] = val.split("\n") 3253 3254 3255 remote_relpath = os.path.join(etpConst['packagesrelativepath'], 3256 branch) 3257 my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \ 3258 expiring_packages]) 3259 3260 for other_branch in branch_pkglist_data: 3261 branch_pkglist = set(branch_pkglist_data[other_branch]) 3262 my_expiring_pkgs -= branch_pkglist 3263 3264 # fallback to normality, set new expiring packages var 3265 expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs] 3266 3267 removal = [] 3268 for package in expiring_packages: 3269 expired = self.is_package_expired(package, branch, repo) 3270 if expired: 3271 removal.append(package) 3272 else: 3273 self.create_expiration_file(package, branch, repo, 3274 gentle = True) 3275 3276 # fill returning data 3277 branch_data['removal'] = removal[:] 3278 3279 if not removal: 3280 self.Entropy.updateProgress( 3281 "[branch:%s] %s" % ( 3282 brown(branch), 3283 blue(_("nothing to remove on this branch")), 3284 ), 3285 importance = 1, 3286 type = "info", 3287 header = blue(" @@ ") 3288 ) 3289 return errors, branch_data 3290 else: 3291 self.Entropy.updateProgress( 3292 "[branch:%s] %s:" % ( 3293 brown(branch), 3294 blue(_("these are the expired packages")), 3295 ), 3296 importance = 1, 3297 type = "info", 3298 header = blue(" @@ ") 3299 ) 3300 for package in removal: 3301 self.Entropy.updateProgress( 3302 "[branch:%s] %s: %s" % ( 3303 brown(branch), 3304 blue(_("remove")), 3305 darkgreen(package), 3306 ), 3307 importance = 1, 3308 type = "info", 3309 header = brown(" # ") 3310 ) 3311 3312 if pretend: 3313 return errors, branch_data 3314 3315 if ask: 3316 rc_question = self.Entropy.askQuestion( 3317 _("Would you like to continue ?")) 3318 if rc_question == "No": 3319 return errors, branch_data 3320 3321 myqueue = [] 3322 for package in removal: 3323 myqueue.append(package+etpConst['packagesmd5fileext']) 3324 myqueue.append(package) 3325 3326 ftp_basedir = os.path.join( 3327 self.Entropy.get_remote_packages_relative_path(repo), branch) 3328 for uri in self.Entropy.get_remote_mirrors(repo): 3329 3330 self.Entropy.updateProgress( 3331 "[branch:%s] %s..." % ( 3332 brown(branch), 3333 blue(_("removing packages remotely")), 3334 ), 3335 importance = 1, 3336 type = "info", 3337 header = blue(" @@ ") 3338 ) 3339 3340 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3341 destroyer = self.FtpServerHandler( 3342 FtpInterface, 3343 self.Entropy, 3344 [uri], 3345 myqueue, 3346 critical_files = [], 3347 ftp_basedir = ftp_basedir, 3348 remove = True, 3349 repo = repo 3350 ) 3351 errors, m_fine_uris, m_broken_uris = destroyer.go() 3352 if errors: 3353 my_broken_uris = [ 3354 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3355 for x in m_broken_uris] 3356 3357 reason = my_broken_uris[0][1] 3358 self.Entropy.updateProgress( 3359 "[branch:%s] %s: %s, %s: %s" % ( 3360 brown(branch), 3361 blue(_("remove errors")), 3362 red(crippled_uri), 3363 blue(_("reason")), 3364 reason, 3365 ), 3366 importance = 1, 3367 type = "warning", 3368 header = brown(" !!! ") 3369 ) 3370 branch_data['errors'] = True 3371 errors = True 3372 3373 self.Entropy.updateProgress( 3374 "[branch:%s] %s..." % ( 3375 brown(branch), 3376 blue(_("removing packages locally")), 3377 ), 3378 importance = 1, 3379 type = "info", 3380 header = blue(" @@ ") 3381 ) 3382 3383 branch_data['removed'] = set() 3384 for package in removal: 3385 package_path = os.path.join( 3386 self.Entropy.get_local_packages_directory(repo), 3387 branch, package) 3388 package_path_hash = package_path + \ 3389 etpConst['packagesmd5fileext'] 3390 package_path_expired = package_path + \ 3391 etpConst['packagesexpirationfileext'] 3392 3393 my_rm_list = (package_path_hash, package_path, 3394 package_path_expired) 3395 for myfile in my_rm_list: 3396 if os.path.isfile(myfile): 3397 self.Entropy.updateProgress( 3398 "[branch:%s] %s: %s" % ( 3399 brown(branch), 3400 blue(_("removing")), 3401 darkgreen(myfile), 3402 ), 3403 importance = 1, 3404 type = "info", 3405 header = brown(" @@ ") 3406 ) 3407 os.remove(myfile) 3408 branch_data['removed'].add(myfile) 3409 3410 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors