entropy.server.interfaces.mirrors

37 38 import socket 39 import entropy.dump as dumpTools 40 import entropy.tools as entropyTools

41 - def __init__(self, ServerInstance, repo = None):

42 43 from entropy.server.interfaces.main import Server as MainServer 44 if not isinstance(ServerInstance, MainServer): 45 mytxt = _("entropy.server.interfaces.main.Server needed") 46 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 47 48 self.Entropy = ServerInstance 49 from entropy.transceivers import FtpServerHandler 50 self.FtpServerHandler = FtpServerHandler 51 from entropy.cache import EntropyCacher 52 self.Cacher = EntropyCacher() 53 self.sys_settings_plugin_id = \ 54 etpConst['system_settings_plugins_ids']['server_plugin'] 55 self.SystemSettings = self.Entropy.SystemSettings 56 57 mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),) 58 self.Entropy.updateProgress( 59 mytxt, 60 importance = 2, 61 type = "info", 62 header = red(" @@ ") 63 ) 64 mytxt = _("mirror") 65 for mirror in self.Entropy.get_remote_mirrors(repo): 66 mirror = self.entropyTools.hide_ftp_password(mirror) 67 self.Entropy.updateProgress( 68 blue("%s: %s") % (mytxt, darkgreen(mirror),), 69 importance = 0, 70 type = "info", 71 header = brown(" # ") 72 )

73 74

75 - def lock_mirrors(self, lock = True, mirrors = None, repo = None):

76 77 if repo == None: 78 repo = self.Entropy.default_repository 79 80 if not mirrors: 81 mirrors = self.Entropy.get_remote_mirrors(repo) 82 83 issues = False 84 for uri in mirrors: 85 86 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 87 88 lock_text = _("unlocking") 89 if lock: 90 lock_text = _("locking") 91 self.Entropy.updateProgress( 92 "[repo:%s|%s] %s %s" % ( 93 brown(repo), 94 darkgreen(crippled_uri), 95 bold(lock_text), 96 blue("%s...") % (_("mirror"),), 97 ), 98 importance = 1, 99 type = "info", 100 header = brown(" * "), 101 back = True 102 ) 103 104 try: 105 ftp = FtpInterface(uri, self.Entropy) 106 except ConnectionError: 107 self.entropyTools.print_traceback() 108 return True # issues 109 my_path = os.path.join( 110 self.Entropy.get_remote_database_relative_path(repo), 111 self.SystemSettings['repositories']['branch']) 112 ftp.set_cwd(my_path, dodir = True) 113 114 lock_file = etpConst['etpdatabaselockfile'] 115 if lock and ftp.is_file_available(lock_file): 116 self.Entropy.updateProgress( 117 "[repo:%s|%s] %s" % ( 118 brown(repo), 119 darkgreen(crippled_uri), 120 blue(_("mirror already locked")), 121 ), 122 importance = 1, 123 type = "info", 124 header = darkgreen(" * ") 125 ) 126 ftp.close() 127 continue 128 elif not lock and not ftp.is_file_available(lock_file): 129 self.Entropy.updateProgress( 130 "[repo:%s|%s] %s" % ( 131 brown(repo), 132 darkgreen(crippled_uri), 133 blue(_("mirror already unlocked")), 134 ), 135 importance = 1, 136 type = "info", 137 header = darkgreen(" * ") 138 ) 139 ftp.close() 140 continue 141 142 if lock: 143 rc_lock = self.do_mirror_lock(uri, ftp, repo = repo) 144 else: 145 rc_lock = self.do_mirror_unlock(uri, ftp, repo = repo) 146 ftp.close() 147 if not rc_lock: 148 issues = True 149 150 if not issues: 151 db_taint_file = self.Entropy.get_local_database_taint_file(repo) 152 if os.path.isfile(db_taint_file): 153 os.remove(db_taint_file) 154 155 return issues

156 157

158 - def lock_mirrors_for_download(self, lock = True, mirrors = None, 159 repo = None):

160 # this functions makes entropy clients to not download anything 161 # from the chosen mirrors. it is used to avoid clients to 162 # download databases while we're uploading a new one 163 164 if mirrors == None: 165 mirrors = [] 166 167 if repo == None: 168 repo = self.Entropy.default_repository 169 170 if not mirrors: 171 mirrors = self.Entropy.get_remote_mirrors(repo) 172 173 issues = False 174 for uri in mirrors: 175 176 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 177 178 lock_text = _("unlocking") 179 if lock: 180 lock_text = _("locking") 181 self.Entropy.updateProgress( 182 "[repo:%s|%s] %s %s..." % ( 183 blue(repo), 184 red(crippled_uri), 185 bold(lock_text), 186 blue(_("mirror for download")), 187 ), 188 importance = 1, 189 type = "info", 190 header = red(" @@ "), 191 back = True 192 ) 193 194 try: 195 ftp = FtpInterface(uri, self.Entropy) 196 except ConnectionError: 197 self.entropyTools.print_traceback() 198 return True # issues 199 my_path = os.path.join( 200 self.Entropy.get_remote_database_relative_path(repo), 201 self.SystemSettings['repositories']['branch']) 202 ftp.set_cwd(my_path, dodir = True) 203 204 lock_file = etpConst['etpdatabasedownloadlockfile'] 205 206 if lock and ftp.is_file_available(lock_file): 207 self.Entropy.updateProgress( 208 "[repo:%s|%s] %s" % ( 209 blue(repo), 210 red(crippled_uri), 211 blue(_("mirror already locked for download")), 212 ), 213 importance = 1, 214 type = "info", 215 header = red(" @@ ") 216 ) 217 ftp.close() 218 continue 219 220 elif not lock and not ftp.is_file_available(lock_file): 221 self.Entropy.updateProgress( 222 "[repo:%s|%s] %s" % ( 223 blue(repo), 224 red(crippled_uri), 225 blue(_("mirror already unlocked for download")), 226 ), 227 importance = 1, 228 type = "info", 229 header = red(" @@ ") 230 ) 231 ftp.close() 232 continue 233 234 if lock: 235 rc_lock = self.do_mirror_lock(uri, ftp, dblock = False, 236 repo = repo) 237 else: 238 rc_lock = self.do_mirror_unlock(uri, ftp, dblock = False, 239 repo = repo) 240 ftp.close() 241 if not rc_lock: 242 issues = True 243 244 return issues

245

246 - def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, 247 repo = None):

248 249 if repo == None: 250 repo = self.Entropy.default_repository 251 252 my_path = os.path.join( 253 self.Entropy.get_remote_database_relative_path(repo), 254 self.SystemSettings['repositories']['branch']) 255 if not ftp_connection: 256 try: 257 ftp_connection = FtpInterface(uri, self.Entropy) 258 except ConnectionError: 259 self.entropyTools.print_traceback() 260 return False # issues 261 ftp_connection.set_cwd(my_path, dodir = True) 262 else: 263 mycwd = ftp_connection.get_cwd() 264 if mycwd != my_path: 265 ftp_connection.set_basedir() 266 ftp_connection.set_cwd(my_path, dodir = True) 267 268 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 269 lock_string = u'' 270 271 if dblock: 272 self.create_local_database_lockfile(repo) 273 lock_file = self.get_database_lockfile(repo) 274 else: 275 # locking/unlocking mirror1 for download 276 lock_string = _('for download') 277 self.create_local_database_download_lockfile(repo) 278 lock_file = self.get_database_download_lockfile(repo) 279 280 rc_upload = ftp_connection.upload_file(lock_file, ascii = True) 281 if rc_upload: 282 self.Entropy.updateProgress( 283 "[repo:%s|%s] %s %s" % ( 284 blue(repo), 285 red(crippled_uri), 286 blue(_("mirror successfully locked")), 287 blue(lock_string), 288 ), 289 importance = 1, 290 type = "info", 291 header = red(" @@ ") 292 ) 293 else: 294 self.Entropy.updateProgress( 295 "[repo:%s|%s] %s: %s - %s %s" % ( 296 blue(repo), 297 red(crippled_uri), 298 blue("lock error"), 299 rc_upload, 300 blue(_("mirror not locked")), 301 blue(lock_string), 302 ), 303 importance = 1, 304 type = "error", 305 header = darkred(" * ") 306 ) 307 self.remove_local_database_lockfile(repo) 308 309 return rc_upload

310 311

312 - def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

313 314 if repo == None: 315 repo = self.Entropy.default_repository 316 317 my_path = os.path.join( 318 self.Entropy.get_remote_database_relative_path(repo), 319 self.SystemSettings['repositories']['branch']) 320 if not ftp_connection: 321 try: 322 ftp_connection = FtpInterface(uri, self.Entropy) 323 except ConnectionError: 324 self.entropyTools.print_traceback() 325 return False # issues 326 ftp_connection.set_cwd(my_path) 327 else: 328 mycwd = ftp_connection.get_cwd() 329 if mycwd != my_path: 330 ftp_connection.set_basedir() 331 ftp_connection.set_cwd(my_path) 332 333 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 334 335 if dblock: 336 dbfile = etpConst['etpdatabaselockfile'] 337 else: 338 dbfile = etpConst['etpdatabasedownloadlockfile'] 339 rc_delete = ftp_connection.delete_file(dbfile) 340 if rc_delete: 341 self.Entropy.updateProgress( 342 "[repo:%s|%s] %s" % ( 343 blue(repo), 344 red(crippled_uri), 345 blue(_("mirror successfully unlocked")), 346 ), 347 importance = 1, 348 type = "info", 349 header = darkgreen(" * ") 350 ) 351 if dblock: 352 self.remove_local_database_lockfile(repo) 353 else: 354 self.remove_local_database_download_lockfile(repo) 355 else: 356 self.Entropy.updateProgress( 357 "[repo:%s|%s] %s: %s - %s" % ( 358 blue(repo), 359 red(crippled_uri), 360 blue(_("unlock error")), 361 rc_delete, 362 blue(_("mirror not unlocked")), 363 ), 364 importance = 1, 365 type = "error", 366 header = darkred(" * ") 367 ) 368 369 return rc_delete

370

371 - def get_database_lockfile(self, repo = None):

372 if repo == None: 373 repo = self.Entropy.default_repository 374 return os.path.join(self.Entropy.get_local_database_dir(repo), 375 etpConst['etpdatabaselockfile'])

376

377 - def get_database_download_lockfile(self, repo = None):

378 if repo == None: 379 repo = self.Entropy.default_repository 380 return os.path.join(self.Entropy.get_local_database_dir(repo), 381 etpConst['etpdatabasedownloadlockfile'])

382

383 - def create_local_database_download_lockfile(self, repo = None):

384 if repo == None: 385 repo = self.Entropy.default_repository 386 lock_file = self.get_database_download_lockfile(repo) 387 f_lock = open(lock_file, "w") 388 f_lock.write("download locked") 389 f_lock.flush() 390 f_lock.close()

391

392 - def create_local_database_lockfile(self, repo = None):

393 if repo == None: 394 repo = self.Entropy.default_repository 395 lock_file = self.get_database_lockfile(repo) 396 f_lock = open(lock_file, "w") 397 f_lock.write("database locked") 398 f_lock.flush() 399 f_lock.close()

400

401 - def remove_local_database_lockfile(self, repo = None):

402 if repo == None: 403 repo = self.Entropy.default_repository 404 lock_file = self.get_database_lockfile(repo) 405 if os.path.isfile(lock_file): 406 os.remove(lock_file)

407

408 - def remove_local_database_download_lockfile(self, repo = None):

409 if repo == None: 410 repo = self.Entropy.default_repository 411 lock_file = self.get_database_download_lockfile(repo) 412 if os.path.isfile(lock_file): 413 os.remove(lock_file)

414

415 - def download_package(self, uri, pkg_relative_path, repo = None):

416 417 if repo == None: 418 repo = self.Entropy.default_repository 419 420 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 421 422 tries = 0 423 while tries < 5: 424 tries += 1 425 426 pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:]) 427 pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path) 428 pkgfile = os.path.basename(pkg_relative_path) 429 430 self.Entropy.updateProgress( 431 "[repo:%s|%s|#%s] %s: %s" % ( 432 brown(repo), 433 darkgreen(crippled_uri), 434 brown(tries), 435 blue(_("connecting to download package")), 436 darkgreen(pkg_to_join_path), 437 ), 438 importance = 1, 439 type = "info", 440 header = darkgreen(" * "), 441 back = True 442 ) 443 444 try: 445 ftp = FtpInterface(uri, self.Entropy) 446 except ConnectionError: 447 self.entropyTools.print_traceback() 448 return False # issues 449 dirpath = os.path.join( 450 self.Entropy.get_remote_packages_relative_path(repo), 451 pkg_to_join_dirpath) 452 ftp.set_cwd(dirpath, dodir = True) 453 454 self.Entropy.updateProgress( 455 "[repo:%s|%s|#%s] %s: %s" % ( 456 brown(repo), 457 darkgreen(crippled_uri), 458 brown(tries), 459 blue(_("downloading package")), 460 darkgreen(pkg_to_join_path), 461 ), 462 importance = 1, 463 type = "info", 464 header = darkgreen(" * ") 465 ) 466 467 download_path = os.path.join( 468 self.Entropy.get_local_packages_directory(repo), 469 pkg_to_join_dirpath) 470 if (not os.path.isdir(download_path)) and \ 471 (not os.access(download_path, os.R_OK)): 472 os.makedirs(download_path) 473 rc_download = ftp.download_file(pkgfile, download_path) 474 if not rc_download: 475 self.Entropy.updateProgress( 476 "[repo:%s|%s|#%s] %s: %s %s" % ( 477 brown(repo), 478 darkgreen(crippled_uri), 479 brown(tries), 480 blue(_("package")), 481 darkgreen(pkg_to_join_path), 482 blue(_("does not exist")), 483 ), 484 importance = 1, 485 type = "error", 486 header = darkred(" !!! ") 487 ) 488 ftp.close() 489 return rc_download 490 491 dbconn = self.Entropy.open_server_repository(read_only = True, 492 no_upload = True, repo = repo) 493 idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path) 494 if idpackage == -1: 495 self.Entropy.updateProgress( 496 "[repo:%s|%s|#%s] %s: %s %s" % ( 497 brown(repo), 498 darkgreen(crippled_uri), 499 brown(tries), 500 blue(_("package")), 501 darkgreen(pkgfile), 502 blue(_("is not listed in the repository !")), 503 ), 504 importance = 1, 505 type = "error", 506 header = darkred(" !!! ") 507 ) 508 ftp.close() 509 return False 510 511 storedmd5 = dbconn.retrieveDigest(idpackage) 512 self.Entropy.updateProgress( 513 "[repo:%s|%s|#%s] %s: %s" % ( 514 brown(repo), 515 darkgreen(crippled_uri), 516 brown(tries), 517 blue(_("verifying checksum of package")), 518 darkgreen(pkgfile), 519 ), 520 importance = 1, 521 type = "info", 522 header = darkgreen(" * "), 523 back = True 524 ) 525 526 pkg_path = os.path.join(download_path, pkgfile) 527 md5check = self.entropyTools.compare_md5(pkg_path, storedmd5) 528 if md5check: 529 self.Entropy.updateProgress( 530 "[repo:%s|%s|#%s] %s: %s %s" % ( 531 brown(repo), 532 darkgreen(crippled_uri), 533 brown(tries), 534 blue(_("package")), 535 darkgreen(pkgfile), 536 blue(_("downloaded successfully")), 537 ), 538 importance = 1, 539 type = "info", 540 header = darkgreen(" * ") 541 ) 542 return True 543 else: 544 self.Entropy.updateProgress( 545 "[repo:%s|%s|#%s] %s: %s %s" % ( 546 brown(repo), 547 darkgreen(crippled_uri), 548 brown(tries), 549 blue(_("package")), 550 darkgreen(pkgfile), 551 blue(_("checksum does not match. re-downloading...")), 552 ), 553 importance = 1, 554 type = "warning", 555 header = darkred(" * ") 556 ) 557 if os.path.isfile(pkg_path): 558 os.remove(pkg_path) 559 560 continue 561 562 # if we get here it means the files hasn't been downloaded properly 563 self.Entropy.updateProgress( 564 "[repo:%s|%s|#%s] %s: %s %s" % ( 565 brown(repo), 566 darkgreen(crippled_uri), 567 brown(tries), 568 blue(_("package")), 569 darkgreen(pkgfile), 570 blue(_("seems broken. Consider to re-package it. Giving up!")), 571 ), 572 importance = 1, 573 type = "error", 574 header = darkred(" !!! ") 575 ) 576 return False

577 578

579 - def get_remote_databases_status(self, repo = None, mirrors = None):

580 581 if repo == None: 582 repo = self.Entropy.default_repository 583 584 if not mirrors: 585 mirrors = self.Entropy.get_remote_mirrors(repo) 586 587 data = [] 588 for uri in mirrors: 589 590 # let it raise an exception if connection is impossible 591 ftp = FtpInterface(uri, self.Entropy) 592 try: 593 my_path = os.path.join( 594 self.Entropy.get_remote_database_relative_path(repo), 595 self.SystemSettings['repositories']['branch']) 596 ftp.set_cwd(my_path, dodir = True) 597 except ftp.ftplib.error_perm: 598 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 599 self.Entropy.updateProgress( 600 "[repo:%s|%s] %s !" % ( 601 brown(repo), 602 darkgreen(crippled_uri), 603 blue(_("mirror has invalid directory structure")), 604 ), 605 importance = 1, 606 type = "warning", 607 header = darkred(" !!! ") 608 ) 609 ftp.close() 610 continue 611 sys_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 612 db_format = sys_set['database_file_format'] 613 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 614 if cmethod == None: 615 raise InvalidDataType("InvalidDataType: %s." % ( 616 _("Wrong database compression method passed"), 617 ) 618 ) 619 compressedfile = etpConst[cmethod[2]] 620 621 revision = 0 622 rc1 = ftp.is_file_available(compressedfile) 623 revfilename = os.path.basename( 624 self.Entropy.get_local_database_revision_file(repo)) 625 rc2 = ftp.is_file_available(revfilename) 626 if rc1 and rc2: 627 revision_localtmppath = os.path.join(etpConst['packagestmpdir'], 628 revfilename) 629 dlcount = 5 630 while dlcount: 631 dled = ftp.download_file(revfilename, 632 etpConst['packagestmpdir'], True) 633 if dled: 634 break 635 dlcount -= 1 636 637 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 638 if os.access(revision_localtmppath, os.R_OK | os.F_OK): 639 f_rev = open(revision_localtmppath,"r") 640 try: 641 revision = int(f_rev.readline().strip()) 642 except ValueError: 643 mytxt = _("mirror hasn't valid database revision file") 644 self.Entropy.updateProgress( 645 "[repo:%s|%s] %s: %s" % ( 646 brown(repo), 647 darkgreen(crippled_uri), 648 blue(mytxt), 649 bold(revision), 650 ), 651 importance = 1, 652 type = "error", 653 header = darkred(" !!! ") 654 ) 655 revision = 0 656 f_rev.close() 657 elif dlcount == 0: 658 self.Entropy.updateProgress( 659 "[repo:%s|%s] %s: %s" % ( 660 brown(repo), 661 darkgreen(crippled_uri), 662 blue(_("unable to download repository revision")), 663 bold(revision), 664 ), 665 importance = 1, 666 type = "error", 667 header = darkred(" !!! ") 668 ) 669 revision = 0 670 else: 671 self.Entropy.updateProgress( 672 "[repo:%s|%s] %s: %s" % ( 673 brown(repo), 674 darkgreen(crippled_uri), 675 blue(_("mirror doesn't have valid revision file")), 676 bold(revision), 677 ), 678 importance = 1, 679 type = "error", 680 header = darkred(" !!! ") 681 ) 682 revision = 0 683 if os.path.isfile(revision_localtmppath): 684 os.remove(revision_localtmppath) 685 686 info = [uri, revision] 687 data.append(info) 688 ftp.close() 689 690 return data

691

692 - def is_local_database_locked(self, repo = None):

693 local_repo = repo 694 if local_repo == None: 695 local_repo = self.Entropy.default_repository 696 lock_file = self.get_database_lockfile(local_repo) 697 return os.path.isfile(lock_file)

698

699 - def get_mirrors_lock(self, repo = None):

700 701 dbstatus = [] 702 for uri in self.Entropy.get_remote_mirrors(repo): 703 data = [uri, False, False] 704 ftp = FtpInterface(uri, self.Entropy) 705 try: 706 my_path = os.path.join( 707 self.Entropy.get_remote_database_relative_path(repo), 708 self.SystemSettings['repositories']['branch']) 709 ftp.set_cwd(my_path) 710 except ftp.ftplib.error_perm: 711 ftp.close() 712 continue 713 if ftp.is_file_available(etpConst['etpdatabaselockfile']): 714 # upload locked 715 data[1] = True 716 if ftp.is_file_available(etpConst['etpdatabasedownloadlockfile']): 717 # download locked 718 data[2] = True 719 ftp.close() 720 dbstatus.append(data) 721 return dbstatus

722

723 - def download_notice_board(self, repo = None):

724 725 if repo == None: 726 repo = self.Entropy.default_repository 727 mirrors = self.Entropy.get_remote_mirrors(repo) 728 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 729 mytmpdir = self.entropyTools.get_random_temp_file() 730 os.makedirs(mytmpdir) 731 732 self.Entropy.updateProgress( 733 "[repo:%s] %s %s" % ( 734 brown(repo), 735 blue(_("downloading notice board from mirrors to")), 736 red(rss_path), 737 ), 738 importance = 1, 739 type = "info", 740 header = blue(" @@ ") 741 ) 742 743 downloaded = False 744 for uri in mirrors: 745 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 746 downloader = self.FtpServerHandler( 747 FtpInterface, self.Entropy, [uri], 748 [rss_path], download = True, 749 local_basedir = mytmpdir, critical_files = [rss_path], 750 repo = repo 751 ) 752 errors, m_fine_uris, m_broken_uris = downloader.go() 753 if not errors: 754 self.Entropy.updateProgress( 755 "[repo:%s] %s: %s" % ( 756 brown(repo), 757 blue(_("notice board downloaded successfully from")), 758 red(crippled_uri), 759 ), 760 importance = 1, 761 type = "info", 762 header = blue(" @@ ") 763 ) 764 downloaded = True 765 break 766 767 if downloaded: 768 shutil.move(os.path.join(mytmpdir, os.path.basename(rss_path)), 769 rss_path) 770 771 return downloaded

772

773 - def upload_notice_board(self, repo = None):

774 775 if repo == None: 776 repo = self.Entropy.default_repository 777 mirrors = self.Entropy.get_remote_mirrors(repo) 778 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 779 780 self.Entropy.updateProgress( 781 "[repo:%s] %s %s" % ( 782 brown(repo), 783 blue(_("uploading notice board from")), 784 red(rss_path), 785 ), 786 importance = 1, 787 type = "info", 788 header = blue(" @@ ") 789 ) 790 791 uploader = self.FtpServerHandler( 792 FtpInterface, 793 self.Entropy, 794 mirrors, 795 [rss_path], 796 critical_files = [rss_path], 797 repo = repo 798 ) 799 errors, m_fine_uris, m_broken_uris = uploader.go() 800 if errors: 801 m_broken_uris = sorted(m_broken_uris) 802 m_broken_uris = [self.entropyTools.extract_ftp_host_from_uri(x) \ 803 for x in m_broken_uris] 804 self.Entropy.updateProgress( 805 "[repo:%s] %s %s" % ( 806 brown(repo), 807 blue(_("notice board upload failed on")), 808 red(', '.join(m_broken_uris)), 809 ), 810 importance = 1, 811 type = "info", 812 header = blue(" @@ ") 813 ) 814 return False 815 self.Entropy.updateProgress( 816 "[repo:%s] %s" % ( 817 brown(repo), 818 blue(_("notice board upload success")), 819 ), 820 importance = 1, 821 type = "info", 822 header = blue(" @@ ") 823 ) 824 return True

825 826

827 - def update_notice_board(self, title, notice_text, link = None, repo = None):

828 829 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 830 rss_description = "Inform about important distribution activities." 831 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 832 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 833 if not link: 834 link = srv_set['rss']['website_url'] 835 836 self.download_notice_board(repo) 837 rss_main = RSS(rss_path, rss_title, rss_description, 838 maxentries = 20) 839 rss_main.add_item(title, link, description = notice_text) 840 rss_main.write_changes() 841 status = self.upload_notice_board(repo) 842 return status

843

844 - def read_notice_board(self, do_download = True, repo = None):

845 846 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 847 if do_download: 848 self.download_notice_board(repo) 849 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 850 return None 851 rss_main = RSS(rss_path, '', '') 852 return rss_main.get_entries()

853

854 - def remove_from_notice_board(self, identifier, repo = None):

855 856 rss_path = self.Entropy.get_local_database_notice_board_file(repo) 857 rss_title = "%s Notice Board" % (self.SystemSettings['system']['name'],) 858 rss_description = "Inform about important distribution activities." 859 if not (os.path.isfile(rss_path) and os.access(rss_path, os.R_OK)): 860 return 0 861 rss_main = RSS(rss_path, rss_title, rss_description) 862 data = rss_main.remove_entry(identifier) 863 rss_main.write_changes() 864 return data

865

866 - def update_rss_feed(self, repo = None):

867 868 product = self.SystemSettings['repositories']['product'] 869 #db_dir = self.Entropy.get_local_database_dir(repo) 870 rss_path = self.Entropy.get_local_database_rss_file(repo) 871 rss_light_path = self.Entropy.get_local_database_rsslight_file(repo) 872 rss_dump_name = etpConst['rss-dump-name'] 873 db_revision_path = self.Entropy.get_local_database_revision_file(repo) 874 875 rss_title = u"%s Online Repository Status" % ( 876 self.SystemSettings['system']['name'],) 877 rss_description = \ 878 u"Keep you updated on what's going on in the %s Repository." % ( 879 self.SystemSettings['system']['name'],) 880 881 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 882 883 rss_main = RSS(rss_path, rss_title, rss_description, 884 maxentries = srv_set['rss']['max_entries']) 885 # load dump 886 db_actions = self.Cacher.pop(rss_dump_name) 887 if db_actions: 888 try: 889 f_rev = open(db_revision_path) 890 revision = f_rev.readline().strip() 891 f_rev.close() 892 except (IOError, OSError): 893 revision = u"N/A" 894 commitmessage = u'' 895 if self.Entropy.rssMessages['commitmessage']: 896 commitmessage = u' :: ' + \ 897 self.Entropy.rssMessages['commitmessage'] 898 899 title = u": " + self.SystemSettings['system']['name'] + u" " + \ 900 product[0].upper() + product[1:] + " " + \ 901 self.SystemSettings['repositories']['branch'] + \ 902 " :: Revision: " + revision + commitmessage 903 904 link = srv_set['rss']['base_url'] 905 # create description 906 added_items = db_actions.get("added") 907 908 if added_items: 909 for atom in added_items: 910 mylink = link + "?search=" + atom.split("~")[0] + \ 911 "&arch=" + etpConst['currentarch'] + "&product="+product 912 description = atom + ": " + added_items[atom]['description'] 913 rss_main.add_item(title = "Added/Updated" + title, 914 link = mylink, description = description) 915 removed_items = db_actions.get("removed") 916 917 if removed_items: 918 for atom in removed_items: 919 description = atom + ": " + \ 920 removed_items[atom]['description'] 921 rss_main.add_item(title = "Removed" + title, link = link, 922 description = description) 923 924 light_items = db_actions.get('light') 925 if light_items: 926 rss_light = RSS(rss_light_path, rss_title, rss_description, 927 maxentries = srv_set['rss']['light_max_entries']) 928 for atom in light_items: 929 mylink = link + "?search=" + atom.split("~")[0] + \ 930 "&arch=" + etpConst['currentarch'] + "&product=" + \ 931 product 932 description = light_items[atom]['description'] 933 rss_light.add_item(title = "[" + revision + "] " + atom, 934 link = mylink, description = description) 935 rss_light.write_changes() 936 937 rss_main.write_changes() 938 self.Entropy.rssMessages.clear() 939 self.dumpTools.removeobj(rss_dump_name)

940 941

942 - def dump_database_to_file(self, db_path, destination_path, opener, 943 exclude_tables = None, repo = None):

944 945 if not exclude_tables: 946 exclude_tables = [] 947 948 f_out = opener(destination_path, "wb") 949 dbconn = self.Entropy.open_server_repository(db_path, 950 just_reading = True, repo = repo) 951 dbconn.doDatabaseExport(f_out, exclude_tables = exclude_tables) 952 self.Entropy.close_server_database(dbconn) 953 f_out.close()

954

955 - def create_file_checksum(self, file_path, checksum_path):

956 mydigest = self.entropyTools.md5sum(file_path) 957 f_ck = open(checksum_path, "w") 958 mystring = "%s %s\n" % (mydigest, os.path.basename(file_path),) 959 f_ck.write(mystring) 960 f_ck.flush() 961 f_ck.close()

962

963 - def compress_file(self, file_path, destination_path, opener):

964 f_out = opener(destination_path, "wb") 965 f_in = open(file_path,"rb") 966 data = f_in.read(8192) 967 while data: 968 f_out.write(data) 969 data = f_in.read(8192) 970 f_in.close() 971 if hasattr(f_out,'flush'): 972 f_out.flush() 973 f_out.close()

974

975 - def get_files_to_sync(self, cmethod, download = False, repo = None, 976 disabled_eapis = None):

977 978 if disabled_eapis == None: 979 disabled_eapis = [] 980 981 critical = [] 982 extra_text_files = [] 983 data = {} 984 data['database_revision_file'] = \ 985 self.Entropy.get_local_database_revision_file(repo) 986 extra_text_files.append(data['database_revision_file']) 987 critical.append(data['database_revision_file']) 988 989 database_ts_file = self.Entropy.get_local_database_timestamp_file(repo) 990 if os.path.isfile(database_ts_file) or download: 991 data['database_timestamp_file'] = database_ts_file 992 if not download: 993 critical.append(database_ts_file) 994 995 database_package_mask_file = \ 996 self.Entropy.get_local_database_mask_file(repo) 997 extra_text_files.append(database_package_mask_file) 998 if os.path.isfile(database_package_mask_file) or download: 999 data['database_package_mask_file'] = database_package_mask_file 1000 if not download: 1001 critical.append(data['database_package_mask_file']) 1002 1003 database_package_system_mask_file = \ 1004 self.Entropy.get_local_database_system_mask_file(repo) 1005 extra_text_files.append(database_package_system_mask_file) 1006 if os.path.isfile(database_package_system_mask_file) or download: 1007 data['database_package_system_mask_file'] = \ 1008 database_package_system_mask_file 1009 if not download: 1010 critical.append(data['database_package_system_mask_file']) 1011 1012 database_package_confl_tagged_file = \ 1013 self.Entropy.get_local_database_confl_tagged_file(repo) 1014 extra_text_files.append(database_package_confl_tagged_file) 1015 if os.path.isfile(database_package_confl_tagged_file) or download: 1016 data['database_package_confl_tagged_file'] = \ 1017 database_package_confl_tagged_file 1018 if not download: 1019 critical.append(data['database_package_confl_tagged_file']) 1020 1021 database_license_whitelist_file = \ 1022 self.Entropy.get_local_database_licensewhitelist_file(repo) 1023 extra_text_files.append(database_license_whitelist_file) 1024 if os.path.isfile(database_license_whitelist_file) or download: 1025 data['database_license_whitelist_file'] = \ 1026 database_license_whitelist_file 1027 if not download: 1028 critical.append(data['database_license_whitelist_file']) 1029 1030 exp_based_pkgs_removal_file = \ 1031 self.Entropy.get_local_exp_based_pkgs_rm_whitelist_file(repo) 1032 extra_text_files.append(exp_based_pkgs_removal_file) 1033 if os.path.isfile(exp_based_pkgs_removal_file) or download: 1034 data['exp_based_pkgs_removal_file'] = exp_based_pkgs_removal_file 1035 if not download: 1036 critical.append(data['exp_based_pkgs_removal_file']) 1037 1038 database_rss_file = self.Entropy.get_local_database_rss_file(repo) 1039 if os.path.isfile(database_rss_file) or download: 1040 data['database_rss_file'] = database_rss_file 1041 if not download: 1042 critical.append(data['database_rss_file']) 1043 database_rss_light_file = \ 1044 self.Entropy.get_local_database_rsslight_file(repo) 1045 1046 extra_text_files.append(database_rss_light_file) 1047 if os.path.isfile(database_rss_light_file) or download: 1048 data['database_rss_light_file'] = database_rss_light_file 1049 if not download: 1050 critical.append(data['database_rss_light_file']) 1051 1052 # EAPI 2,3 1053 if not download: # we don't need to get the dump 1054 if 3 not in disabled_eapis: 1055 data['metafiles_path'] = \ 1056 self.Entropy.get_local_database_compressed_metafiles_file( 1057 repo) 1058 critical.append(data['metafiles_path']) 1059 if 2 not in disabled_eapis: 1060 data['dump_path'] = os.path.join( 1061 self.Entropy.get_local_database_dir(repo), 1062 etpConst[cmethod[3]]) 1063 critical.append(data['dump_path']) 1064 1065 data['dump_path_digest'] = os.path.join( 1066 self.Entropy.get_local_database_dir(repo), 1067 etpConst[cmethod[4]]) 1068 critical.append(data['dump_path_digest']) 1069 1070 data['dump_path_light'] = os.path.join( 1071 self.Entropy.get_local_database_dir(repo), 1072 etpConst[cmethod[5]]) 1073 critical.append(data['dump_path_light']) 1074 1075 data['dump_path_digest_light'] = os.path.join( 1076 self.Entropy.get_local_database_dir(repo), 1077 etpConst[cmethod[6]]) 1078 critical.append(data['dump_path_digest_light']) 1079 1080 data['database_path'] = \ 1081 self.Entropy.get_local_database_file(repo) 1082 critical.append(data['database_path']) 1083 1084 # EAPI 1 1085 if 1 not in disabled_eapis: 1086 data['compressed_database_path'] = os.path.join( 1087 self.Entropy.get_local_database_dir(repo), etpConst[cmethod[2]]) 1088 critical.append(data['compressed_database_path']) 1089 data['compressed_database_path_digest'] = os.path.join( 1090 self.Entropy.get_local_database_dir(repo), 1091 etpConst['etpdatabasehashfile'] 1092 ) 1093 critical.append(data['compressed_database_path_digest']) 1094 1095 # SSL cert file, just for reference 1096 ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file() 1097 extra_text_files.append(ssl_ca_cert) 1098 if os.path.isfile(ssl_ca_cert): 1099 data['ssl_ca_cert_file'] = ssl_ca_cert 1100 if not download: 1101 critical.append(ssl_ca_cert) 1102 ssl_server_cert = self.Entropy.get_local_database_server_cert_file() 1103 extra_text_files.append(ssl_server_cert) 1104 if os.path.isfile(ssl_server_cert): 1105 data['ssl_server_cert_file'] = ssl_server_cert 1106 if not download: 1107 critical.append(ssl_server_cert) 1108 1109 # Some information regarding how packages are built 1110 spm_files = [ 1111 (etpConst['spm']['global_make_conf'], "global_make_conf"), 1112 (etpConst['spm']['global_package_keywords'], 1113 "global_package_keywords"), 1114 (etpConst['spm']['global_package_use'], "global_package_use"), 1115 (etpConst['spm']['global_package_mask'], "global_package_mask"), 1116 (etpConst['spm']['global_package_unmask'], "global_package_unmask"), 1117 ] 1118 for myfile, myname in spm_files: 1119 if os.path.isfile(myfile) and os.access(myfile, os.R_OK): 1120 data[myname] = myfile 1121 extra_text_files.append(myfile) 1122 1123 make_profile = etpConst['spm']['global_make_profile'] 1124 rnd_tmp_file = self.Entropy.entropyTools.get_random_temp_file() 1125 mytmpdir = os.path.dirname(rnd_tmp_file) 1126 mytmpfile = os.path.join(mytmpdir, 1127 etpConst['spm']['global_make_profile_link_name']) 1128 extra_text_files.append(mytmpfile) 1129 1130 if os.path.islink(make_profile): 1131 mylink = os.readlink(make_profile) 1132 f_mkp = open(mytmpfile,"w") 1133 f_mkp.write(mylink) 1134 f_mkp.flush() 1135 f_mkp.close() 1136 data['global_make_profile'] = mytmpfile 1137 1138 return data, critical, extra_text_files

1139

1140 - def _show_package_sets_messages(self, repo):

1141 1142 self.Entropy.updateProgress( 1143 "[repo:%s] %s:" % ( 1144 brown(repo), 1145 blue(_("configured package sets")), 1146 ), 1147 importance = 0, 1148 type = "info", 1149 header = darkgreen(" * ") 1150 ) 1151 sets_data = self.Entropy.package_set_list(matchRepo = repo) 1152 if not sets_data: 1153 self.Entropy.updateProgress( 1154 "%s" % (_("None configured"),), 1155 importance = 0, 1156 type = "info", 1157 header = brown(" # ") 1158 ) 1159 return 1160 for s_repo, s_name, s_sets in sets_data: 1161 self.Entropy.updateProgress( 1162 blue("%s" % (s_name,)), 1163 importance = 0, 1164 type = "info", 1165 header = brown(" # ") 1166 )

1167

1168 - def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):

1169 1170 self.Entropy.updateProgress( 1171 "[repo:%s|%s|%s:%s] %s" % ( 1172 brown(repo), 1173 darkgreen(crippled_uri), 1174 red("EAPI"), 1175 bold("3"), 1176 blue(_("preparing uncompressed database for the upload")), 1177 ), 1178 importance = 0, 1179 type = "info", 1180 header = darkgreen(" * ") 1181 ) 1182 self.Entropy.updateProgress( 1183 "%s: %s" % (_("database path"), blue(database_path),), 1184 importance = 0, 1185 type = "info", 1186 header = brown(" # ") 1187 )

1188

1189 - def _show_eapi2_upload_messages(self, crippled_uri, database_path, 1190 upload_data, cmethod, repo):

1191 1192 if repo == None: 1193 repo = self.Entropy.default_repository 1194 1195 self.Entropy.updateProgress( 1196 "[repo:%s|%s|%s:%s] %s" % ( 1197 brown(repo), 1198 darkgreen(crippled_uri), 1199 red("EAPI"), 1200 bold("2"), 1201 blue(_("creating compressed database dump + checksum")), 1202 ), 1203 importance = 0, 1204 type = "info", 1205 header = darkgreen(" * ") 1206 ) 1207 self.Entropy.updateProgress( 1208 "%s: %s" % (_("database path"), blue(database_path),), 1209 importance = 0, 1210 type = "info", 1211 header = brown(" # ") 1212 ) 1213 self.Entropy.updateProgress( 1214 "%s: %s" % (_("dump"), blue(upload_data['dump_path']),), 1215 importance = 0, 1216 type = "info", 1217 header = brown(" # ") 1218 ) 1219 self.Entropy.updateProgress( 1220 "%s: %s" % ( 1221 _("dump checksum"), 1222 blue(upload_data['dump_path_digest']), 1223 ), 1224 importance = 0, 1225 type = "info", 1226 header = brown(" # ") 1227 ) 1228 self.Entropy.updateProgress( 1229 "%s: %s" % ( 1230 _("dump light"), 1231 blue(upload_data['dump_path_light']), 1232 ), 1233 importance = 0, 1234 type = "info", 1235 header = brown(" # ") 1236 ) 1237 self.Entropy.updateProgress( 1238 "%s: %s" % ( 1239 _("dump light checksum"), 1240 blue(upload_data['dump_path_digest_light']), 1241 ), 1242 importance = 0, 1243 type = "info", 1244 header = brown(" # ") 1245 ) 1246 1247 self.Entropy.updateProgress( 1248 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1249 importance = 0, 1250 type = "info", 1251 header = brown(" # ") 1252 )

1253

1254 - def _show_eapi1_upload_messages(self, crippled_uri, database_path, 1255 upload_data, cmethod, repo):

1256 1257 self.Entropy.updateProgress( 1258 "[repo:%s|%s|%s:%s] %s" % ( 1259 brown(repo), 1260 darkgreen(crippled_uri), 1261 red("EAPI"), 1262 bold("1"), 1263 blue(_("compressing database + checksum")), 1264 ), 1265 importance = 0, 1266 type = "info", 1267 header = darkgreen(" * "), 1268 back = True 1269 ) 1270 self.Entropy.updateProgress( 1271 "%s: %s" % (_("database path"), blue(database_path),), 1272 importance = 0, 1273 type = "info", 1274 header = brown(" # ") 1275 ) 1276 self.Entropy.updateProgress( 1277 "%s: %s" % ( 1278 _("compressed database path"), 1279 blue(upload_data['compressed_database_path']), 1280 ), 1281 importance = 0, 1282 type = "info", 1283 header = brown(" # ") 1284 ) 1285 self.Entropy.updateProgress( 1286 "%s: %s" % ( 1287 _("compressed checksum"), 1288 blue(upload_data['compressed_database_path_digest']), 1289 ), 1290 importance = 0, 1291 type = "info", 1292 header = brown(" # ") 1293 ) 1294 self.Entropy.updateProgress( 1295 "%s: %s" % (_("opener"), blue(str(cmethod[0])),), 1296 importance = 0, 1297 type = "info", 1298 header = brown(" # ") 1299 )

1300

1301 - def _create_metafiles_file(self, compressed_dest_path, file_list, repo):

1302 1303 found_file_list = [x for x in file_list if os.path.isfile(x) and \ 1304 os.access(x, os.F_OK) and os.access(x, os.R_OK)] 1305 not_found_file_list = ["%s\n" % (os.path.basename(x),) for x in \ 1306 file_list if x not in found_file_list] 1307 metafile_not_found_file = \ 1308 self.Entropy.get_local_database_metafiles_not_found_file(repo) 1309 1310 f_meta = open(metafile_not_found_file,"w") 1311 f_meta.writelines(not_found_file_list) 1312 f_meta.flush() 1313 f_meta.close() 1314 found_file_list.append(metafile_not_found_file) 1315 if os.path.isfile(compressed_dest_path): 1316 os.remove(compressed_dest_path) 1317 self.entropyTools.compress_files(compressed_dest_path, found_file_list)

1318

1319 - def create_mirror_directories(self, ftp_connection, path_to_create):

1320 bdir = "" 1321 for mydir in path_to_create.split("/"): 1322 bdir += "/"+mydir 1323 if not ftp_connection.is_file_available(bdir): 1324 try: 1325 ftp_connection.mkdir(bdir) 1326 except Exception, err: 1327 error = unicode(err) 1328 if (error.find("550") == -1) and \ 1329 (error.find("File exist") == -1): 1330 1331 mytxt = "%s %s, %s: %s" % ( 1332 _("cannot create mirror directory"), 1333 bdir, 1334 _("error"), 1335 err, 1336 ) 1337 raise OnlineMirrorError("OnlineMirrorError: %s" % ( 1338 mytxt,)) 1339 raise

1340

1341 - def mirror_lock_check(self, uri, repo = None):

1342 1343 if repo == None: 1344 repo = self.Entropy.default_repository 1345 1346 gave_up = False 1347 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1348 try: 1349 ftp = FtpInterface(uri, self.Entropy) 1350 except ConnectionError: 1351 self.entropyTools.print_traceback() 1352 return True # gave up 1353 my_path = os.path.join( 1354 self.Entropy.get_remote_database_relative_path(repo), 1355 self.SystemSettings['repositories']['branch']) 1356 ftp.set_cwd(my_path, dodir = True) 1357 1358 lock_file = self.get_database_lockfile(repo) 1359 lock_filename = os.path.basename(lock_file) 1360 if not os.path.isfile(lock_file) and \ 1361 ftp.is_file_available(lock_filename): 1362 1363 self.Entropy.updateProgress( 1364 "[repo:%s|%s|%s] %s, %s" % ( 1365 brown(str(repo)), 1366 darkgreen(crippled_uri), 1367 red(_("locking")), 1368 darkblue(_("mirror already locked")), 1369 blue(_("waiting up to 2 minutes before giving up")), 1370 ), 1371 importance = 1, 1372 type = "warning", 1373 header = brown(" * "), 1374 back = True 1375 ) 1376 unlocked = False 1377 count = 0 1378 while count < 120: 1379 count += 1 1380 time.sleep(1) 1381 if not ftp.is_file_available(os.path.basename(lock_file)): 1382 self.Entropy.updateProgress( 1383 red("[repo:%s|%s|%s] %s !" % ( 1384 repo, 1385 crippled_uri, 1386 _("locking"), 1387 _("mirror unlocked"), 1388 ) 1389 ), 1390 importance = 1, 1391 type = "info", 1392 header = darkgreen(" * ") 1393 ) 1394 unlocked = True 1395 break 1396 if not unlocked: 1397 gave_up = True 1398 1399 ftp.close() 1400 return gave_up

1401

1402 - def shrink_database_and_close(self, repo = None):

1403 dbconn = self.Entropy.open_server_repository(read_only = False, 1404 no_upload = True, repo = repo, indexing = False) 1405 dbconn.dropAllIndexes() 1406 dbconn.vacuum() 1407 dbconn.vacuum() 1408 dbconn.commitChanges() 1409 self.Entropy.close_server_database(dbconn)

1410

1411 - def update_repository_timestamp(self, repo = None):

1412 if repo == None: 1413 repo = self.Entropy.default_repository 1414 ts_file = self.Entropy.get_local_database_timestamp_file(repo) 1415 current_ts = self.Entropy.get_current_timestamp() 1416 ts_f = open(ts_file,"w") 1417 ts_f.write(current_ts) 1418 ts_f.flush() 1419 ts_f.close()

1420

1421 - def sync_database_treeupdates(self, repo = None):

1422 1423 if repo == None: 1424 repo = self.Entropy.default_repository 1425 dbconn = self.Entropy.open_server_repository(read_only = False, 1426 no_upload = True, repo = repo) 1427 # grab treeupdates from other databases and inject 1428 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1429 server_repos = srv_set['repositories'].keys() 1430 all_actions = set() 1431 for myrepo in server_repos: 1432 1433 # avoid __default__ 1434 if myrepo == etpConst['clientserverrepoid']: 1435 continue 1436 1437 mydbc = self.Entropy.open_server_repository(just_reading = True, 1438 repo = myrepo) 1439 actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True) 1440 for data in actions: 1441 all_actions.add(data) 1442 if not actions: 1443 continue 1444 backed_up_entries = dbconn.listAllTreeUpdatesActions() 1445 try: 1446 # clear first 1447 dbconn.removeTreeUpdatesActions(repo) 1448 dbconn.insertTreeUpdatesActions(all_actions, repo) 1449 except dbapi2.Error, err: 1450 self.entropyTools.print_traceback() 1451 mytxt = "%s, %s: %s. %s" % ( 1452 _("Troubles with treeupdates"), 1453 _("error"), 1454 err, 1455 _("Bumping old data back"), 1456 ) 1457 self.Entropy.updateProgress( 1458 mytxt, 1459 importance = 1, 1460 type = "warning" 1461 ) 1462 # restore previous data 1463 dbconn.bumpTreeUpdatesActions(backed_up_entries) 1464 1465 dbconn.commitChanges() 1466 self.Entropy.close_server_database(dbconn)

1467

1468 - def upload_database(self, uris, lock_check = False, pretend = False, 1469 repo = None):

1470 1471 if repo == None: 1472 repo = self.Entropy.default_repository 1473 1474 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1475 if srv_set['rss']['enabled']: 1476 self.update_rss_feed(repo = repo) 1477 1478 upload_errors = False 1479 broken_uris = set() 1480 fine_uris = set() 1481 1482 disabled_eapis = sorted(srv_set['disabled_eapis']) 1483 1484 for uri in uris: 1485 1486 db_format = srv_set['database_file_format'] 1487 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1488 if cmethod == None: 1489 raise InvalidDataType("InvalidDataType: %s." % ( 1490 _("wrong database compression method passed"), 1491 ) 1492 ) 1493 1494 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1495 database_path = self.Entropy.get_local_database_file(repo) 1496 1497 if disabled_eapis: 1498 self.Entropy.updateProgress( 1499 "[repo:%s|%s|%s] %s: %s" % ( 1500 blue(repo), 1501 red(crippled_uri), 1502 darkgreen(_("upload")), 1503 darkred(_("disabled EAPI")), 1504 bold(', '.join(disabled_eapis)), 1505 ), 1506 importance = 1, 1507 type = "warning", 1508 header = darkgreen(" * ") 1509 ) 1510 1511 # create/update timestamp file 1512 self.update_repository_timestamp(repo) 1513 1514 upload_data, critical, text_files = self.get_files_to_sync(cmethod, 1515 repo = repo, disabled_eapis = disabled_eapis) 1516 1517 if lock_check: 1518 given_up = self.mirror_lock_check(uri, repo = repo) 1519 if given_up: 1520 upload_errors = True 1521 broken_uris.add(uri) 1522 continue 1523 1524 self.lock_mirrors_for_download(True, [uri], repo = repo) 1525 1526 self.Entropy.updateProgress( 1527 "[repo:%s|%s|%s] %s" % ( 1528 blue(repo), 1529 red(crippled_uri), 1530 darkgreen(_("upload")), 1531 darkgreen(_("preparing to upload database to mirror")), 1532 ), 1533 importance = 1, 1534 type = "info", 1535 header = darkgreen(" * ") 1536 ) 1537 1538 # Package Sets info 1539 self._show_package_sets_messages(repo) 1540 1541 self.sync_database_treeupdates(repo) 1542 self.Entropy.update_database_package_sets(repo) 1543 self.Entropy.close_server_databases() 1544 1545 # backup current database to avoid re-indexing 1546 old_dbpath = self.Entropy.get_local_database_file(repo) 1547 backup_dbpath = old_dbpath + ".up_backup" 1548 copy_back = False 1549 if not pretend: 1550 try: 1551 if os.access(backup_dbpath, os.R_OK | os.F_OK): 1552 os.remove(backup_dbpath) 1553 shutil.copy2(old_dbpath, backup_dbpath) 1554 copy_back = True 1555 except shutil.Error: 1556 copy_back = False 1557 1558 self.shrink_database_and_close(repo) 1559 1560 # EAPI 3 1561 if 3 not in disabled_eapis: 1562 self._create_metafiles_file(upload_data['metafiles_path'], 1563 text_files, repo) 1564 self._show_eapi3_upload_messages(crippled_uri, database_path, 1565 repo) 1566 1567 # EAPI 2 1568 if 2 not in disabled_eapis: 1569 self._show_eapi2_upload_messages(crippled_uri, database_path, 1570 upload_data, cmethod, repo) 1571 1572 # create compressed dump + checksum 1573 self.dump_database_to_file(database_path, 1574 upload_data['dump_path'], cmethod[0], repo = repo) 1575 self.create_file_checksum(upload_data['dump_path'], 1576 upload_data['dump_path_digest']) 1577 1578 self.dump_database_to_file(database_path, 1579 upload_data['dump_path_light'], cmethod[0], 1580 exclude_tables = ["content"], repo = repo) 1581 self.create_file_checksum(upload_data['dump_path_light'], 1582 upload_data['dump_path_digest_light']) 1583 1584 1585 # EAPI 1 1586 if 1 not in disabled_eapis: 1587 self._show_eapi1_upload_messages(crippled_uri, database_path, 1588 upload_data, cmethod, repo) 1589 # compress the database 1590 self.compress_file(database_path, 1591 upload_data['compressed_database_path'], cmethod[0]) 1592 self.create_file_checksum(database_path, 1593 upload_data['compressed_database_path_digest']) 1594 1595 if not pretend: 1596 # upload 1597 uploader = self.FtpServerHandler( 1598 FtpInterface, 1599 self.Entropy, 1600 [uri], 1601 [upload_data[x] for x in upload_data], 1602 critical_files = critical, 1603 repo = repo 1604 ) 1605 errors, m_fine_uris, m_broken_uris = uploader.go() 1606 if errors: 1607 my_broken_uris = sorted([ 1608 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1609 x[1]) for x in m_broken_uris]) 1610 self.Entropy.updateProgress( 1611 "[repo:%s|%s|%s] %s" % ( 1612 repo, 1613 crippled_uri, 1614 _("errors"), 1615 _("upload failed, not unlocking and continuing"), 1616 ), 1617 importance = 0, 1618 type = "error", 1619 header = darkred(" !!! ") 1620 ) 1621 # get reason 1622 reason = my_broken_uris[0][1] 1623 self.Entropy.updateProgress( 1624 blue("%s: %s" % (_("reason"), reason,)), 1625 importance = 0, 1626 type = "error", 1627 header = blue(" # ") 1628 ) 1629 upload_errors = True 1630 broken_uris |= m_broken_uris 1631 continue 1632 1633 # copy db back 1634 if copy_back and os.path.isfile(backup_dbpath): 1635 self.Entropy.close_server_databases() 1636 further_backup_dbpath = old_dbpath+".security_backup" 1637 if os.path.isfile(further_backup_dbpath): 1638 os.remove(further_backup_dbpath) 1639 shutil.copy2(old_dbpath, further_backup_dbpath) 1640 shutil.move(backup_dbpath, old_dbpath) 1641 1642 # unlock 1643 self.lock_mirrors_for_download(False, [uri], repo = repo) 1644 fine_uris |= m_fine_uris 1645 1646 if not fine_uris: 1647 upload_errors = True 1648 return upload_errors, broken_uris, fine_uris

1649 1650

1651 - def download_database(self, uris, lock_check = False, pretend = False, 1652 repo = None):

1653 1654 if repo == None: 1655 repo = self.Entropy.default_repository 1656 1657 download_errors = False 1658 broken_uris = set() 1659 fine_uris = set() 1660 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 1661 disabled_eapis = sorted(srv_set['disabled_eapis']) 1662 1663 for uri in uris: 1664 1665 db_format = srv_set['database_file_format'] 1666 cmethod = etpConst['etpdatabasecompressclasses'].get(db_format) 1667 if cmethod == None: 1668 raise InvalidDataType("InvalidDataType: %s." % ( 1669 _("wrong database compression method passed"), 1670 ) 1671 ) 1672 1673 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 1674 database_path = self.Entropy.get_local_database_file(repo) 1675 database_dir_path = os.path.dirname( 1676 self.Entropy.get_local_database_file(repo)) 1677 download_data, critical, text_files = self.get_files_to_sync( 1678 cmethod, download = True, 1679 repo = repo, disabled_eapis = disabled_eapis) 1680 mytmpdir = self.entropyTools.get_random_temp_file() 1681 os.makedirs(mytmpdir) 1682 1683 self.Entropy.updateProgress( 1684 "[repo:%s|%s|%s] %s" % ( 1685 brown(repo), 1686 darkgreen(crippled_uri), 1687 red(_("download")), 1688 blue(_("preparing to download database from mirror")), 1689 ), 1690 importance = 1, 1691 type = "info", 1692 header = darkgreen(" * ") 1693 ) 1694 files_to_sync = sorted(download_data.keys()) 1695 for myfile in files_to_sync: 1696 self.Entropy.updateProgress( 1697 "%s: %s" % ( 1698 blue(_("download path")), 1699 brown(unicode(download_data[myfile])), 1700 ), 1701 importance = 0, 1702 type = "info", 1703 header = brown(" # ") 1704 ) 1705 1706 if lock_check: 1707 given_up = self.mirror_lock_check(uri, repo = repo) 1708 if given_up: 1709 download_errors = True 1710 broken_uris.add(uri) 1711 continue 1712 1713 # avoid having others messing while we're downloading 1714 self.lock_mirrors(True, [uri], repo = repo) 1715 1716 if not pretend: 1717 # download 1718 downloader = self.FtpServerHandler( 1719 FtpInterface, self.Entropy, [uri], 1720 [download_data[x] for x in download_data], download = True, 1721 local_basedir = mytmpdir, critical_files = critical, 1722 repo = repo 1723 ) 1724 errors, m_fine_uris, m_broken_uris = downloader.go() 1725 if errors: 1726 my_broken_uris = sorted([ 1727 (self.entropyTools.extract_ftp_host_from_uri(x[0]), 1728 x[1]) for x in m_broken_uris]) 1729 self.Entropy.updateProgress( 1730 "[repo:%s|%s|%s] %s" % ( 1731 brown(repo), 1732 darkgreen(crippled_uri), 1733 red(_("errors")), 1734 blue(_("failed to download from mirror")), 1735 ), 1736 importance = 0, 1737 type = "error", 1738 header = darkred(" !!! ") 1739 ) 1740 # get reason 1741 reason = my_broken_uris[0][1] 1742 self.Entropy.updateProgress( 1743 blue("%s: %s" % (_("reason"), reason,)), 1744 importance = 0, 1745 type = "error", 1746 header = blue(" # ") 1747 ) 1748 download_errors = True 1749 broken_uris |= m_broken_uris 1750 self.lock_mirrors(False, [uri], repo = repo) 1751 continue 1752 1753 # all fine then, we need to move data from mytmpdir 1754 # to database_dir_path 1755 1756 # EAPI 1 -- unpack database 1757 if 1 not in disabled_eapis: 1758 compressed_db_filename = os.path.basename( 1759 download_data['compressed_database_path']) 1760 uncompressed_db_filename = os.path.basename(database_path) 1761 compressed_file = os.path.join(mytmpdir, 1762 compressed_db_filename) 1763 uncompressed_file = os.path.join(mytmpdir, 1764 uncompressed_db_filename) 1765 self.entropyTools.uncompress_file(compressed_file, 1766 uncompressed_file, cmethod[0]) 1767 1768 # now move 1769 for myfile in os.listdir(mytmpdir): 1770 fromfile = os.path.join(mytmpdir, myfile) 1771 tofile = os.path.join(database_dir_path, myfile) 1772 shutil.move(fromfile, tofile) 1773 self.Entropy.ClientService.setup_default_file_perms(tofile) 1774 1775 if os.path.isdir(mytmpdir): 1776 shutil.rmtree(mytmpdir) 1777 if os.path.isdir(mytmpdir): 1778 os.rmdir(mytmpdir) 1779 1780 1781 fine_uris.add(uri) 1782 self.lock_mirrors(False, [uri], repo = repo) 1783 1784 return download_errors, fine_uris, broken_uris

1785

1786 - def calculate_database_sync_queues(self, repo = None):

1787 1788 if repo == None: 1789 repo = self.Entropy.default_repository 1790 1791 remote_status = self.get_remote_databases_status(repo) 1792 local_revision = self.Entropy.get_local_database_revision(repo) 1793 upload_queue = [] 1794 download_latest = () 1795 1796 # all mirrors are empty ? I rule 1797 if not [x for x in remote_status if x[1]]: 1798 upload_queue = remote_status[:] 1799 else: 1800 highest_remote_revision = max([x[1] for x in remote_status]) 1801 1802 if local_revision < highest_remote_revision: 1803 for remote_item in remote_status: 1804 if remote_item[1] == highest_remote_revision: 1805 download_latest = x 1806 break 1807 1808 if download_latest: 1809 upload_queue = [x for x in remote_status if \ 1810 (x[1] < highest_remote_revision)] 1811 else: 1812 upload_queue = [x for x in remote_status if \ 1813 (x[1] < local_revision)] 1814 1815 return download_latest, upload_queue

1816

1817 - def sync_databases(self, no_upload = False, unlock_mirrors = False, 1818 repo = None):

1819 1820 if repo == None: 1821 repo = self.Entropy.default_repository 1822 1823 while 1: 1824 1825 db_locked = False 1826 if self.is_local_database_locked(repo): 1827 db_locked = True 1828 1829 lock_data = self.get_mirrors_lock(repo) 1830 mirrors_locked = [x for x in lock_data if x[1]] 1831 1832 if not mirrors_locked and db_locked: 1833 # mirrors not locked remotely but only locally 1834 mylock_file = self.get_database_lockfile(repo) 1835 if os.access(mylock_file, os.W_OK | os.F_OK): 1836 os.remove(mylock_file) 1837 continue 1838 1839 break 1840 1841 if mirrors_locked and not db_locked: 1842 mytxt = "%s, %s %s" % ( 1843 _("Mirrors are locked, someone is working on the repository"), 1844 _("try again later"), 1845 "...", 1846 ) 1847 raise OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,)) 1848 1849 download_latest, upload_queue = self.calculate_database_sync_queues( 1850 repo) 1851 1852 if not download_latest and not upload_queue: 1853 self.Entropy.updateProgress( 1854 "[repo:%s|%s] %s" % ( 1855 brown(repo), 1856 red(_("sync")), # something short please 1857 blue(_("database already in sync")), 1858 ), 1859 importance = 1, 1860 type = "info", 1861 header = blue(" @@ ") 1862 ) 1863 return 0, set(), set() 1864 1865 if download_latest: 1866 download_uri = download_latest[0] 1867 download_errors, fine_uris, broken_uris = self.download_database( 1868 [download_uri], repo = repo) 1869 if download_errors: 1870 self.Entropy.updateProgress( 1871 "[repo:%s|%s] %s: %s" % ( 1872 brown(repo), 1873 red(_("sync")), 1874 blue(_("database sync failed")), 1875 red(_("download issues")), 1876 ), 1877 importance = 1, 1878 type = "error", 1879 header = darkred(" !!! ") 1880 ) 1881 return 1, fine_uris, broken_uris 1882 1883 if upload_queue and not no_upload: 1884 1885 deps_not_found = self.Entropy.dependencies_test(repo = repo) 1886 if deps_not_found and not self.Entropy.community_repo: 1887 self.Entropy.updateProgress( 1888 "[repo:%s|%s] %s: %s" % ( 1889 brown(repo), 1890 red(_("sync")), 1891 blue(_("database sync forbidden")), 1892 red(_("dependencies_test() reported errors")), 1893 ), 1894 importance = 1, 1895 type = "error", 1896 header = darkred(" !!! ") 1897 ) 1898 return 3, set(), set() 1899 1900 problems = self.Entropy.check_config_file_updates() 1901 if problems: 1902 return 4, set(), set() 1903 1904 self.Entropy.updateProgress( 1905 "[repo:%s|%s] %s" % ( 1906 brown(repo), 1907 red(_("config files")), # something short please 1908 blue(_("no configuration files to commit. All fine.")), 1909 ), 1910 importance = 1, 1911 type = "info", 1912 header = blue(" @@ "), 1913 back = True 1914 ) 1915 1916 uris = [x[0] for x in upload_queue] 1917 errors, fine_uris, broken_uris = self.upload_database(uris, 1918 repo = repo) 1919 if errors: 1920 self.Entropy.updateProgress( 1921 "[repo:%s|%s] %s: %s" % ( 1922 brown(repo), 1923 red(_("sync")), 1924 blue(_("database sync failed")), 1925 red(_("upload issues")), 1926 ), 1927 importance = 1, 1928 type = "error", 1929 header = darkred(" !!! ") 1930 ) 1931 return 2, fine_uris, broken_uris 1932 1933 1934 self.Entropy.updateProgress( 1935 "[repo:%s|%s] %s" % ( 1936 brown(repo), 1937 red(_("sync")), 1938 blue(_("database sync completed successfully")), 1939 ), 1940 importance = 1, 1941 type = "info", 1942 header = darkgreen(" * ") 1943 ) 1944 1945 if unlock_mirrors: 1946 self.lock_mirrors(False, repo = repo) 1947 return 0, set(), set()

1948 1949

1950 - def calculate_local_upload_files(self, branch, repo = None):

1951 upload_files = 0 1952 upload_packages = set() 1953 upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo), 1954 branch) 1955 1956 pkg_ext = etpConst['packagesext'] 1957 pkg_md5_ext = etpConst['packagesmd5fileext'] 1958 for package in os.listdir(upload_dir): 1959 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 1960 upload_packages.add(package) 1961 if package.endswith(pkg_ext): 1962 upload_files += 1 1963 1964 return upload_files, upload_packages

1965

1966 - def calculate_local_package_files(self, branch, repo = None):

1967 local_files = 0 1968 local_packages = set() 1969 packages_dir = os.path.join( 1970 self.Entropy.get_local_packages_directory(repo), branch) 1971 1972 if not os.path.isdir(packages_dir): 1973 os.makedirs(packages_dir) 1974 1975 pkg_ext = etpConst['packagesext'] 1976 pkg_md5_ext = etpConst['packagesmd5fileext'] 1977 for package in os.listdir(packages_dir): 1978 if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext): 1979 local_packages.add(package) 1980 if package.endswith(pkg_ext): 1981 local_files += 1 1982 1983 return local_files, local_packages

1984 1985

1986 - def _show_local_sync_stats(self, upload_files, local_files):

1987 self.Entropy.updateProgress( 1988 "%s:" % ( 1989 blue(_("Local statistics")), 1990 ), 1991 importance = 1, 1992 type = "info", 1993 header = red(" @@ ") 1994 ) 1995 self.Entropy.updateProgress( 1996 red("%s:\t\t%s %s" % ( 1997 blue(_("upload directory")), 1998 bold(str(upload_files)), 1999 red(_("files ready")), 2000 ) 2001 ), 2002 importance = 0, 2003 type = "info", 2004 header = red(" @@ ") 2005 ) 2006 self.Entropy.updateProgress( 2007 red("%s:\t\t%s %s" % ( 2008 blue(_("packages directory")), 2009 bold(str(local_files)), 2010 red(_("files ready")), 2011 ) 2012 ), 2013 importance = 0, 2014 type = "info", 2015 header = red(" @@ ") 2016 )

2017

2018 - def _show_sync_queues(self, upload, download, removal, copy, metainfo, 2019 branch):

2020 2021 # show stats 2022 for package, size in upload: 2023 package = darkgreen(os.path.basename(package)) 2024 size = blue(self.entropyTools.bytes_into_human(size)) 2025 self.Entropy.updateProgress( 2026 "[branch:%s|%s] %s [%s]" % ( 2027 brown(branch), 2028 blue(_("upload")), 2029 darkgreen(package), 2030 size, 2031 ), 2032 importance = 0, 2033 type = "info", 2034 header = red(" # ") 2035 ) 2036 for package, size in download: 2037 package = darkred(os.path.basename(package)) 2038 size = blue(self.entropyTools.bytes_into_human(size)) 2039 self.Entropy.updateProgress( 2040 "[branch:%s|%s] %s [%s]" % ( 2041 brown(branch), 2042 darkred(_("download")), 2043 blue(package), 2044 size, 2045 ), 2046 importance = 0, 2047 type = "info", 2048 header = red(" # ") 2049 ) 2050 for package, size in copy: 2051 package = darkblue(os.path.basename(package)) 2052 size = blue(self.entropyTools.bytes_into_human(size)) 2053 self.Entropy.updateProgress( 2054 "[branch:%s|%s] %s [%s]" % ( 2055 brown(branch), 2056 darkgreen(_("copy")), 2057 brown(package), 2058 size, 2059 ), 2060 importance = 0, 2061 type = "info", 2062 header = red(" # ") 2063 ) 2064 for package, size in removal: 2065 package = brown(os.path.basename(package)) 2066 size = blue(self.entropyTools.bytes_into_human(size)) 2067 self.Entropy.updateProgress( 2068 "[branch:%s|%s] %s [%s]" % ( 2069 brown(branch), 2070 red(_("remove")), 2071 red(package), 2072 size, 2073 ), 2074 importance = 0, 2075 type = "info", 2076 header = red(" # ") 2077 ) 2078 2079 self.Entropy.updateProgress( 2080 "%s:\t\t\t%s" % ( 2081 blue(_("Packages to be removed")), 2082 darkred(str(len(removal))), 2083 ), 2084 importance = 0, 2085 type = "info", 2086 header = blue(" @@ ") 2087 ) 2088 self.Entropy.updateProgress( 2089 "%s:\t\t%s" % ( 2090 darkgreen(_("Packages to be moved locally")), 2091 darkgreen(str(len(copy))), 2092 ), 2093 importance = 0, 2094 type = "info", 2095 header = blue(" @@ ") 2096 ) 2097 self.Entropy.updateProgress( 2098 "%s:\t\t\t%s" % ( 2099 bold(_("Packages to be uploaded")), 2100 bold(str(len(upload))), 2101 ), 2102 importance = 0, 2103 type = "info", 2104 header = blue(" @@ ") 2105 ) 2106 2107 self.Entropy.updateProgress( 2108 "%s:\t\t\t%s" % ( 2109 darkred(_("Total removal size")), 2110 darkred( 2111 self.entropyTools.bytes_into_human(metainfo['removal']) 2112 ), 2113 ), 2114 importance = 0, 2115 type = "info", 2116 header = blue(" @@ ") 2117 ) 2118 2119 self.Entropy.updateProgress( 2120 "%s:\t\t\t%s" % ( 2121 blue(_("Total upload size")), 2122 blue(self.entropyTools.bytes_into_human(metainfo['upload'])), 2123 ), 2124 importance = 0, 2125 type = "info", 2126 header = blue(" @@ ") 2127 ) 2128 self.Entropy.updateProgress( 2129 "%s:\t\t\t%s" % ( 2130 brown(_("Total download size")), 2131 brown(self.entropyTools.bytes_into_human(metainfo['download'])), 2132 ), 2133 importance = 0, 2134 type = "info", 2135 header = blue(" @@ ") 2136 )

2137 2138

2139 - def calculate_remote_package_files(self, uri, branch, ftp_connection = None, 2140 repo = None):

2141 2142 remote_files = 0 2143 close_conn = False 2144 remote_packages_data = {} 2145 2146 my_path = os.path.join( 2147 self.Entropy.get_remote_packages_relative_path(repo), branch) 2148 if ftp_connection == None: 2149 close_conn = True 2150 # let it raise an exception if things go bad 2151 ftp_connection = FtpInterface(uri, self.Entropy) 2152 ftp_connection.set_cwd(my_path, dodir = True) 2153 2154 remote_packages = ftp_connection.list_dir() 2155 remote_packages_info = ftp_connection.get_raw_list() 2156 if close_conn: 2157 ftp_connection.close() 2158 2159 for tbz2 in remote_packages: 2160 if tbz2.endswith(etpConst['packagesext']): 2161 remote_files += 1 2162 2163 for remote_package in remote_packages_info: 2164 remote_packages_data[remote_package.split()[8]] = \ 2165 int(remote_package.split()[4]) 2166 2167 return remote_files, remote_packages, remote_packages_data

2168

2169 - def calculate_packages_to_sync(self, uri, branch, repo = None):

2170 2171 if repo == None: 2172 repo = self.Entropy.default_repository 2173 2174 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2175 upload_files, upload_packages = self.calculate_local_upload_files( 2176 branch, repo) 2177 local_files, local_packages = self.calculate_local_package_files(branch, 2178 repo) 2179 self._show_local_sync_stats(upload_files, local_files) 2180 2181 self.Entropy.updateProgress( 2182 "%s: %s" % (blue(_("Remote statistics for")), red(crippled_uri),), 2183 importance = 1, 2184 type = "info", 2185 header = red(" @@ ") 2186 ) 2187 remote_files, remote_packages, remote_packages_data = \ 2188 self.calculate_remote_package_files(uri, branch, repo = repo) 2189 self.Entropy.updateProgress( 2190 "%s:\t\t\t%s %s" % ( 2191 blue(_("remote packages")), 2192 bold(str(remote_files)), 2193 red(_("files stored")), 2194 ), 2195 importance = 0, 2196 type = "info", 2197 header = red(" @@ ") 2198 ) 2199 2200 mytxt = blue("%s ...") % ( 2201 _("Calculating queues"), 2202 ) 2203 self.Entropy.updateProgress( 2204 mytxt, 2205 importance = 1, 2206 type = "info", 2207 header = red(" @@ ") 2208 ) 2209 2210 upload_queue, download_queue, removal_queue, fine_queue = \ 2211 self.calculate_sync_queues(upload_packages, local_packages, 2212 remote_packages, remote_packages_data, branch, repo) 2213 return upload_queue, download_queue, removal_queue, fine_queue, \ 2214 remote_packages_data

2215

2216 - def calculate_sync_queues( 2217 self, 2218 upload_packages, 2219 local_packages, 2220 remote_packages, 2221 remote_packages_data, 2222 branch, 2223 repo = None 2224 ):

2225 2226 upload_queue = set() 2227 download_queue = set() 2228 removal_queue = set() 2229 fine_queue = set() 2230 2231 for local_package in upload_packages: 2232 if local_package in remote_packages: 2233 2234 local_filepath = os.path.join( 2235 self.Entropy.get_local_upload_directory(repo), branch, 2236 local_package) 2237 2238 local_size = self.entropyTools.get_file_size(local_filepath) 2239 remote_size = remote_packages_data.get(local_package) 2240 if remote_size == None: 2241 remote_size = 0 2242 if (local_size != remote_size): 2243 # size does not match, adding to the upload queue 2244 upload_queue.add(local_package) 2245 else: 2246 # just move from upload to packages 2247 fine_queue.add(local_package) 2248 else: 2249 # always force upload of packages in uploaddir 2250 upload_queue.add(local_package) 2251 2252 # if a package is in the packages directory but not online, 2253 # we have to upload it we have local_packages and remote_packages 2254 for local_package in local_packages: 2255 if local_package in remote_packages: 2256 local_filepath = os.path.join( 2257 self.Entropy.get_local_packages_directory(repo), branch, 2258 local_package) 2259 local_size = self.entropyTools.get_file_size(local_filepath) 2260 remote_size = remote_packages_data.get(local_package) 2261 if remote_size == None: 2262 remote_size = 0 2263 if (local_size != remote_size) and (local_size != 0): 2264 # size does not match, adding to the upload queue 2265 if local_package not in fine_queue: 2266 upload_queue.add(local_package) 2267 else: 2268 # this means that the local package does not exist 2269 # so, we need to download it 2270 upload_queue.add(local_package) 2271 2272 # Fill download_queue and removal_queue 2273 for remote_package in remote_packages: 2274 if remote_package in local_packages: 2275 local_filepath = os.path.join( 2276 self.Entropy.get_local_packages_directory(repo), branch, 2277 remote_package) 2278 local_size = self.entropyTools.get_file_size(local_filepath) 2279 remote_size = remote_packages_data.get(remote_package) 2280 if remote_size == None: 2281 remote_size = 0 2282 if (local_size != remote_size) and (local_size != 0): 2283 # size does not match, remove first 2284 # do it only if the package has not been 2285 # added to the upload_queue 2286 if remote_package not in upload_queue: 2287 # remotePackage == localPackage 2288 # just remove something that differs 2289 # from the content of the mirror 2290 removal_queue.add(remote_package) 2291 # then add to the download queue 2292 download_queue.add(remote_package) 2293 else: 2294 # this means that the local package does not exist 2295 # so, we need to download it 2296 # ignore .tmp files 2297 if not remote_package.endswith(".tmp"): 2298 download_queue.add(remote_package) 2299 2300 # Collect packages that don't exist anymore in the database 2301 # so we can filter them out from the download queue 2302 dbconn = self.Entropy.open_server_repository(just_reading = True, 2303 repo = repo) 2304 db_files = dbconn.listBranchPackagesTbz2(branch, do_sort = False, 2305 full_path = True) 2306 db_files = set([os.path.basename(x) for x in db_files if \ 2307 (self.Entropy.get_branch_from_download_relative_uri(x) == branch)]) 2308 2309 exclude = set() 2310 for myfile in download_queue: 2311 if myfile.endswith(etpConst['packagesext']): 2312 if myfile not in db_files: 2313 exclude.add(myfile) 2314 download_queue -= exclude 2315 2316 exclude = set() 2317 for myfile in upload_queue: 2318 if myfile.endswith(etpConst['packagesext']): 2319 if myfile not in db_files: 2320 exclude.add(myfile) 2321 upload_queue -= exclude 2322 2323 exclude = set() 2324 for myfile in download_queue: 2325 if myfile in upload_queue: 2326 exclude.add(myfile) 2327 download_queue -= exclude 2328 2329 return upload_queue, download_queue, removal_queue, fine_queue

2330 2331

2332 - def expand_queues(self, upload_queue, download_queue, removal_queue, 2333 remote_packages_data, branch, repo):

2334 2335 metainfo = { 2336 'removal': 0, 2337 'download': 0, 2338 'upload': 0, 2339 } 2340 removal = [] 2341 download = [] 2342 do_copy = [] 2343 upload = [] 2344 2345 for item in removal_queue: 2346 if not item.endswith(etpConst['packagesext']): 2347 continue 2348 local_filepath = os.path.join( 2349 self.Entropy.get_local_packages_directory(repo), branch, item) 2350 size = self.entropyTools.get_file_size(local_filepath) 2351 metainfo['removal'] += size 2352 removal.append((local_filepath, size)) 2353 2354 for item in download_queue: 2355 if not item.endswith(etpConst['packagesext']): 2356 continue 2357 local_filepath = os.path.join( 2358 self.Entropy.get_local_upload_directory(repo), branch, item) 2359 if not os.path.isfile(local_filepath): 2360 size = remote_packages_data.get(item) 2361 if size == None: 2362 size = 0 2363 size = int(size) 2364 metainfo['removal'] += size 2365 download.append((local_filepath, size)) 2366 else: 2367 size = self.entropyTools.get_file_size(local_filepath) 2368 do_copy.append((local_filepath, size)) 2369 2370 for item in upload_queue: 2371 if not item.endswith(etpConst['packagesext']): 2372 continue 2373 local_filepath = os.path.join( 2374 self.Entropy.get_local_upload_directory(repo), branch, item) 2375 local_filepath_pkgs = os.path.join( 2376 self.Entropy.get_local_packages_directory(repo), branch, item) 2377 if os.path.isfile(local_filepath): 2378 size = self.entropyTools.get_file_size(local_filepath) 2379 upload.append((local_filepath, size)) 2380 else: 2381 size = self.entropyTools.get_file_size(local_filepath_pkgs) 2382 upload.append((local_filepath_pkgs, size)) 2383 metainfo['upload'] += size 2384 2385 return upload, download, removal, do_copy, metainfo

2386 2387

2388 - def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

2389 2390 if repo == None: 2391 repo = self.Entropy.default_repository 2392 2393 for itemdata in removal_queue: 2394 2395 remove_filename = itemdata[0] 2396 remove_filepath = os.path.join( 2397 self.Entropy.get_local_packages_directory(repo), branch, 2398 remove_filename) 2399 remove_filepath_hash = remove_filepath + \ 2400 etpConst['packagesmd5fileext'] 2401 self.Entropy.updateProgress( 2402 "[repo:%s|%s|%s] %s: %s [%s]" % ( 2403 brown(repo), 2404 red("sync"), 2405 brown(branch), 2406 blue(_("removing package+hash")), 2407 darkgreen(remove_filename), 2408 blue(self.entropyTools.bytes_into_human(itemdata[1])), 2409 ), 2410 importance = 0, 2411 type = "info", 2412 header = darkred(" * ") 2413 ) 2414 2415 if os.path.isfile(remove_filepath): 2416 os.remove(remove_filepath) 2417 if os.path.isfile(remove_filepath_hash): 2418 os.remove(remove_filepath_hash) 2419 2420 self.Entropy.updateProgress( 2421 "[repo:%s|%s|%s] %s" % ( 2422 brown(repo), 2423 red(_("sync")), 2424 brown(branch), 2425 blue(_("removal complete")), 2426 ), 2427 importance = 0, 2428 type = "info", 2429 header = darkred(" * ") 2430 )

2431 2432

2433 - def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

2434 2435 if repo == None: 2436 repo = self.Entropy.default_repository 2437 2438 for itemdata in copy_queue: 2439 2440 from_file = itemdata[0] 2441 from_file_hash = from_file + etpConst['packagesmd5fileext'] 2442 to_file = os.path.join( 2443 self.Entropy.get_local_packages_directory(repo), branch, 2444 os.path.basename(from_file)) 2445 to_file_hash = to_file+etpConst['packagesmd5fileext'] 2446 expiration_file = to_file+etpConst['packagesexpirationfileext'] 2447 self.Entropy.updateProgress( 2448 "[repo:%s|%s|%s] %s: %s" % ( 2449 brown(repo), 2450 red("sync"), 2451 brown(branch), 2452 blue(_("copying file+hash to repository")), 2453 darkgreen(from_file), 2454 ), 2455 importance = 0, 2456 type = "info", 2457 header = darkred(" * ") 2458 ) 2459 2460 if not os.path.isdir(os.path.dirname(to_file)): 2461 os.makedirs(os.path.dirname(to_file)) 2462 2463 shutil.copy2(from_file, to_file) 2464 if not os.path.isfile(from_file_hash): 2465 self.create_file_checksum(from_file, from_file_hash) 2466 shutil.copy2(from_file_hash, to_file_hash) 2467 2468 # clear expiration file 2469 if os.path.isfile(expiration_file): 2470 os.remove(expiration_file)

2471 2472

2473 - def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

2474 2475 if repo == None: 2476 repo = self.Entropy.default_repository 2477 2478 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2479 myqueue = [] 2480 for itemdata in upload_queue: 2481 upload_item = itemdata[0] 2482 hash_file = upload_item + etpConst['packagesmd5fileext'] 2483 if not os.path.isfile(hash_file): 2484 self.entropyTools.create_md5_file(upload_item) 2485 myqueue.append(hash_file) 2486 myqueue.append(upload_item) 2487 2488 ftp_basedir = os.path.join( 2489 self.Entropy.get_remote_packages_relative_path(repo), branch) 2490 uploader = self.FtpServerHandler(FtpInterface, 2491 self.Entropy, [uri], myqueue, critical_files = myqueue, 2492 use_handlers = True, ftp_basedir = ftp_basedir, 2493 handlers_data = {'branch': branch }, repo = repo) 2494 errors, m_fine_uris, m_broken_uris = uploader.go() 2495 if errors: 2496 my_broken_uris = [ 2497 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) for \ 2498 x in m_broken_uris] 2499 reason = my_broken_uris[0][1] 2500 self.Entropy.updateProgress( 2501 "[branch:%s] %s: %s, %s: %s" % ( 2502 brown(branch), 2503 blue(_("upload errors")), 2504 red(crippled_uri), 2505 blue(_("reason")), 2506 darkgreen(unicode(reason)), 2507 ), 2508 importance = 1, 2509 type = "error", 2510 header = darkred(" !!! ") 2511 ) 2512 return errors, m_fine_uris, m_broken_uris 2513 2514 self.Entropy.updateProgress( 2515 "[branch:%s] %s: %s" % ( 2516 brown(branch), 2517 blue(_("upload completed successfully")), 2518 red(crippled_uri), 2519 ), 2520 importance = 1, 2521 type = "info", 2522 header = blue(" @@ ") 2523 ) 2524 return errors, m_fine_uris, m_broken_uris

2525 2526

2527 - def _sync_run_download_queue(self, uri, download_queue, branch, 2528 repo = None):

2529 2530 if repo == None: 2531 repo = self.Entropy.default_repository 2532 2533 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2534 myqueue = [] 2535 for package in download_queue: 2536 hash_file = package + etpConst['packagesmd5fileext'] 2537 myqueue.append(package) 2538 myqueue.append(hash_file) 2539 2540 ftp_basedir = os.path.join( 2541 self.Entropy.get_remote_packages_relative_path(repo), branch) 2542 local_basedir = os.path.join( 2543 self.Entropy.get_local_packages_directory(repo), branch) 2544 downloader = self.FtpServerHandler( 2545 FtpInterface, self.Entropy, [uri], myqueue, 2546 critical_files = myqueue, use_handlers = True, 2547 ftp_basedir = ftp_basedir, local_basedir = local_basedir, 2548 handlers_data = {'branch': branch }, download = True, repo = repo) 2549 2550 errors, m_fine_uris, m_broken_uris = downloader.go() 2551 if errors: 2552 my_broken_uris = [ 2553 (self.entropyTools.extract_ftp_host_from_uri(x), y,) \ 2554 for x, y in m_broken_uris] 2555 reason = my_broken_uris[0][1] 2556 self.Entropy.updateProgress( 2557 "[repo:%s|%s|%s] %s: %s, %s: %s" % ( 2558 brown(repo), 2559 red(_("sync")), 2560 brown(branch), 2561 blue(_("download errors")), 2562 darkgreen(crippled_uri), 2563 blue(_("reason")), 2564 reason, 2565 ), 2566 importance = 1, 2567 type = "error", 2568 header = darkred(" !!! ") 2569 ) 2570 return errors, m_fine_uris, m_broken_uris 2571 2572 self.Entropy.updateProgress( 2573 "[repo:%s|%s|%s] %s: %s" % ( 2574 brown(repo), 2575 red(_("sync")), 2576 brown(branch), 2577 blue(_("download completed successfully")), 2578 darkgreen(crippled_uri), 2579 ), 2580 importance = 1, 2581 type = "info", 2582 header = darkgreen(" * ") 2583 ) 2584 return errors, m_fine_uris, m_broken_uris

2585 2586

2587 - def sync_packages(self, ask = True, pretend = False, packages_check = False, 2588 repo = None):

2589 2590 if repo == None: 2591 repo = self.Entropy.default_repository 2592 2593 self.Entropy.updateProgress( 2594 "[repo:%s|%s] %s" % ( 2595 repo, 2596 red(_("sync")), 2597 darkgreen(_("starting packages sync")), 2598 ), 2599 importance = 1, 2600 type = "info", 2601 header = red(" @@ "), 2602 back = True 2603 ) 2604 2605 successfull_mirrors = set() 2606 broken_mirrors = set() 2607 check_data = () 2608 mirrors_tainted = False 2609 mirror_errors = False 2610 mirrors_errors = False 2611 2612 for uri in self.Entropy.get_remote_mirrors(repo): 2613 2614 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 2615 mirror_errors = False 2616 2617 self.Entropy.updateProgress( 2618 "[repo:%s|%s|branch:%s] %s: %s" % ( 2619 repo, 2620 red(_("sync")), 2621 brown(self.SystemSettings['repositories']['branch']), 2622 blue(_("packages sync")), 2623 bold(crippled_uri), 2624 ), 2625 importance = 1, 2626 type = "info", 2627 header = red(" @@ ") 2628 ) 2629 2630 try: 2631 upload_queue, download_queue, removal_queue, fine_queue, \ 2632 remote_packages_data = self.calculate_packages_to_sync(uri, 2633 self.SystemSettings['repositories']['branch'], repo) 2634 except self.socket.error, err: 2635 self.Entropy.updateProgress( 2636 "[repo:%s|%s|branch:%s] %s: %s, %s %s" % ( 2637 repo, 2638 red(_("sync")), 2639 self.SystemSettings['repositories']['branch'], 2640 darkred(_("socket error")), 2641 err, 2642 darkred(_("on")), 2643 crippled_uri, 2644 ), 2645 importance = 1, 2646 type = "error", 2647 header = darkgreen(" * ") 2648 ) 2649 continue 2650 2651 if (not upload_queue) and (not download_queue) and \ 2652 (not removal_queue): 2653 self.Entropy.updateProgress( 2654 "[repo:%s|%s|branch:%s] %s: %s" % ( 2655 repo, 2656 red(_("sync")), 2657 self.SystemSettings['repositories']['branch'], 2658 darkgreen(_("nothing to do on")), 2659 crippled_uri, 2660 ), 2661 importance = 1, 2662 type = "info", 2663 header = darkgreen(" * ") 2664 ) 2665 successfull_mirrors.add(uri) 2666 continue 2667 2668 self.Entropy.updateProgress( 2669 "%s:" % (blue(_("Expanding queues")),), 2670 importance = 1, 2671 type = "info", 2672 header = red(" ** ") 2673 ) 2674 2675 upload, download, removal, copy, metainfo = self.expand_queues( 2676 upload_queue, 2677 download_queue, 2678 removal_queue, 2679 remote_packages_data, 2680 self.SystemSettings['repositories']['branch'], 2681 repo 2682 ) 2683 del upload_queue, download_queue, removal_queue, \ 2684 remote_packages_data 2685 self._show_sync_queues(upload, download, removal, copy, metainfo, 2686 self.SystemSettings['repositories']['branch']) 2687 2688 if not len(upload)+len(download)+len(removal)+len(copy): 2689 2690 self.Entropy.updateProgress( 2691 "[repo:%s|%s|branch:%s] %s %s" % ( 2692 self.Entropy.default_repository, 2693 red(_("sync")), 2694 self.SystemSettings['repositories']['branch'], 2695 blue(_("nothing to sync for")), 2696 crippled_uri, 2697 ), 2698 importance = 1, 2699 type = "info", 2700 header = darkgreen(" @@ ") 2701 ) 2702 2703 successfull_mirrors.add(uri) 2704 continue 2705 2706 if pretend: 2707 successfull_mirrors.add(uri) 2708 continue 2709 2710 if ask: 2711 rc_sync = self.Entropy.askQuestion( 2712 _("Would you like to run the steps above ?")) 2713 if rc_sync == "No": 2714 continue 2715 2716 try: 2717 2718 if removal: 2719 self._sync_run_removal_queue(removal, 2720 self.SystemSettings['repositories']['branch'], repo) 2721 2722 if copy: 2723 self._sync_run_copy_queue(copy, 2724 self.SystemSettings['repositories']['branch'], repo) 2725 2726 if upload or download: 2727 mirrors_tainted = True 2728 2729 if upload: 2730 d_errors, m_fine_uris, \ 2731 m_broken_uris = self._sync_run_upload_queue( 2732 uri, upload, 2733 self.SystemSettings['repositories']['branch'], repo) 2734 2735 if d_errors: 2736 mirror_errors = True 2737 2738 if download: 2739 my_downlist = [x[0] for x in download] 2740 d_errors, m_fine_uris, \ 2741 m_broken_uris = self._sync_run_download_queue( 2742 uri, my_downlist, 2743 self.SystemSettings['repositories']['branch'], repo) 2744 2745 if d_errors: 2746 mirror_errors = True 2747 if not mirror_errors: 2748 successfull_mirrors.add(uri) 2749 else: 2750 mirrors_errors = True 2751 2752 except KeyboardInterrupt: 2753 self.Entropy.updateProgress( 2754 "[repo:%s|%s|branch:%s] %s" % ( 2755 repo, 2756 red(_("sync")), 2757 self.SystemSettings['repositories']['branch'], 2758 darkgreen(_("keyboard interrupt !")), 2759 ), 2760 importance = 1, 2761 type = "info", 2762 header = darkgreen(" * ") 2763 ) 2764 continue 2765 2766 except Exception, err: 2767 2768 self.entropyTools.print_traceback() 2769 mirrors_errors = True 2770 broken_mirrors.add(uri) 2771 self.Entropy.updateProgress( 2772 "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % ( 2773 repo, 2774 red(_("sync")), 2775 self.SystemSettings['repositories']['branch'], 2776 darkred(_("exception caught")), 2777 Exception, 2778 _("error"), 2779 err, 2780 ), 2781 importance = 1, 2782 type = "error", 2783 header = darkred(" !!! ") 2784 ) 2785 2786 exc_txt = self.Entropy.entropyTools.print_exception( 2787 returndata = True) 2788 for line in exc_txt: 2789 self.Entropy.updateProgress( 2790 unicode(line), 2791 importance = 1, 2792 type = "error", 2793 header = darkred(": ") 2794 ) 2795 2796 if len(successfull_mirrors) > 0: 2797 self.Entropy.updateProgress( 2798 "[repo:%s|%s|branch:%s] %s" % ( 2799 repo, 2800 red(_("sync")), 2801 self.SystemSettings['repositories']['branch'], 2802 darkred( 2803 _("at least one mirror synced properly!")), 2804 ), 2805 importance = 1, 2806 type = "error", 2807 header = darkred(" !!! ") 2808 ) 2809 continue 2810 2811 # if at least one server has been synced successfully, move files 2812 if (len(successfull_mirrors) > 0) and not pretend: 2813 self.remove_expiration_files( 2814 self.SystemSettings['repositories']['branch'], repo) 2815 2816 if packages_check: 2817 check_data = self.Entropy.verify_local_packages([], ask = ask, 2818 repo = repo) 2819 2820 return mirrors_tainted, mirrors_errors, successfull_mirrors, \ 2821 broken_mirrors, check_data

2822

2823 - def remove_expiration_files(self, branch, repo = None):

2824 2825 if repo == None: 2826 repo = self.Entropy.default_repository 2827 2828 branch_dir = os.path.join( 2829 self.Entropy.get_local_upload_directory(repo), branch) 2830 branchcontent = os.listdir(branch_dir) 2831 for xfile in branchcontent: 2832 source = os.path.join(self.Entropy.get_local_upload_directory(repo), 2833 branch, xfile) 2834 destdir = os.path.join( 2835 self.Entropy.get_local_packages_directory(repo), branch) 2836 if not os.path.isdir(destdir): 2837 os.makedirs(destdir) 2838 dest = os.path.join(destdir, xfile) 2839 shutil.move(source, dest) 2840 # clear expiration file 2841 dest_expiration = dest + etpConst['packagesexpirationfileext'] 2842 if os.path.isfile(dest_expiration): 2843 os.remove(dest_expiration)

2844 2845

2846 - def is_package_expired(self, package_file, branch, repo = None):

2847 2848 pkg_path = os.path.join( 2849 self.Entropy.get_local_packages_directory(repo), branch, 2850 package_file) 2851 pkg_path += etpConst['packagesexpirationfileext'] 2852 if not os.path.isfile(pkg_path): 2853 return False 2854 2855 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2856 mtime = self.entropyTools.get_file_unix_mtime(pkg_path) 2857 days = srv_set['packages_expiration_days'] 2858 delta = int(days)*24*3600 2859 currmtime = time.time() 2860 file_delta = currmtime - mtime 2861 2862 if file_delta > delta: 2863 return True 2864 return False

2865

2866 - def create_expiration_file(self, package_file, branch, repo = None, 2867 gentle = False):

2868 2869 pkg_path = os.path.join( 2870 self.Entropy.get_local_packages_directory(repo), branch, 2871 package_file) 2872 pkg_path += etpConst['packagesexpirationfileext'] 2873 if gentle and os.path.isfile(pkg_path): 2874 return 2875 f_exp = open(pkg_path,"w") 2876 f_exp.flush() 2877 f_exp.close()

2878 2879

2880 - def collect_expiring_packages(self, branch, repo = None):

2881 2882 dbconn = self.Entropy.open_server_repository(just_reading = True, 2883 repo = repo) 2884 database_bins = dbconn.listBranchPackagesTbz2(branch, do_sort = False, 2885 full_path = True) 2886 bins_dir = os.path.join( 2887 self.Entropy.get_local_packages_directory(repo), branch) 2888 repo_bins = set() 2889 2890 if os.path.isdir(bins_dir): 2891 repo_bins = os.listdir(bins_dir) 2892 repo_bins = set([ 2893 os.path.join('packages', etpSys['arch'], branch, x) for x \ 2894 in repo_bins if x.endswith(etpConst['packagesext'])]) 2895 repo_bins -= database_bins 2896 2897 return set([os.path.basename(x) for x in repo_bins])

2898 2899

2900 - def tidy_mirrors(self, ask = True, pretend = False, repo = None):

2901 2902 if repo == None: 2903 repo = self.Entropy.default_repository 2904 2905 self.Entropy.updateProgress( 2906 "[repo:%s|%s|branch:%s] %s" % ( 2907 brown(repo), 2908 red(_("tidy")), 2909 blue(self.SystemSettings['repositories']['branch']), 2910 blue(_("collecting expired packages")), 2911 ), 2912 importance = 1, 2913 type = "info", 2914 header = red(" @@ ") 2915 ) 2916 2917 branch_data = {} 2918 errors = False 2919 2920 branch_data['errors'] = False 2921 2922 branch = self.SystemSettings['repositories']['branch'] 2923 srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] 2924 last_branch = srv_set['branches'][-1] 2925 if branch != last_branch: 2926 mytxt = _("not the latest branch, skipping tidy for consistence.") 2927 self.Entropy.updateProgress( 2928 "[branch:%s] %s" % ( 2929 brown(branch), 2930 blue(mytxt), 2931 ), 2932 importance = 1, 2933 type = "info", 2934 header = blue(" @@ ") 2935 ) 2936 branch_data['errors'] = True 2937 return True, branch_data 2938 2939 self.Entropy.updateProgress( 2940 "[branch:%s] %s" % ( 2941 brown(branch), 2942 blue(_("collecting expired packages in the selected branches")), 2943 ), 2944 importance = 1, 2945 type = "info", 2946 header = blue(" @@ ") 2947 ) 2948 2949 # collect removed packages 2950 expiring_packages = self.collect_expiring_packages(branch, repo) 2951 2952 removal = [] 2953 for package in expiring_packages: 2954 expired = self.is_package_expired(package, branch, repo) 2955 if expired: 2956 removal.append(package) 2957 else: 2958 self.create_expiration_file(package, branch, repo, 2959 gentle = True) 2960 2961 # fill returning data 2962 branch_data['removal'] = removal[:] 2963 2964 if not removal: 2965 self.Entropy.updateProgress( 2966 "[branch:%s] %s" % ( 2967 brown(branch), 2968 blue(_("nothing to remove on this branch")), 2969 ), 2970 importance = 1, 2971 type = "info", 2972 header = blue(" @@ ") 2973 ) 2974 return errors, branch_data 2975 else: 2976 self.Entropy.updateProgress( 2977 "[branch:%s] %s:" % ( 2978 brown(branch), 2979 blue(_("these are the expired packages")), 2980 ), 2981 importance = 1, 2982 type = "info", 2983 header = blue(" @@ ") 2984 ) 2985 for package in removal: 2986 self.Entropy.updateProgress( 2987 "[branch:%s] %s: %s" % ( 2988 brown(branch), 2989 blue(_("remove")), 2990 darkgreen(package), 2991 ), 2992 importance = 1, 2993 type = "info", 2994 header = brown(" # ") 2995 ) 2996 2997 if pretend: 2998 return errors, branch_data 2999 3000 if ask: 3001 rc_question = self.Entropy.askQuestion( 3002 _("Would you like to continue ?")) 3003 if rc_question == "No": 3004 return errors, branch_data 3005 3006 myqueue = [] 3007 for package in removal: 3008 myqueue.append(package+etpConst['packagesmd5fileext']) 3009 myqueue.append(package) 3010 3011 ftp_basedir = os.path.join( 3012 self.Entropy.get_remote_packages_relative_path(repo), branch) 3013 for uri in self.Entropy.get_remote_mirrors(repo): 3014 3015 self.Entropy.updateProgress( 3016 "[branch:%s] %s..." % ( 3017 brown(branch), 3018 blue(_("removing packages remotely")), 3019 ), 3020 importance = 1, 3021 type = "info", 3022 header = blue(" @@ ") 3023 ) 3024 3025 crippled_uri = self.entropyTools.extract_ftp_host_from_uri(uri) 3026 destroyer = self.FtpServerHandler( 3027 FtpInterface, 3028 self.Entropy, 3029 [uri], 3030 myqueue, 3031 critical_files = [], 3032 ftp_basedir = ftp_basedir, 3033 remove = True, 3034 repo = repo 3035 ) 3036 errors, m_fine_uris, m_broken_uris = destroyer.go() 3037 if errors: 3038 my_broken_uris = [ 3039 (self.entropyTools.extract_ftp_host_from_uri(x[0]), x[1]) \ 3040 for x in m_broken_uris] 3041 3042 reason = my_broken_uris[0][1] 3043 self.Entropy.updateProgress( 3044 "[branch:%s] %s: %s, %s: %s" % ( 3045 brown(branch), 3046 blue(_("remove errors")), 3047 red(crippled_uri), 3048 blue(_("reason")), 3049 reason, 3050 ), 3051 importance = 1, 3052 type = "warning", 3053 header = brown(" !!! ") 3054 ) 3055 branch_data['errors'] = True 3056 errors = True 3057 3058 self.Entropy.updateProgress( 3059 "[branch:%s] %s..." % ( 3060 brown(branch), 3061 blue(_("removing packages locally")), 3062 ), 3063 importance = 1, 3064 type = "info", 3065 header = blue(" @@ ") 3066 ) 3067 3068 branch_data['removed'] = set() 3069 for package in removal: 3070 package_path = os.path.join( 3071 self.Entropy.get_local_packages_directory(repo), 3072 branch, package) 3073 package_path_hash = package_path + \ 3074 etpConst['packagesmd5fileext'] 3075 package_path_expired = package_path + \ 3076 etpConst['packagesexpirationfileext'] 3077 3078 my_rm_list = (package_path_hash, package_path, 3079 package_path_expired) 3080 for myfile in my_rm_list: 3081 if os.path.isfile(myfile): 3082 self.Entropy.updateProgress( 3083 "[branch:%s] %s: %s" % ( 3084 brown(branch), 3085 blue(_("removing")), 3086 darkgreen(myfile), 3087 ), 3088 importance = 1, 3089 type = "info", 3090 header = brown(" @@ ") 3091 ) 3092 os.remove(myfile) 3093 branch_data['removed'].add(myfile) 3094 3095 return errors, branch_data

Source Code for Module entropy.server.interfaces.mirrors