entropy.client.interfaces.repository

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Client Repositories Management Interface}. 10 11 """ 12 13 import os 14 import sys 15 import time 16 import tempfile 17 import shutil 18 import subprocess 19 import socket 20 21 from entropy.i18n import _ 22 from entropy.db import dbapi2, EntropyRepository 23 from entropy.cache import EntropyCacher 24 from entropy.misc import TimeScheduled 25 from entropy.const import etpConst, etpUi, etpCache, const_setup_perms, \ 26 const_debug_write 27 from entropy.exceptions import RepositoryError, SystemDatabaseError, \ 28 ConnectionError 29 from entropy.output import blue, darkred, red, darkgreen, purple, brown, bold 30 import entropy.tools 31 from entropy.dump import dumpobj 32

33 -class Repository:

34

35 - def __init__(self, EquoInstance, reponames = [], forceUpdate = False, 36 noEquoCheck = False, fetchSecurity = True):

37 38 self.LockScanner = None 39 from entropy.client.interfaces import Client 40 if not isinstance(EquoInstance, Client): 41 mytxt = "A valid Entropy Client instance or subclass is needed" 42 raise AttributeError(mytxt) 43 44 self.supported_download_items = ( 45 "db", "dblight", "ck", "cklight", "compck", 46 "lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck", 47 "meta_file", "notice_board" 48 ) 49 self.big_socket_timeout = 10 50 self.Entropy = EquoInstance 51 self.Cacher = EntropyCacher() 52 self.dbapi2 = dbapi2 53 self.reponames = reponames 54 self.forceUpdate = forceUpdate 55 self.syncErrors = False 56 self.dbupdated = False 57 self.newEquo = False 58 self.fetchSecurity = fetchSecurity 59 self.noEquoCheck = noEquoCheck 60 self.alreadyUpdated = 0 61 self.notAvailable = 0 62 self.valid_eapis = [1, 2, 3] 63 self.reset_dbformat_eapi(None) 64 self.current_repository_got_locked = False 65 self.updated_repos = set() 66 67 avail_data = self.Entropy.SystemSettings['repositories']['available'] 68 # check self.Entropy.SystemSettings['repositories']['available'] 69 if not avail_data: 70 mytxt = "No repositories specified in %s" % ( 71 etpConst['repositoriesconf'],) 72 raise AttributeError(mytxt) 73 74 if not self.reponames: 75 self.reponames.extend(list(avail_data.keys()))

76

77 - def __del__(self):

78 if self.LockScanner != None: 79 self.LockScanner.kill()

80

81 - def get_eapi3_connection(self, repository):

82 # get database url 83 avail_data = self.Entropy.SystemSettings['repositories']['available'] 84 dburl = avail_data[repository].get('service_uri') 85 if dburl is None: 86 return None 87 88 port = avail_data[repository]['service_port'] 89 try: 90 from entropy.services.ugc.interfaces import Client 91 from entropy.client.services.ugc.commands import Client as \ 92 CommandsClient 93 eapi3_socket = Client(self.Entropy, CommandsClient, 94 output_header = "\t", socket_timeout = self.big_socket_timeout) 95 eapi3_socket.connect(dburl, port) 96 return eapi3_socket 97 except (ConnectionError, socket.error,): 98 return None

99

100 - def check_eapi3_availability(self, repository):

101 conn = self.get_eapi3_connection(repository) 102 if conn is None: 103 return False 104 try: 105 conn.disconnect() 106 except (socket.error, AttributeError,): 107 return False 108 return True

109

110 - def reset_dbformat_eapi(self, repository):

111 112 self.dbformat_eapi = 2 113 if repository != None: 114 eapi_avail = self.check_eapi3_availability(repository) 115 if eapi_avail: 116 self.dbformat_eapi = 3 117 118 # FIXME, find a way to do that without needing sqlite3 exec. 119 if not os.access("/usr/bin/sqlite3", os.X_OK) or entropy.tools.islive(): 120 self.dbformat_eapi = 1 121 else: 122 rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", 123 shell = True) 124 if rc != 0: self.dbformat_eapi = 1 125 126 eapi_env = os.getenv("FORCE_EAPI") 127 if eapi_env != None: 128 try: 129 myeapi = int(eapi_env) 130 except (ValueError, TypeError,): 131 return 132 if myeapi in self.valid_eapis: 133 self.dbformat_eapi = myeapi

134 135

136 - def __validate_repository_id(self, repoid):

137 if repoid not in self.reponames: 138 mytxt = "Repository is not listed in self.reponames" 139 raise AttributeError(mytxt)

140

141 - def __validate_compression_method(self, repo):

142 143 self.__validate_repository_id(repo) 144 145 repo_settings = self.Entropy.SystemSettings['repositories'] 146 dbc_format = repo_settings['available'][repo]['dbcformat'] 147 cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format) 148 if cmethod is None: 149 mytxt = "Wrong database compression method" 150 raise AttributeError(mytxt) 151 152 return cmethod

153

154 - def __ensure_repository_path(self, repo):

155 156 self.__validate_repository_id(repo) 157 158 avail_data = self.Entropy.SystemSettings['repositories']['available'] 159 repo_data = avail_data[repo] 160 161 # create dir if it doesn't exist 162 if not os.path.isdir(repo_data['dbpath']): 163 os.makedirs(repo_data['dbpath'], 0o775) 164 165 const_setup_perms(etpConst['etpdatabaseclientdir'], 166 etpConst['entropygid'])

167

168 - def _construct_paths(self, item, repo, cmethod):

169 170 if item not in self.supported_download_items: 171 mytxt = "Supported items: %s" % (self.supported_download_items,) 172 raise AttributeError(mytxt) 173 174 items_needing_cmethod = ( 175 "db", "dblight", "cklight", "dbdump", "dbdumpck", 176 "dbdumplight", "dbdumplightck", "compck", 177 ) 178 if (item in items_needing_cmethod) and (cmethod is None): 179 mytxt = "For %s, cmethod can't be None" % (item,) 180 raise AttributeError(mytxt) 181 182 avail_data = self.Entropy.SystemSettings['repositories']['available'] 183 repo_data = avail_data[repo] 184 185 repo_db = repo_data['database'] 186 repo_dbpath = repo_data['dbpath'] 187 ec_hash = etpConst['etpdatabasehashfile'] 188 repo_lock_file = etpConst['etpdatabasedownloadlockfile'] 189 notice_board_filename = os.path.basename(repo_data['notice_board']) 190 meta_file = etpConst['etpdatabasemetafilesfile'] 191 md5_ext = etpConst['packagesmd5fileext'] 192 ec_cm2 = None 193 ec_cm3 = None 194 ec_cm4 = None 195 ec_cm5 = None 196 ec_cm6 = None 197 ec_cm7 = None 198 ec_cm8 = None 199 if cmethod != None: 200 ec_cm2 = etpConst[cmethod[2]] 201 ec_cm3 = etpConst[cmethod[3]] 202 ec_cm4 = etpConst[cmethod[4]] 203 ec_cm5 = etpConst[cmethod[5]] 204 ec_cm6 = etpConst[cmethod[6]] 205 ec_cm7 = etpConst[cmethod[7]] 206 ec_cm8 = etpConst[cmethod[8]] 207 208 mymap = { 209 'db': ("%s/%s" % (repo_db, ec_cm2,), "%s/%s" % (repo_dbpath, ec_cm2,),), 210 'dblight': ("%s/%s" % (repo_db, ec_cm7,), "%s/%s" % (repo_dbpath, ec_cm7,),), 211 'dbdump': ("%s/%s" % (repo_db, ec_cm3,), "%s/%s" % (repo_dbpath, ec_cm3,),), 212 'dbdumplight': ("%s/%s" % (repo_db, ec_cm5,), "%s/%s" % (repo_dbpath, ec_cm5,),), 213 'ck': ("%s/%s" % (repo_db, ec_hash,), "%s/%s" % (repo_dbpath, ec_hash,),), 214 'cklight': ("%s/%s" % (repo_db, ec_cm8,), "%s/%s" % (repo_dbpath, ec_cm8,),), 215 'compck': ("%s/%s%s" % (repo_db, ec_cm2, md5_ext,), "%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),), 216 'dbdumpck': ("%s/%s" % (repo_db, ec_cm4,), "%s/%s" % (repo_dbpath, ec_cm4,),), 217 'dbdumplightck': ("%s/%s" % (repo_db, ec_cm6,), "%s/%s" % (repo_dbpath, ec_cm6,),), 218 'lock': ("%s/%s" % (repo_db, repo_lock_file,), "%s/%s" % (repo_dbpath, repo_lock_file,),), 219 'notice_board': (repo_data['notice_board'], "%s/%s" % (repo_dbpath, notice_board_filename,),), 220 'meta_file': ("%s/%s" % (repo_db, meta_file,), "%s/%s" % (repo_dbpath, meta_file,),), 221 } 222 223 return mymap.get(item)

224

225 - def __remove_repository_files(self, repo, cmethod):

226 227 dbfilenameid = cmethod[2] 228 dblightfilenameid = cmethod[7] 229 self.__validate_repository_id(repo) 230 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 231 232 def remove_eapi1(): 233 234 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabasehashfile']): 235 os.remove(repo_dbpath+"/"+etpConst['etpdatabasehashfile']) 236 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 237 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 238 239 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid]): 240 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid]) 241 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 242 etpConst['packagesmd5fileext']): 243 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 244 etpConst['packagesmd5fileext']) 245 246 if os.path.isfile(repo_dbpath+"/"+etpConst[dbfilenameid]): 247 os.remove(repo_dbpath+"/"+etpConst[dbfilenameid]) 248 if os.path.isfile(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 249 etpConst['packagesmd5fileext']): 250 os.remove(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 251 etpConst['packagesmd5fileext'])

252 253 if self.dbformat_eapi == 1: 254 remove_eapi1() 255 elif self.dbformat_eapi in (2, 3,): 256 remove_eapi1() 257 if os.path.isfile(repo_dbpath+"/"+cmethod[6]): 258 os.remove(repo_dbpath+"/"+cmethod[6]) 259 if os.path.isfile(repo_dbpath+"/"+etpConst[cmethod[5]]): 260 os.remove(repo_dbpath+"/"+etpConst[cmethod[5]]) 261 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 262 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 263 else: 264 mytxt = "self.dbformat_eapi must be in (1,2,3,)" 265 raise AttributeError(mytxt)

266

267 - def __unpack_downloaded_database(self, repo, cmethod):

268 269 self.__validate_repository_id(repo) 270 rc = 0 271 path = None 272 sys_set_repos = self.Entropy.SystemSettings['repositories']['available'] 273 repo_data = sys_set_repos[repo] 274 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[7]] 275 if self.dbformat_eapi == 2: 276 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[5]] 277 278 279 if self.dbformat_eapi in (1, 2,): 280 try: 281 282 myfunc = getattr(entropy.tools, cmethod[1]) 283 path = myfunc(myfile) 284 # rename path correctly 285 if self.dbformat_eapi == 1: 286 new_path = os.path.join(os.path.dirname(path), 287 etpConst['etpdatabasefile']) 288 os.rename(path, new_path) 289 path = new_path 290 291 except (OSError, EOFError): 292 rc = 1 293 if os.path.isfile(myfile): 294 os.remove(myfile) 295 296 else: 297 mytxt = "self.dbformat_eapi must be in (1,2)" 298 raise AttributeError(mytxt) 299 300 if rc == 0: 301 self.Entropy.setup_default_file_perms(path) 302 303 return rc

304

305 - def _verify_file_checksum(self, file_path, md5_checksum_path):

306 ck_f = open(md5_checksum_path, "r") 307 md5hash = ck_f.readline().strip() 308 md5hash = md5hash.split()[0] 309 ck_f.close() 310 return entropy.tools.compare_md5(file_path, md5hash)

311

312 - def __verify_database_checksum(self, repo, cmethod = None):

313 314 self.__validate_repository_id(repo) 315 sys_settings_repos = self.Entropy.SystemSettings['repositories'] 316 avail_config = sys_settings_repos['available'][repo] 317 318 sep = os.path.sep 319 if self.dbformat_eapi == 1: 320 dbfile = avail_config['dbpath'] + sep + etpConst[cmethod[7]] 321 md5file = avail_config['dbpath'] + sep + etpConst[cmethod[8]] 322 323 elif self.dbformat_eapi == 2: 324 dbfile = avail_config['dbpath'] + sep + etpConst[cmethod[5]] 325 md5file = avail_config['dbpath'] + sep + etpConst[cmethod[6]] 326 327 else: 328 mytxt = "self.dbformat_eapi must be in (1,2)" 329 raise AttributeError(mytxt) 330 331 if not (os.access(md5file, os.R_OK) and os.path.isfile(md5file)): 332 return -1 333 334 return self._verify_file_checksum(dbfile, md5file)

335 336 # @returns -1 if the file is not available 337 # @returns int>0 if the revision has been retrieved

338 - def get_online_repository_revision(self, repo):

339 340 self.__validate_repository_id(repo) 341 avail_data = self.Entropy.SystemSettings['repositories']['available'] 342 repo_data = avail_data[repo] 343 344 url = repo_data['database'] + "/" + etpConst['etpdatabaserevisionfile'] 345 status = entropy.tools.get_remote_data(url, 346 timeout = self.big_socket_timeout) 347 if status: 348 status = status[0].strip() 349 try: 350 status = int(status) 351 except ValueError: 352 status = -1 353 return status 354 else: 355 return -1

356

357 - def get_online_eapi3_lock(self, repo):

358 self.__validate_repository_id(repo) 359 360 avail_data = self.Entropy.SystemSettings['repositories']['available'] 361 repo_data = avail_data[repo] 362 363 url = repo_data['database'] + "/" + etpConst['etpdatabaseeapi3lockfile'] 364 data = entropy.tools.get_remote_data(url) 365 if not data: 366 return False 367 return True

368

369 - def is_repository_eapi3_locked(self, repo):

370 self.__validate_repository_id(repo) 371 return self.get_online_eapi3_lock(repo)

372

373 - def is_repository_updatable(self, repo):

374 375 self.__validate_repository_id(repo) 376 377 onlinestatus = self.get_online_repository_revision(repo) 378 if (onlinestatus != -1): 379 localstatus = self.Entropy.get_repository_revision(repo) 380 if (localstatus == onlinestatus) and (not self.forceUpdate): 381 return False 382 return True

383

384 - def is_repository_unlocked(self, repo):

385 386 self.__validate_repository_id(repo) 387 388 rc = self.download_item("lock", repo, disallow_redirect = True) 389 if rc: # cannot download database 390 self.syncErrors = True 391 return False 392 return True

393

394 - def clear_repository_cache(self, repo):

395 self.__validate_repository_id(repo) 396 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'], 397 etpConst['dbnamerepoprefix'], repo,)) 398 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'], 399 etpConst['dbnamerepoprefix'], repo,))

400

401 - def download_item(self, item, repo, cmethod = None, lock_status_func = None, 402 disallow_redirect = True):

403 404 self.__validate_repository_id(repo) 405 url, filepath = self._construct_paths(item, repo, cmethod) 406 407 # to avoid having permissions issues 408 # it's better to remove the file before, 409 # otherwise new permissions won't be written 410 if os.path.isfile(filepath): 411 os.remove(filepath) 412 filepath_dir = os.path.dirname(filepath) 413 if not os.path.isdir(filepath_dir) and not os.path.lexists(filepath_dir): 414 os.makedirs(filepath_dir, 0o775) 415 const_setup_perms(filepath_dir, etpConst['entropygid']) 416 417 fetchConn = self.Entropy.urlFetcher( 418 url, 419 filepath, 420 resume = False, 421 abort_check_func = lock_status_func, 422 disallow_redirect = disallow_redirect 423 ) 424 fetchConn.progress = self.Entropy.progress 425 426 rc = fetchConn.download() 427 del fetchConn 428 if rc in ("-1", "-2", "-3", "-4"): 429 return False 430 self.Entropy.setup_default_file_perms(filepath) 431 return True

432

433 - def check_downloaded_database(self, repo, cmethod):

434 435 dbfilename = etpConst[cmethod[7]] 436 if self.dbformat_eapi == 2: 437 dbfilename = etpConst[cmethod[5]] 438 # verify checksum 439 mytxt = "%s %s %s" % ( 440 red(_("Checking downloaded database")), 441 darkgreen(dbfilename), 442 red("..."), 443 ) 444 self.Entropy.updateProgress( 445 mytxt, 446 importance = 0, 447 back = True, 448 type = "info", 449 header = "\t" 450 ) 451 db_status = self.__verify_database_checksum(repo, cmethod) 452 if db_status == -1: 453 mytxt = "%s. %s !" % ( 454 red(_("Cannot open digest")), 455 red(_("Cannot verify database integrity")), 456 ) 457 self.Entropy.updateProgress( 458 mytxt, 459 importance = 1, 460 type = "warning", 461 header = "\t" 462 ) 463 elif db_status: 464 mytxt = "%s: %s" % ( 465 red(_("Downloaded database status")), 466 bold(_("OK")), 467 ) 468 self.Entropy.updateProgress( 469 mytxt, 470 importance = 1, 471 type = "info", 472 header = "\t" 473 ) 474 else: 475 mytxt = "%s: %s" % ( 476 red(_("Downloaded database status")), 477 darkred(_("ERROR")), 478 ) 479 self.Entropy.updateProgress( 480 mytxt, 481 importance = 1, 482 type = "error", 483 header = "\t" 484 ) 485 mytxt = "%s. %s" % ( 486 red(_("An error occured while checking database integrity")), 487 red(_("Giving up")), 488 ) 489 self.Entropy.updateProgress( 490 mytxt, 491 importance = 1, 492 type = "error", 493 header = "\t" 494 ) 495 return 1 496 return 0

497 498

499 - def show_repository_information(self, repo, count_info):

500 501 avail_data = self.Entropy.SystemSettings['repositories']['available'] 502 repo_data = avail_data[repo] 503 504 self.Entropy.updateProgress( 505 bold("%s") % ( repo_data['description'] ), 506 importance = 2, 507 type = "info", 508 count = count_info, 509 header = blue(" # ") 510 ) 511 mytxt = "%s: %s" % (red(_("Database URL")), 512 darkgreen(repo_data['database']),) 513 self.Entropy.updateProgress( 514 mytxt, 515 importance = 1, 516 type = "info", 517 header = blue(" # ") 518 ) 519 mytxt = "%s: %s" % (red(_("Database local path")), 520 darkgreen(repo_data['dbpath']),) 521 self.Entropy.updateProgress( 522 mytxt, 523 importance = 0, 524 type = "info", 525 header = blue(" # ") 526 ) 527 mytxt = "%s: %s" % (red(_("Database EAPI")), 528 darkgreen(str(self.dbformat_eapi)),) 529 self.Entropy.updateProgress( 530 mytxt, 531 importance = 0, 532 type = "info", 533 header = blue(" # ") 534 )

535

536 - def get_eapi3_local_database(self, repo):

537 538 avail_data = self.Entropy.SystemSettings['repositories']['available'] 539 repo_data = avail_data[repo] 540 541 dbfile = os.path.join(repo_data['dbpath'], etpConst['etpdatabasefile']) 542 mydbconn = None 543 try: 544 mydbconn = self.Entropy.open_generic_database(dbfile, 545 xcache = False, indexing_override = False) 546 mydbconn.validateDatabase() 547 except ( 548 self.Entropy.dbapi2.OperationalError, 549 self.Entropy.dbapi2.IntegrityError, 550 SystemDatabaseError, 551 IOError, 552 OSError,): 553 mydbconn = None 554 return mydbconn

555

556 - def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, 557 session):

558 559 product = self.Entropy.SystemSettings['repositories']['product'] 560 data = eapi3_interface.CmdInterface.differential_packages_comparison( 561 session, idpackages, repo, etpConst['currentarch'], product 562 ) 563 if isinstance(data, bool): # then it's probably == False 564 return False, False, False 565 elif not isinstance(data, dict): 566 return None, None, None 567 elif 'added' not in data or \ 568 'removed' not in data or \ 569 'checksum' not in data: 570 return None, None, None 571 return data['added'], data['removed'], data['checksum']

572

573 - def get_eapi3_repository_metadata(self, eapi3_interface, repo, session):

574 product = self.Entropy.SystemSettings['repositories']['product'] 575 data = eapi3_interface.CmdInterface.get_repository_metadata( 576 session, repo, etpConst['currentarch'], product 577 ) 578 if not isinstance(data, dict): 579 return {} 580 return data

581

582 - def handle_eapi3_database_sync(self, repo, threshold = 300, 583 chunk_size = 12):

584 585 def prepare_exit(mysock, session = None): 586 try: 587 if session != None: 588 mysock.close_session(session) 589 mysock.disconnect() 590 except (socket.error,): 591 pass

592 593 eapi3_interface = self.get_eapi3_connection(repo) 594 if eapi3_interface is None: 595 return False 596 597 session = eapi3_interface.open_session() 598 599 # AttributeError because mydbconn can be is None 600 try: 601 mydbconn = self.get_eapi3_local_database(repo) 602 myidpackages = mydbconn.listAllIdpackages() 603 except (self.dbapi2.DatabaseError, self.dbapi2.IntegrityError, 604 self.dbapi2.OperationalError, AttributeError,): 605 606 prepare_exit(eapi3_interface, session) 607 return False 608 609 added_ids, removed_ids, checksum = self.get_eapi3_database_differences( 610 eapi3_interface, repo, 611 myidpackages, session 612 ) 613 if (None in (added_ids, removed_ids, checksum)) or \ 614 (not added_ids and not removed_ids and self.forceUpdate): 615 mydbconn.closeDB() 616 prepare_exit(eapi3_interface, session) 617 return False 618 619 elif not checksum: # {added_ids, removed_ids, checksum} == False 620 mydbconn.closeDB() 621 prepare_exit(eapi3_interface, session) 622 mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), 623 darkred(_("remote database suddenly locked")),) 624 self.Entropy.updateProgress( 625 mytxt, 626 importance = 0, 627 type = "info", 628 header = blue(" # "), 629 ) 630 return None 631 632 # is it worth it? 633 if len(added_ids) > threshold: 634 mytxt = "%s: %s (%s: %s/%s)" % ( 635 blue(_("EAPI3 Service")), 636 darkred(_("skipping differential sync")), 637 brown(_("threshold")), 638 blue(str(len(added_ids))), 639 darkred(str(threshold)), 640 ) 641 self.Entropy.updateProgress( 642 mytxt, 643 importance = 0, 644 type = "info", 645 header = blue(" # "), 646 ) 647 mydbconn.closeDB() 648 prepare_exit(eapi3_interface, session) 649 return False 650 651 count = 0 652 added_segments = [] 653 mytmp = set() 654 655 for idpackage in added_ids: 656 count += 1 657 mytmp.add(idpackage) 658 if count % chunk_size == 0: 659 added_segments.append(list(mytmp)) 660 mytmp.clear() 661 if mytmp: added_segments.append(list(mytmp)) 662 del mytmp 663 664 # fetch and store 665 count = 0 666 maxcount = len(added_segments) 667 product = self.Entropy.SystemSettings['repositories']['product'] 668 for segment in added_segments: 669 670 count += 1 671 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) 672 self.Entropy.updateProgress( 673 mytxt, importance = 0, type = "info", 674 header = "\t", back = True, count = (count, maxcount,) 675 ) 676 fetch_count = 0 677 max_fetch_count = 5 678 679 while True: 680 681 # anti loop protection 682 if fetch_count > max_fetch_count: 683 mydbconn.closeDB() 684 prepare_exit(eapi3_interface, session) 685 return False 686 687 fetch_count += 1 688 cmd_intf = eapi3_interface.CmdInterface 689 pkgdata = cmd_intf.get_strict_package_information( 690 session, segment, repo, etpConst['currentarch'], product 691 ) 692 if pkgdata is None: 693 mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), 694 darkred(str(segment)),) 695 self.Entropy.updateProgress( 696 mytxt, importance = 1, type = "warning", 697 header = "\t", count = (count, maxcount,) 698 ) 699 continue 700 elif not pkgdata: # pkgdata == False 701 mytxt = "%s: %s" % ( 702 blue(_("Service status")), 703 darkred("remote database suddenly locked"), 704 ) 705 self.Entropy.updateProgress( 706 mytxt, importance = 1, type = "info", 707 header = "\t", count = (count, maxcount,) 708 ) 709 mydbconn.closeDB() 710 prepare_exit(eapi3_interface, session) 711 return None 712 elif isinstance(pkgdata, tuple): 713 mytxt = "%s: %s, %s. %s" % ( 714 blue(_("Service status")), 715 pkgdata[0], pkgdata[1], 716 darkred("Error processing the command"), 717 ) 718 self.Entropy.updateProgress( 719 mytxt, importance = 1, type = "info", 720 header = "\t", count = (count, maxcount,) 721 ) 722 mydbconn.closeDB() 723 prepare_exit(eapi3_interface, session) 724 return None 725 726 try: 727 for idpackage in pkgdata: 728 dumpobj( 729 "%s%s" % (etpCache['eapi3_fetch'], idpackage,), 730 pkgdata[idpackage], 731 ignore_exceptions = False 732 ) 733 except (IOError, EOFError, OSError,) as e: 734 mytxt = "%s: %s: %s." % ( 735 blue(_("Local status")), 736 darkred("Error storing data"), 737 e, 738 ) 739 self.Entropy.updateProgress( 740 mytxt, importance = 1, type = "info", 741 header = "\t", count = (count, maxcount,) 742 ) 743 mydbconn.closeDB() 744 prepare_exit(eapi3_interface, session) 745 return None 746 747 break 748 749 del added_segments 750 751 repo_metadata = self.get_eapi3_repository_metadata(eapi3_interface, 752 repo, session) 753 metadata_elements = ("sets", "treeupdates_actions", 754 "treeupdates_digest", "library_idpackages",) 755 for elem in metadata_elements: 756 if elem not in repo_metadata: 757 mydbconn.closeDB() 758 prepare_exit(eapi3_interface, session) 759 mytxt = "%s: %s" % ( 760 blue(_("EAPI3 Service status")), 761 darkred(_("cannot fetch repository metadata")), 762 ) 763 self.Entropy.updateProgress( 764 mytxt, 765 importance = 0, 766 type = "info", 767 header = blue(" # "), 768 ) 769 return None 770 771 # update treeupdates 772 try: 773 mydbconn.setRepositoryUpdatesDigest(repo, 774 repo_metadata['treeupdates_digest']) 775 mydbconn.bumpTreeUpdatesActions( 776 repo_metadata['treeupdates_actions']) 777 except (self.dbapi2.Error,): 778 mydbconn.closeDB() 779 prepare_exit(eapi3_interface, session) 780 mytxt = "%s: %s" % ( 781 blue(_("EAPI3 Service status")), 782 darkred(_("cannot update treeupdates data")), 783 ) 784 self.Entropy.updateProgress( 785 mytxt, 786 importance = 0, 787 type = "info", 788 header = blue(" # "), 789 ) 790 return None 791 792 # update package sets 793 try: 794 mydbconn.clearPackageSets() 795 mydbconn.insertPackageSets(repo_metadata['sets']) 796 except (self.dbapi2.Error,): 797 mydbconn.closeDB() 798 prepare_exit(eapi3_interface, session) 799 mytxt = "%s: %s" % ( 800 blue(_("EAPI3 Service status")), 801 darkred(_("cannot update package sets data")), 802 ) 803 self.Entropy.updateProgress( 804 mytxt, 805 importance = 0, 806 type = "info", 807 header = blue(" # "), 808 ) 809 return None 810 811 # I don't need you anymore 812 # disconnect socket 813 prepare_exit(eapi3_interface, session) 814 815 # now that we have all stored, add 816 count = 0 817 maxcount = len(added_ids) 818 for idpackage in added_ids: 819 count += 1 820 mydata = self.Cacher.pop("%s%s" % ( 821 etpCache['eapi3_fetch'], idpackage,)) 822 if mydata is None: 823 mytxt = "%s: %s" % ( 824 blue(_("Fetch error on segment while adding")), 825 darkred(str(segment)), 826 ) 827 self.Entropy.updateProgress( 828 mytxt, importance = 1, type = "warning", 829 header = "\t", count = (count, maxcount,) 830 ) 831 mydbconn.closeDB() 832 return False 833 834 mytxt = "%s %s" % ( 835 blue(_("Injecting package")), 836 darkgreen(mydata['atom']), 837 ) 838 self.Entropy.updateProgress( 839 mytxt, importance = 0, type = "info", 840 header = "\t", back = True, count = (count, maxcount,) 841 ) 842 try: 843 mydbconn.addPackage( 844 mydata, revision = mydata['revision'], 845 idpackage = idpackage, do_commit = False, 846 formatted_content = True 847 ) 848 except (self.dbapi2.Error,) as err: 849 if etpUi['debug']: 850 entropy.tools.print_traceback() 851 self.Entropy.updateProgress("%s: %s" % ( 852 blue(_("repository error while adding packages")), 853 err,), 854 importance = 1, type = "warning", 855 header = "\t", count = (count, maxcount,) 856 ) 857 mydbconn.closeDB() 858 return False 859 860 self.Entropy.updateProgress( 861 blue(_("Packages injection complete")), importance = 0, 862 type = "info", header = "\t", 863 ) 864 865 # now remove 866 maxcount = len(removed_ids) 867 count = 0 868 # preload atoms names to improve speed during removePackage 869 atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids) 870 for idpackage in removed_ids: 871 myatom = atoms_map.get(idpackage) 872 count += 1 873 mytxt = "%s: %s" % ( 874 blue(_("Removing package")), 875 darkred(str(myatom)),) 876 self.Entropy.updateProgress( 877 mytxt, importance = 0, type = "info", 878 header = "\t", back = True, count = (count, maxcount,) 879 ) 880 try: 881 mydbconn.removePackage(idpackage, do_cleanup = False, 882 do_commit = False) 883 except (self.dbapi2.Error,): 884 self.Entropy.updateProgress( 885 blue(_("repository error while removing packages")), 886 importance = 1, type = "warning", 887 header = "\t", count = (count, maxcount,) 888 ) 889 mydbconn.closeDB() 890 return False 891 892 self.Entropy.updateProgress( 893 blue(_("Packages removal complete")), 894 importance = 0, type = "info", 895 header = "\t", 896 ) 897 898 mydbconn.commitChanges() 899 mydbconn.clearCache() 900 # now verify if both checksums match 901 result = False 902 mychecksum = mydbconn.checksum(do_order = True, 903 strict = False, strings = True) 904 if checksum == mychecksum: 905 result = True 906 else: 907 self.Entropy.updateProgress( 908 blue(_("Database checksum doesn't match remote.")), 909 importance = 0, type = "info", header = "\t", 910 ) 911 mytxt = "%s: %s" % (_('local'), mychecksum,) 912 self.Entropy.updateProgress( 913 mytxt, importance = 0, 914 type = "info", header = "\t", 915 ) 916 mytxt = "%s: %s" % (_('remote'), checksum,) 917 self.Entropy.updateProgress( 918 mytxt, importance = 0, 919 type = "info", header = "\t", 920 ) 921 922 mydbconn.closeDB() 923 return result 924

925 - def _run_post_update_repository_hook(self, repoid):

926 my_repos = self.Entropy.SystemSettings['repositories'] 927 branch = my_repos['branch'] 928 avail_data = my_repos['available'] 929 repo_data = avail_data[repoid] 930 post_update_script = repo_data['post_repo_update_script'] 931 932 if not (os.path.isfile(post_update_script) and \ 933 os.access(post_update_script, os.R_OK)): 934 # not found! 935 const_debug_write(__name__, 936 "_run_post_update_repository_hook: not found") 937 return 0 938 939 args = ["/bin/sh", post_update_script, repoid, 940 etpConst['systemroot'] + "/", branch] 941 const_debug_write(__name__, 942 "_run_post_update_repository_hook: run: %s" % (args,)) 943 proc = subprocess.Popen(args, stdin = sys.stdin, 944 stdout = sys.stdout, stderr = sys.stderr) 945 # it is possible to ignore errors because 946 # if it's a critical thing, upstream dev just have to fix 947 # the script and will be automagically re-run 948 br_rc = proc.wait() 949 const_debug_write(__name__, 950 "_run_post_update_repository_hook: rc: %s" % (br_rc,)) 951 952 return br_rc

953

954 - def run_sync(self):

955 956 self.dbupdated = False 957 repocount = 0 958 repolength = len(self.reponames) 959 for repo in self.reponames: 960 961 repocount += 1 962 self.reset_dbformat_eapi(repo) 963 self.show_repository_information(repo, (repocount, repolength)) 964 965 if not self.forceUpdate: 966 updated = self.handle_repository_update(repo) 967 if updated: 968 self.Entropy.cycleDone() 969 self.alreadyUpdated += 1 970 continue 971 972 locked = self.handle_repository_lock(repo) 973 if locked: 974 self.notAvailable += 1 975 self.Entropy.cycleDone() 976 continue 977 978 # clear database interface cache belonging to this repository 979 self.clear_repository_cache(repo) 980 self.__ensure_repository_path(repo) 981 982 # dealing with EAPI 983 # setting some vars 984 do_skip = False 985 skip_this_repo = False 986 db_checksum_down_status = False 987 do_db_update_transfer = False 988 rc = 0 989 990 my_repos = self.Entropy.SystemSettings['repositories'] 991 avail_data = my_repos['available'] 992 repo_data = avail_data[repo] 993 994 # some variables 995 dumpfile = os.path.join(repo_data['dbpath'], 996 etpConst['etpdatabasedumplight']) 997 dbfile = os.path.join(repo_data['dbpath'], 998 etpConst['etpdatabasefile']) 999 dbfile_old = dbfile+".sync" 1000 cmethod = self.__validate_compression_method(repo) 1001 1002 while True: 1003 1004 if do_skip: 1005 break 1006 1007 if self.dbformat_eapi < 3: 1008 1009 down_status = self.handle_database_download(repo, cmethod) 1010 if not down_status: 1011 self.Entropy.cycleDone() 1012 self.notAvailable += 1 1013 do_skip = True 1014 skip_this_repo = True 1015 continue 1016 db_checksum_down_status = \ 1017 self.handle_database_checksum_download(repo, cmethod) 1018 break 1019 1020 elif self.dbformat_eapi == 3 and not \ 1021 (os.path.isfile(dbfile) and os.access(dbfile, os.W_OK)): 1022 1023 do_db_update_transfer = None 1024 self.dbformat_eapi -= 1 1025 continue 1026 1027 elif self.dbformat_eapi == 3: 1028 1029 status = False 1030 try: 1031 status = self.handle_eapi3_database_sync(repo) 1032 except socket.error as err: 1033 mytxt = "%s: %s" % ( 1034 blue(_("EAPI3 Service error")), 1035 darkred(repr(err)), 1036 ) 1037 self.Entropy.updateProgress( 1038 mytxt, 1039 importance = 0, 1040 type = "info", 1041 header = blue(" # "), 1042 ) 1043 except: 1044 # avoid broken entries, deal with every exception 1045 self.__remove_repository_files(repo, cmethod) 1046 raise 1047 1048 if status is None: # remote db not available anymore ? 1049 time.sleep(5) 1050 locked = self.handle_repository_lock(repo) 1051 if locked: 1052 self.Entropy.cycleDone() 1053 self.notAvailable += 1 1054 do_skip = True 1055 skip_this_repo = True 1056 else: # ah, well... dunno then... 1057 do_db_update_transfer = None 1058 self.dbformat_eapi -= 1 1059 continue 1060 elif not status: # (status == False) 1061 # set to none and completely skip database alignment 1062 do_db_update_transfer = None 1063 self.dbformat_eapi -= 1 1064 continue 1065 1066 break 1067 1068 if skip_this_repo: 1069 continue 1070 1071 if self.dbformat_eapi in (1, 2,): 1072 1073 # new policy, always deny repository if 1074 # its database checksum cannot be fetched 1075 if not db_checksum_down_status: 1076 # delete all 1077 self.__remove_repository_files(repo, cmethod) 1078 self.syncErrors = True 1079 self.Entropy.cycleDone() 1080 continue 1081 1082 rc = self.check_downloaded_database(repo, cmethod) 1083 if rc != 0: 1084 # delete all 1085 self.__remove_repository_files(repo, cmethod) 1086 self.syncErrors = True 1087 self.Entropy.cycleDone() 1088 continue 1089 1090 # if do_db_update_transfer == False and not None 1091 if (do_db_update_transfer is not None) and not \ 1092 do_db_update_transfer: 1093 1094 if os.access(dbfile, os.R_OK | os.W_OK) and \ 1095 os.path.isfile(dbfile): 1096 try: 1097 os.rename(dbfile, dbfile_old) 1098 do_db_update_transfer = True 1099 except OSError: 1100 do_db_update_transfer = False 1101 1102 unpack_status = self.handle_downloaded_database_unpack(repo, 1103 cmethod) 1104 1105 if not unpack_status: 1106 # delete all 1107 self.__remove_repository_files(repo, cmethod) 1108 self.syncErrors = True 1109 self.Entropy.cycleDone() 1110 continue 1111 1112 # re-validate 1113 if not os.path.isfile(dbfile): 1114 do_db_update_transfer = False 1115 1116 elif os.path.isfile(dbfile) and not do_db_update_transfer and \ 1117 (self.dbformat_eapi != 1): 1118 os.remove(dbfile) 1119 1120 if self.dbformat_eapi == 2: 1121 rc = self.do_eapi2_inject_downloaded_dump(dumpfile, 1122 dbfile, cmethod) 1123 1124 if do_db_update_transfer: 1125 self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old) 1126 1127 if self.dbformat_eapi == 2: 1128 # remove the dump 1129 os.remove(dumpfile) 1130 1131 if rc != 0: 1132 # delete all 1133 self.__remove_repository_files(repo, cmethod) 1134 self.syncErrors = True 1135 self.Entropy.cycleDone() 1136 if os.path.isfile(dbfile_old): 1137 os.remove(dbfile_old) 1138 continue 1139 1140 if os.path.isfile(dbfile) and os.access(dbfile, os.W_OK): 1141 try: 1142 self.Entropy.setup_default_file_perms(dbfile) 1143 except OSError: # notification applet 1144 pass 1145 1146 # database is going to be updated 1147 self.dbupdated = True 1148 self.do_standard_items_download(repo) 1149 self.Entropy.update_repository_revision(repo) 1150 if self.Entropy.indexing: 1151 self.do_database_indexing(repo) 1152 1153 try: 1154 spm_class = self.Entropy.Spm_class() 1155 spm_class.entropy_client_post_repository_update_hook( 1156 self.Entropy, repo) 1157 except Exception as e: 1158 entropy.tools.print_traceback() 1159 mytxt = "%s: %s" % ( 1160 blue(_("Configuration files update error, not critical, continuing")), 1161 darkred(repr(e)), 1162 ) 1163 self.Entropy.updateProgress(mytxt, importance = 0, 1164 type = "info", header = blue(" # "),) 1165 1166 # execute post update repo hook 1167 self._run_post_update_repository_hook(repo) 1168 1169 self.updated_repos.add(repo) 1170 self.Entropy.cycleDone() 1171 1172 # remove garbage 1173 if os.access(dbfile_old, os.R_OK) and os.path.isfile(dbfile_old): 1174 os.remove(dbfile_old) 1175 1176 # keep them closed 1177 self.Entropy.close_all_repositories() 1178 self.Entropy.validate_repositories() 1179 self.Entropy.close_all_repositories() 1180 1181 # clean caches, fetch security 1182 if self.dbupdated: 1183 self.Entropy.purge_cache(client_purge = False) 1184 if self.fetchSecurity: 1185 self.do_update_security_advisories() 1186 # do treeupdates 1187 if isinstance(self.Entropy.clientDbconn, EntropyRepository) and \ 1188 entropy.tools.is_root(): # only as root due to Portage 1189 for repo in self.reponames: 1190 try: 1191 dbc = self.Entropy.open_repository(repo) 1192 except RepositoryError: 1193 # download failed and repo is not available, skip! 1194 continue 1195 self.Entropy.repository_packages_spm_sync(repo, dbc) 1196 self.Entropy.close_all_repositories() 1197 1198 if self.syncErrors: 1199 self.Entropy.updateProgress( 1200 red(_("Something bad happened. Please have a look.")), 1201 importance = 1, 1202 type = "warning", 1203 header = darkred(" @@ ") 1204 ) 1205 self.syncErrors = True 1206 self.Entropy.resources_remove_lock() 1207 return 128 1208 1209 if not self.noEquoCheck: 1210 self.check_entropy_updates() 1211 1212 return 0

1213

1214 - def check_entropy_updates(self):

1215 rc = False 1216 if not self.noEquoCheck: 1217 try: 1218 rc, pkg_match = self.Entropy.check_package_update( 1219 "sys-apps/entropy", deep = True) 1220 except: 1221 pass 1222 if rc: 1223 self.newEquo = True 1224 mytxt = "%s: %s. %s." % ( 1225 bold("Equo/Entropy"), 1226 blue(_("a new release is available")), 1227 darkred(_("Mind to install it before any other package")), 1228 ) 1229 self.Entropy.updateProgress( 1230 mytxt, 1231 importance = 1, 1232 type = "info", 1233 header = bold(" !!! ") 1234 )

1235

1236 - def handle_downloaded_database_unpack(self, repo, cmethod):

1237 1238 file_to_unpack = etpConst['etpdatabasedump'] 1239 if self.dbformat_eapi == 1: 1240 file_to_unpack = etpConst['etpdatabasefile'] 1241 mytxt = "%s %s %s" % (red(_("Unpacking database to")), 1242 darkgreen(file_to_unpack), red("..."),) 1243 self.Entropy.updateProgress( 1244 mytxt, 1245 importance = 0, 1246 type = "info", 1247 header = "\t" 1248 ) 1249 1250 myrc = self.__unpack_downloaded_database(repo, cmethod) 1251 if myrc != 0: 1252 mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")), 1253 red(_("Skipping repository")),) 1254 self.Entropy.updateProgress( 1255 mytxt, 1256 importance = 1, 1257 type = "warning", 1258 header = "\t" 1259 ) 1260 return False 1261 return True

1262 1263

1264 - def handle_database_checksum_download(self, repo, cmethod):

1265 1266 hashfile = etpConst[cmethod[8]] 1267 downitem = 'cklight' 1268 if self.dbformat_eapi == 2: # EAPI = 2 1269 hashfile = etpConst[cmethod[6]] 1270 downitem = 'dbdumplightck' 1271 1272 mytxt = "%s %s %s" % ( 1273 red(_("Downloading checksum")), 1274 darkgreen(hashfile), 1275 red("..."), 1276 ) 1277 # download checksum 1278 self.Entropy.updateProgress( 1279 mytxt, 1280 importance = 0, 1281 type = "info", 1282 header = "\t" 1283 ) 1284 1285 db_down_status = self.download_item(downitem, repo, cmethod, 1286 disallow_redirect = True) 1287 if not db_down_status and (downitem != 'cklight'): 1288 # fallback to old method, deprecated 1289 db_down_status = self.download_item('cklight', repo, cmethod, 1290 disallow_redirect = True) 1291 1292 if not db_down_status: 1293 mytxt = "%s %s !" % ( 1294 red(_("Cannot fetch checksum")), 1295 red(_("Cannot verify database integrity")), 1296 ) 1297 self.Entropy.updateProgress( 1298 mytxt, 1299 importance = 1, 1300 type = "warning", 1301 header = "\t" 1302 ) 1303 return db_down_status

1304

1305 - def load_background_repository_lock_check(self, repo):

1306 # kill previous 1307 self.current_repository_got_locked = False 1308 self.kill_previous_repository_lock_scanner() 1309 self.LockScanner = TimeScheduled(30, self.repository_lock_scanner, repo) 1310 self.LockScanner.start()

1311

1312 - def kill_previous_repository_lock_scanner(self):

1313 if self.LockScanner != None: 1314 self.LockScanner.kill()

1315

1316 - def repository_lock_scanner(self, repo):

1317 locked = self.handle_repository_lock(repo) 1318 if locked: 1319 self.current_repository_got_locked = True

1320

1321 - def repository_lock_scanner_status(self):

1322 # raise an exception if repo got suddenly locked 1323 if self.current_repository_got_locked: 1324 mytxt = "Current repository got suddenly locked. Download aborted." 1325 raise RepositoryError('RepositoryError %s' % (mytxt,))

1326

1327 - def handle_database_download(self, repo, cmethod):

1328 1329 def show_repo_locked_message(): 1330 mytxt = "%s: %s." % ( 1331 bold(_("Attention")), 1332 red(_("remote database got suddenly locked")), 1333 ) 1334 self.Entropy.updateProgress( 1335 mytxt, 1336 importance = 1, 1337 type = "warning", 1338 header = "\t" 1339 )

1340 1341 # starting to download 1342 mytxt = "%s ..." % (red(_("Downloading repository database")),) 1343 self.Entropy.updateProgress( 1344 mytxt, 1345 importance = 1, 1346 type = "info", 1347 header = "\t" 1348 ) 1349 1350 down_status = False 1351 if self.dbformat_eapi == 2: 1352 # start a check in background 1353 self.load_background_repository_lock_check(repo) 1354 down_status = self.download_item("dbdumplight", repo, cmethod, 1355 lock_status_func = self.repository_lock_scanner_status, 1356 disallow_redirect = True) 1357 if self.current_repository_got_locked: 1358 self.kill_previous_repository_lock_scanner() 1359 show_repo_locked_message() 1360 return False 1361 if not down_status: # fallback to old db 1362 # start a check in background 1363 self.load_background_repository_lock_check(repo) 1364 self.dbformat_eapi = 1 1365 down_status = self.download_item("dblight", repo, cmethod, 1366 lock_status_func = self.repository_lock_scanner_status, 1367 disallow_redirect = True) 1368 if self.current_repository_got_locked: 1369 self.kill_previous_repository_lock_scanner() 1370 show_repo_locked_message() 1371 return False 1372 1373 if not down_status: 1374 mytxt = "%s: %s." % (bold(_("Attention")), 1375 red(_("database does not exist online")),) 1376 self.Entropy.updateProgress( 1377 mytxt, 1378 importance = 1, 1379 type = "warning", 1380 header = "\t" 1381 ) 1382 1383 self.kill_previous_repository_lock_scanner() 1384 return down_status 1385

1386 - def handle_repository_update(self, repo):

1387 # check if database is already updated to the latest revision 1388 update = self.is_repository_updatable(repo) 1389 if not update: 1390 mytxt = "%s: %s." % (bold(_("Attention")), 1391 red(_("database is already up to date")),) 1392 self.Entropy.updateProgress( 1393 mytxt, 1394 importance = 1, 1395 type = "info", 1396 header = "\t" 1397 ) 1398 return True 1399 # also check for eapi3 lock 1400 if self.dbformat_eapi == 3: 1401 locked = self.is_repository_eapi3_locked(repo) 1402 if locked: 1403 mytxt = "%s: %s." % (bold(_("Attention")), 1404 red(_("database will be ready soon")),) 1405 self.Entropy.updateProgress( 1406 mytxt, 1407 importance = 1, 1408 type = "info", 1409 header = "\t" 1410 ) 1411 return True 1412 return False

1413

1414 - def handle_repository_lock(self, repo):

1415 # get database lock 1416 unlocked = self.is_repository_unlocked(repo) 1417 if not unlocked: 1418 mytxt = "%s: %s. %s." % ( 1419 bold(_("Attention")), 1420 red(_("Repository is being updated")), 1421 red(_("Try again in a few minutes")), 1422 ) 1423 self.Entropy.updateProgress( 1424 mytxt, 1425 importance = 1, 1426 type = "warning", 1427 header = "\t" 1428 ) 1429 return True 1430 return False

1431

1432 - def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

1433 1434 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, 1435 indexing_override = False) 1436 old_dbconn = self.Entropy.open_generic_database(dbfile_old, 1437 xcache = False, indexing_override = False) 1438 upd_rc = 0 1439 try: 1440 upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") 1441 except (self.dbapi2.OperationalError, self.dbapi2.IntegrityError, 1442 self.dbapi2.DatabaseError,): 1443 pass 1444 old_dbconn.closeDB() 1445 dbconn.closeDB() 1446 if upd_rc > 0: 1447 # -1 means no changes, == force used 1448 # 0 means too much hassle 1449 os.rename(dbfile_old, dbfile) 1450 return upd_rc

1451

1452 - def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

1453 1454 # load the dump into database 1455 mytxt = "%s %s, %s %s" % ( 1456 red(_("Injecting downloaded dump")), 1457 darkgreen(etpConst[cmethod[5]]), 1458 red(_("please wait")), 1459 red("..."), 1460 ) 1461 self.Entropy.updateProgress( 1462 mytxt, 1463 importance = 0, 1464 type = "info", 1465 header = "\t" 1466 ) 1467 dbconn = self.Entropy.open_generic_database(dbfile, 1468 xcache = False, indexing_override = False) 1469 rc = dbconn.doDatabaseImport(dumpfile, dbfile) 1470 dbconn.closeDB() 1471 return rc

1472

1473 - def do_update_security_advisories(self):

1474 # update Security Advisories 1475 try: 1476 securityConn = self.Entropy.Security() 1477 securityConn.fetch_advisories() 1478 except Exception as e: 1479 entropy.tools.print_traceback(f = self.Entropy.clientLog) 1480 mytxt = "%s: %s" % (red(_("Advisories fetch error")), e,) 1481 self.Entropy.updateProgress( 1482 mytxt, 1483 importance = 1, 1484 type = "warning", 1485 header = darkred(" @@ ") 1486 )

1487

1488 - def do_standard_items_download(self, repo):

1489 1490 repos_data = self.Entropy.SystemSettings['repositories'] 1491 repo_data = repos_data['available'][repo] 1492 notice_board = os.path.basename(repo_data['local_notice_board']) 1493 1494 objects_to_unpack = ("meta_file",) 1495 1496 download_items = [ 1497 ( 1498 "meta_file", 1499 etpConst['etpdatabasemetafilesfile'], 1500 False, 1501 "%s %s %s" % ( 1502 red(_("Downloading repository metafile")), 1503 darkgreen(etpConst['etpdatabasemetafilesfile']), 1504 red("..."), 1505 ) 1506 ), 1507 ( 1508 "notice_board", 1509 notice_board, 1510 True, 1511 "%s %s %s" % ( 1512 red(_("Downloading Notice Board")), 1513 darkgreen(notice_board), 1514 red("..."), 1515 ) 1516 ), 1517 ] 1518 1519 def my_show_info(txt): 1520 self.Entropy.updateProgress( 1521 txt, 1522 importance = 0, 1523 type = "info", 1524 header = "\t", 1525 back = True 1526 )

1527 1528 def my_show_down_status(message, mytype): 1529 self.Entropy.updateProgress( 1530 message, 1531 importance = 0, 1532 type = mytype, 1533 header = "\t" 1534 ) 1535 1536 def my_show_file_unpack(fp): 1537 self.Entropy.updateProgress( 1538 "%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),), 1539 header = blue("\t << ") 1540 ) 1541 1542 def my_show_file_rm(fp): 1543 self.Entropy.updateProgress( 1544 "%s: %s" % (darkgreen(_("removed meta file")), purple(fp),), 1545 header = blue("\t << ") 1546 ) 1547 1548 for item, myfile, ignorable, mytxt in download_items: 1549 1550 my_show_info(mytxt) 1551 mystatus = self.download_item(item, repo, disallow_redirect = True) 1552 mytype = 'info' 1553 1554 # download failed, is it critical? 1555 if not mystatus: 1556 if ignorable: 1557 message = "%s: %s." % (blue(myfile), 1558 red(_("not available, it's ok"))) 1559 else: 1560 mytype = 'warning' 1561 message = "%s: %s." % (blue(myfile), 1562 darkred(_("not available, not very ok!"))) 1563 my_show_down_status(message, mytype) 1564 continue 1565 1566 myurl, mypath = self._construct_paths(item, repo, None) 1567 message = "%s: %s." % (blue(myfile), 1568 darkgreen(_("available, w00t!"))) 1569 my_show_down_status(message, mytype) 1570 if item not in objects_to_unpack: 1571 continue 1572 if not (os.path.isfile(mypath) and os.access(mypath, os.R_OK)): 1573 continue 1574 1575 tmpdir = tempfile.mkdtemp() 1576 repo_dir = repo_data['dbpath'] 1577 try: 1578 done = entropy.tools.universal_uncompress(mypath, tmpdir, 1579 catch_empty = True) 1580 if not done: 1581 mytype = 'warning' 1582 message = "%s: %s." % (blue(myfile), 1583 darkred(_("cannot be unpacked, not very ok!"))) 1584 my_show_down_status(message, mytype) 1585 continue 1586 myfiles_to_move = set(os.listdir(tmpdir)) 1587 1588 # exclude files not available by default 1589 files_not_found_file = etpConst['etpdatabasemetafilesnotfound'] 1590 if files_not_found_file in myfiles_to_move: 1591 myfiles_to_move.remove(files_not_found_file) 1592 fnf_path = os.path.join(tmpdir, files_not_found_file) 1593 1594 if os.path.isfile(fnf_path) and \ 1595 os.access(fnf_path, os.R_OK): 1596 with open(fnf_path, "r") as f: 1597 f_nf = [x.strip() for x in f.readlines()] 1598 1599 for myfile in f_nf: 1600 myfile = os.path.basename(myfile) # avoid lamerz 1601 myfpath = os.path.join(repo_dir, myfile) 1602 if os.path.isfile(myfpath) and \ 1603 os.access(myfpath, os.W_OK): 1604 try: 1605 os.remove(myfpath) 1606 my_show_file_rm(myfile) 1607 except OSError: 1608 pass 1609 1610 for myfile in sorted(myfiles_to_move): 1611 from_mypath = os.path.join(tmpdir, myfile) 1612 to_mypath = os.path.join(repo_dir, myfile) 1613 try: 1614 os.rename(from_mypath, to_mypath) 1615 my_show_file_unpack(myfile) 1616 except OSError: 1617 continue 1618 1619 finally: 1620 shutil.rmtree(tmpdir, True) 1621 1622 1623 mytxt = "%s: %s" % ( 1624 red(_("Repository revision")), 1625 bold(str(self.Entropy.get_repository_revision(repo))), 1626 ) 1627 self.Entropy.updateProgress( 1628 mytxt, 1629 importance = 1, 1630 type = "info", 1631 header = "\t" 1632 ) 1633

1634 - def do_database_indexing(self, repo):

1635 1636 # renice a bit, to avoid eating resources 1637 old_prio = self.Entropy.set_priority(15) 1638 mytxt = red("%s ...") % (_("Indexing Repository metadata"),) 1639 self.Entropy.updateProgress( 1640 mytxt, 1641 importance = 1, 1642 type = "info", 1643 header = "\t", 1644 back = True 1645 ) 1646 dbconn = self.Entropy.open_repository(repo) 1647 dbconn.createAllIndexes() 1648 dbconn.commitChanges(force = True) 1649 # get list of indexes 1650 repo_indexes = dbconn.listAllIndexes() 1651 if self.Entropy.clientDbconn != None: 1652 try: # client db can be absent 1653 client_indexes = self.Entropy.clientDbconn.listAllIndexes() 1654 if repo_indexes != client_indexes: 1655 self.Entropy.clientDbconn.createAllIndexes() 1656 except: 1657 pass 1658 self.Entropy.set_priority(old_prio)

1659

1660 - def sync(self):

1661 1662 # close them 1663 self.Entropy.close_all_repositories() 1664 1665 # let's dance! 1666 mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),) 1667 self.Entropy.updateProgress( 1668 mytxt, 1669 importance = 2, 1670 type = "info", 1671 header = darkred(" @@ ") 1672 ) 1673 1674 gave_up = self.Entropy.lock_check(self.Entropy.resources_check_lock) 1675 if gave_up: 1676 return 3 1677 1678 locked = self.Entropy.application_lock_check() 1679 if locked: 1680 return 4 1681 1682 # lock 1683 acquired = self.Entropy.resources_create_lock() 1684 if not acquired: 1685 return 4 # app locked during lock acquire 1686 try: 1687 rc = self.run_sync() 1688 finally: 1689 self.Entropy.resources_remove_lock() 1690 if rc: 1691 return rc 1692 1693 # remove lock 1694 self.Entropy.resources_remove_lock() 1695 1696 if (self.notAvailable >= len(self.reponames)): 1697 return 2 1698 elif (self.notAvailable > 0): 1699 return 1 1700 1701 return 0

1702

Source Code for Module entropy.client.interfaces.repository