entropy.client.interfaces.repository

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Client Repositories Management Interface}. 10 11 """ 12 13 from __future__ import with_statement 14 import os 15 import time 16 import shutil 17 import subprocess 18 import random 19 from entropy.i18n import _ 20 from entropy.db import dbapi2, EntropyRepository 21 from entropy.misc import TimeScheduled 22 from entropy.const import * 23 from entropy.exceptions import * 24 from entropy.output import * 25

26 -class Repository:

27 28 import entropy.dump as dumpTools 29 import entropy.tools as entropyTools 30 import socket

31 - def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):

32 33 self.LockScanner = None 34 from entropy.client.interfaces import Client 35 if not isinstance(EquoInstance,Client): 36 mytxt = _("A valid Equo instance or subclass is needed") 37 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 38 39 self.supported_download_items = ( 40 "db", "dblight", "rev", "ck", "cklight", "compck", 41 "lock","mask", "system_mask", "dbdump", "conflicting_tagged", 42 "dbdumplight", "dbdumplightck", "dbdumpck", "lic_whitelist", 43 "make.conf", "package.mask", "package.unmask", "package.keywords", 44 "profile.link", "package.use", "server.cert", "ca.cert", 45 "meta_file", "notice_board", "critical_updates", "keywords" 46 ) 47 self.big_socket_timeout = 10 48 self.Entropy = EquoInstance 49 from entropy.cache import EntropyCacher 50 self.Cacher = EntropyCacher() 51 self.dbapi2 = dbapi2 52 self.reponames = reponames 53 self.forceUpdate = forceUpdate 54 self.syncErrors = False 55 self.dbupdated = False 56 self.newEquo = False 57 self.fetchSecurity = fetchSecurity 58 self.noEquoCheck = noEquoCheck 59 self.alreadyUpdated = 0 60 self.notAvailable = 0 61 self.valid_eapis = [1,2,3] 62 self.reset_dbformat_eapi(None) 63 self.current_repository_got_locked = False 64 self.updated_repos = set() 65 66 # check self.Entropy.SystemSettings['repositories']['available'] 67 if not self.Entropy.SystemSettings['repositories']['available']: 68 mytxt = _("No repositories specified in %s") % (etpConst['repositoriesconf'],) 69 raise MissingParameter("MissingParameter: %s" % (mytxt,)) 70 71 if not self.reponames: 72 self.reponames.extend(self.Entropy.SystemSettings['repositories']['available'].keys()[:])

73

74 - def __del__(self):

75 if self.LockScanner != None: 76 self.LockScanner.kill()

77

78 - def get_eapi3_connection(self, repository):

79 # get database url 80 dburl = self.Entropy.SystemSettings['repositories']['available'][repository]['plain_database'] 81 if dburl.startswith("file://"): 82 return None 83 try: 84 dburl = dburl.split("/")[2] 85 except IndexError: 86 return None 87 port = self.Entropy.SystemSettings['repositories']['available'][repository]['service_port'] 88 try: 89 from entropy.services.ugc.interfaces import Client 90 from entropy.client.services.ugc.commands import Client as CommandsClient 91 eapi3_socket = Client(self.Entropy, CommandsClient, 92 output_header = "\t", socket_timeout = self.big_socket_timeout) 93 eapi3_socket.connect(dburl, port) 94 return eapi3_socket 95 except (ConnectionError,self.socket.error,): 96 return None

97

98 - def check_eapi3_availability(self, repository):

99 conn = self.get_eapi3_connection(repository) 100 if conn == None: return False 101 try: 102 conn.disconnect() 103 except (self.socket.error,AttributeError,): 104 return False 105 return True

106

107 - def reset_dbformat_eapi(self, repository):

108 109 self.dbformat_eapi = 2 110 if repository != None: 111 eapi_avail = self.check_eapi3_availability(repository) 112 if eapi_avail: 113 self.dbformat_eapi = 3 114 115 # FIXME, find a way to do that without needing sqlite3 exec. 116 if not os.access("/usr/bin/sqlite3",os.X_OK) or self.entropyTools.islive(): 117 self.dbformat_eapi = 1 118 else: 119 rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True) 120 if rc != 0: self.dbformat_eapi = 1 121 122 eapi_env = os.getenv("FORCE_EAPI") 123 if eapi_env != None: 124 try: 125 myeapi = int(eapi_env) 126 except (ValueError,TypeError,): 127 return 128 if myeapi in self.valid_eapis: 129 self.dbformat_eapi = myeapi

130 131

132 - def __validate_repository_id(self, repoid):

133 if repoid not in self.reponames: 134 mytxt = _("Repository is not listed in self.reponames") 135 raise InvalidData("InvalidData: %s" % (mytxt,))

136

137 - def __validate_compression_method(self, repo):

138 139 self.__validate_repository_id(repo) 140 141 repo_settings = self.Entropy.SystemSettings['repositories'] 142 dbc_format = repo_settings['available'][repo]['dbcformat'] 143 cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format) 144 if cmethod == None: 145 mytxt = _("Wrong database compression method") 146 raise InvalidDataType("InvalidDataType: %s" % (mytxt,)) 147 148 return cmethod

149

150 - def __ensure_repository_path(self, repo):

151 152 self.__validate_repository_id(repo) 153 154 # create dir if it doesn't exist 155 if not os.path.isdir(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']): 156 os.makedirs(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],0775) 157 158 const_setup_perms(etpConst['etpdatabaseclientdir'],etpConst['entropygid'])

159

160 - def _construct_paths(self, item, repo, cmethod):

161 162 if item not in self.supported_download_items: 163 mytxt = _("Supported items: %s") % (self.supported_download_items,) 164 raise InvalidData("InvalidData: %s" % (mytxt,)) 165 166 items_needing_cmethod = ( 167 "db", "dblight", "cklight", "dbdump", "dbdumpck", 168 "dbdumplight", "dbdumplightck", "compck", 169 ) 170 if (item in items_needing_cmethod) and (cmethod == None): 171 mytxt = _("For %s, cmethod can't be None") % (item,) 172 raise InvalidData("InvalidData: %s" % (mytxt,)) 173 174 repo_db = self.Entropy.SystemSettings['repositories']['available'][repo]['database'] 175 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 176 ec_rev = etpConst['etpdatabaserevisionfile'] 177 ec_hash = etpConst['etpdatabasehashfile'] 178 ec_maskfile = etpConst['etpdatabasemaskfile'] 179 ec_sysmaskfile = etpConst['etpdatabasesytemmaskfile'] 180 ec_keywords = etpConst['etpdatabasekeywordsfile'] 181 ec_confl_taged = etpConst['etpdatabaseconflictingtaggedfile'] 182 ec_crit_updates = etpConst['etpdatabasecriticalfile'] 183 make_conf_file = os.path.basename(etpConst['spm']['global_make_conf']) 184 pkg_mask_file = os.path.basename(etpConst['spm']['global_package_mask']) 185 pkg_unmask_file = os.path.basename(etpConst['spm']['global_package_unmask']) 186 pkg_keywords_file = os.path.basename(etpConst['spm']['global_package_keywords']) 187 pkg_use_file = os.path.basename(etpConst['spm']['global_package_use']) 188 sys_profile_lnk = etpConst['spm']['global_make_profile_link_name'] 189 pkg_lic_wl_file = etpConst['etpdatabaselicwhitelistfile'] 190 repo_lock_file = etpConst['etpdatabasedownloadlockfile'] 191 ca_cert_file = etpConst['etpdatabasecacertfile'] 192 server_cert_file = etpConst['etpdatabaseservercertfile'] 193 notice_board_filename = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board']) 194 meta_file = etpConst['etpdatabasemetafilesfile'] 195 md5_ext = etpConst['packagesmd5fileext'] 196 ec_cm2 = None 197 ec_cm3 = None 198 ec_cm4 = None 199 ec_cm5 = None 200 ec_cm6 = None 201 ec_cm7 = None 202 ec_cm8 = None 203 if cmethod != None: 204 ec_cm2 = etpConst[cmethod[2]] 205 ec_cm3 = etpConst[cmethod[3]] 206 ec_cm4 = etpConst[cmethod[4]] 207 ec_cm5 = etpConst[cmethod[5]] 208 ec_cm6 = etpConst[cmethod[6]] 209 ec_cm7 = etpConst[cmethod[7]] 210 ec_cm8 = etpConst[cmethod[8]] 211 212 mymap = { 213 'db': ("%s/%s" % (repo_db,ec_cm2,),"%s/%s" % (repo_dbpath,ec_cm2,),), 214 'dblight': ("%s/%s" % (repo_db,ec_cm7,),"%s/%s" % (repo_dbpath,ec_cm7,),), 215 'dbdump': ("%s/%s" % (repo_db,ec_cm3,),"%s/%s" % (repo_dbpath,ec_cm3,),), 216 'dbdumplight': ("%s/%s" % (repo_db,ec_cm5,),"%s/%s" % (repo_dbpath,ec_cm5,),), 217 'rev': ("%s/%s" % (repo_db,ec_rev,),"%s/%s" % (repo_dbpath,ec_rev,),), 218 'ck': ("%s/%s" % (repo_db,ec_hash,),"%s/%s" % (repo_dbpath,ec_hash,),), 219 'cklight': ("%s/%s" % (repo_db,ec_cm8,),"%s/%s" % (repo_dbpath,ec_cm8,),), 220 'compck': ("%s/%s%s" % (repo_db, ec_cm2, md5_ext,),"%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),), 221 'dbdumpck': ("%s/%s" % (repo_db,ec_cm4,),"%s/%s" % (repo_dbpath,ec_cm4,),), 222 'dbdumplightck': ("%s/%s" % (repo_db,ec_cm6,),"%s/%s" % (repo_dbpath,ec_cm6,),), 223 'mask': ("%s/%s" % (repo_db,ec_maskfile,),"%s/%s" % (repo_dbpath,ec_maskfile,),), 224 'keywords': ("%s/%s" % (repo_db,ec_keywords,),"%s/%s" % (repo_dbpath,ec_keywords,),), 225 'system_mask': ("%s/%s" % (repo_db,ec_sysmaskfile,),"%s/%s" % (repo_dbpath,ec_sysmaskfile,),), 226 'conflicting_tagged': ("%s/%s" % (repo_db,ec_confl_taged,),"%s/%s" % (repo_dbpath,ec_confl_taged,),), 227 'critical_updates': ("%s/%s" % (repo_db,ec_crit_updates,),"%s/%s" % (repo_dbpath,ec_crit_updates,),), 228 'make.conf': ("%s/%s" % (repo_db,make_conf_file,),"%s/%s" % (repo_dbpath,make_conf_file,),), 229 'package.mask': ("%s/%s" % (repo_db,pkg_mask_file,),"%s/%s" % (repo_dbpath,pkg_mask_file,),), 230 'package.unmask': ("%s/%s" % (repo_db,pkg_unmask_file,),"%s/%s" % (repo_dbpath,pkg_unmask_file,),), 231 'package.keywords': ("%s/%s" % (repo_db,pkg_keywords_file,),"%s/%s" % (repo_dbpath,pkg_keywords_file,),), 232 'package.use': ("%s/%s" % (repo_db,pkg_use_file,),"%s/%s" % (repo_dbpath,pkg_use_file,),), 233 'profile.link': ("%s/%s" % (repo_db,sys_profile_lnk,),"%s/%s" % (repo_dbpath,sys_profile_lnk,),), 234 'lic_whitelist': ("%s/%s" % (repo_db,pkg_lic_wl_file,),"%s/%s" % (repo_dbpath,pkg_lic_wl_file,),), 235 'lock': ("%s/%s" % (repo_db,repo_lock_file,),"%s/%s" % (repo_dbpath,repo_lock_file,),), 236 'server.cert': ("%s/%s" % (repo_db,server_cert_file,),"%s/%s" % (repo_dbpath,server_cert_file,),), 237 'ca.cert': ("%s/%s" % (repo_db,ca_cert_file,),"%s/%s" % (repo_dbpath,ca_cert_file,),), 238 'notice_board': (self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board'],"%s/%s" % (repo_dbpath,notice_board_filename,),), 239 'meta_file': ("%s/%s" % (repo_db,meta_file,),"%s/%s" % (repo_dbpath,meta_file,),), 240 } 241 242 return mymap.get(item)

243

244 - def __remove_repository_files(self, repo, cmethod):

245 246 dbfilenameid = cmethod[2] 247 dblightfilenameid = cmethod[7] 248 self.__validate_repository_id(repo) 249 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 250 251 def remove_eapi1(): 252 253 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabasehashfile']): 254 os.remove(repo_dbpath+"/"+etpConst['etpdatabasehashfile']) 255 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 256 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 257 258 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid]): 259 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid]) 260 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 261 etpConst['packagesmd5fileext']): 262 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 263 etpConst['packagesmd5fileext']) 264 265 if os.path.isfile(repo_dbpath+"/"+etpConst[dbfilenameid]): 266 os.remove(repo_dbpath+"/"+etpConst[dbfilenameid]) 267 if os.path.isfile(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 268 etpConst['packagesmd5fileext']): 269 os.remove(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 270 etpConst['packagesmd5fileext'])

271 272 if self.dbformat_eapi == 1: 273 remove_eapi1() 274 elif self.dbformat_eapi in (2,3,): 275 remove_eapi1() 276 if os.path.isfile(repo_dbpath+"/"+cmethod[6]): 277 os.remove(repo_dbpath+"/"+cmethod[6]) 278 if os.path.isfile(repo_dbpath+"/"+etpConst[cmethod[5]]): 279 os.remove(repo_dbpath+"/"+etpConst[cmethod[5]]) 280 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 281 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 282 else: 283 mytxt = _("self.dbformat_eapi must be in (1,2,3,)") 284 raise InvalidData('InvalidData: %s' % (mytxt,))

285

286 - def __unpack_downloaded_database(self, repo, cmethod):

287 288 self.__validate_repository_id(repo) 289 rc = 0 290 path = None 291 sys_set_repos = self.Entropy.SystemSettings['repositories']['available'] 292 repo_data = sys_set_repos[repo] 293 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[7]] 294 if self.dbformat_eapi == 2: 295 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[5]] 296 297 298 if self.dbformat_eapi in (1, 2,): 299 try: 300 301 myfunc = getattr(self.entropyTools, cmethod[1]) 302 path = myfunc(myfile) 303 # rename path correctly 304 if self.dbformat_eapi == 1: 305 new_path = os.path.join(os.path.dirname(path), 306 etpConst['etpdatabasefile']) 307 os.rename(path, new_path) 308 path = new_path 309 310 except (OSError, EOFError): 311 rc = 1 312 if os.path.isfile(myfile): 313 os.remove(myfile) 314 315 else: 316 mytxt = _("self.dbformat_eapi must be in (1,2)") 317 raise InvalidData('InvalidData: %s' % (mytxt,)) 318 319 if rc == 0: 320 self.Entropy.setup_default_file_perms(path) 321 322 return rc

323

324 - def _verify_file_checksum(self, file_path, md5_checksum_path):

325 ck_f = open(md5_checksum_path, "r") 326 md5hash = ck_f.readline().strip() 327 md5hash = md5hash.split()[0] 328 ck_f.close() 329 return self.entropyTools.compare_md5(file_path, md5hash)

330

331 - def __verify_database_checksum(self, repo, cmethod = None):

332 333 self.__validate_repository_id(repo) 334 sys_settings_repos = self.Entropy.SystemSettings['repositories'] 335 avail_config = sys_settings_repos['available'][repo] 336 337 if self.dbformat_eapi == 1: 338 dbfile = avail_config['dbpath'] + "/" + etpConst[cmethod[7]] 339 md5file = avail_config['dbpath'] + "/" + etpConst[cmethod[8]] 340 341 elif self.dbformat_eapi == 2: 342 dbfile = avail_config['dbpath'] + "/" + etpConst[cmethod[5]] 343 md5file = avail_config['dbpath'] + "/" + etpConst[cmethod[6]] 344 345 else: 346 mytxt = _("self.dbformat_eapi must be in (1,2)") 347 raise InvalidData('InvalidData: %s' % (mytxt,)) 348 349 if not os.access(md5file, os.F_OK | os.R_OK): 350 return -1 351 352 return self._verify_file_checksum(dbfile, md5file)

353 354 # @returns -1 if the file is not available 355 # @returns int>0 if the revision has been retrieved

356 - def get_online_repository_revision(self, repo):

357 358 self.__validate_repository_id(repo) 359 360 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaserevisionfile'] 361 status = self.entropyTools.get_remote_data(url, 362 timeout = self.big_socket_timeout) 363 if status: 364 status = status[0].strip() 365 try: 366 status = int(status) 367 except ValueError: 368 status = -1 369 return status 370 else: 371 return -1

372

373 - def get_online_eapi3_lock(self, repo):

374 self.__validate_repository_id(repo) 375 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaseeapi3lockfile'] 376 data = self.entropyTools.get_remote_data(url) 377 if not data: 378 return False 379 return True

380

381 - def is_repository_eapi3_locked(self, repo):

382 self.__validate_repository_id(repo) 383 return self.get_online_eapi3_lock(repo)

384

385 - def is_repository_updatable(self, repo):

386 387 self.__validate_repository_id(repo) 388 389 onlinestatus = self.get_online_repository_revision(repo) 390 if (onlinestatus != -1): 391 localstatus = self.Entropy.get_repository_revision(repo) 392 if (localstatus == onlinestatus) and (not self.forceUpdate): 393 return False 394 return True

395

396 - def is_repository_unlocked(self, repo):

397 398 self.__validate_repository_id(repo) 399 400 rc = self.download_item("lock", repo, disallow_redirect = True) 401 if rc: # cannot download database 402 self.syncErrors = True 403 return False 404 return True

405

406 - def clear_repository_cache(self, repo):

407 self.__validate_repository_id(repo) 408 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repo,)) 409 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repo,))

410 411 # this function can be reimplemented

412 - def download_item(self, item, repo, cmethod = None, lock_status_func = None, disallow_redirect = True):

413 414 self.__validate_repository_id(repo) 415 url, filepath = self._construct_paths(item, repo, cmethod) 416 417 # to avoid having permissions issues 418 # it's better to remove the file before, 419 # otherwise new permissions won't be written 420 if os.path.isfile(filepath): 421 os.remove(filepath) 422 filepath_dir = os.path.dirname(filepath) 423 if not os.path.isdir(filepath_dir) and not os.path.lexists(filepath_dir): 424 os.makedirs(filepath_dir,0775) 425 const_setup_perms(filepath_dir, etpConst['entropygid']) 426 427 fetchConn = self.Entropy.urlFetcher( 428 url, 429 filepath, 430 resume = False, 431 abort_check_func = lock_status_func, 432 disallow_redirect = disallow_redirect 433 ) 434 fetchConn.progress = self.Entropy.progress 435 436 rc = fetchConn.download() 437 del fetchConn 438 if rc in ("-1","-2","-3","-4"): 439 return False 440 self.Entropy.setup_default_file_perms(filepath) 441 return True

442

443 - def check_downloaded_database(self, repo, cmethod):

444 445 dbfilename = etpConst[cmethod[7]] 446 if self.dbformat_eapi == 2: 447 dbfilename = etpConst[cmethod[5]] 448 # verify checksum 449 mytxt = "%s %s %s" % ( 450 red(_("Checking downloaded database")), 451 darkgreen(dbfilename), 452 red("..."), 453 ) 454 self.Entropy.updateProgress( 455 mytxt, 456 importance = 0, 457 back = True, 458 type = "info", 459 header = "\t" 460 ) 461 db_status = self.__verify_database_checksum(repo, cmethod) 462 if db_status == -1: 463 mytxt = "%s. %s !" % ( 464 red(_("Cannot open digest")), 465 red(_("Cannot verify database integrity")), 466 ) 467 self.Entropy.updateProgress( 468 mytxt, 469 importance = 1, 470 type = "warning", 471 header = "\t" 472 ) 473 elif db_status: 474 mytxt = "%s: %s" % ( 475 red(_("Downloaded database status")), 476 bold(_("OK")), 477 ) 478 self.Entropy.updateProgress( 479 mytxt, 480 importance = 1, 481 type = "info", 482 header = "\t" 483 ) 484 else: 485 mytxt = "%s: %s" % ( 486 red(_("Downloaded database status")), 487 darkred(_("ERROR")), 488 ) 489 self.Entropy.updateProgress( 490 mytxt, 491 importance = 1, 492 type = "error", 493 header = "\t" 494 ) 495 mytxt = "%s. %s" % ( 496 red(_("An error occured while checking database integrity")), 497 red(_("Giving up")), 498 ) 499 self.Entropy.updateProgress( 500 mytxt, 501 importance = 1, 502 type = "error", 503 header = "\t" 504 ) 505 return 1 506 return 0

507 508

509 - def show_repository_information(self, repo, count_info):

510 511 self.Entropy.updateProgress( 512 bold("%s") % ( self.Entropy.SystemSettings['repositories']['available'][repo]['description'] ), 513 importance = 2, 514 type = "info", 515 count = count_info, 516 header = blue(" # ") 517 ) 518 mytxt = "%s: %s" % (red(_("Database URL")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['database']),) 519 self.Entropy.updateProgress( 520 mytxt, 521 importance = 1, 522 type = "info", 523 header = blue(" # ") 524 ) 525 mytxt = "%s: %s" % (red(_("Database local path")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']),) 526 self.Entropy.updateProgress( 527 mytxt, 528 importance = 0, 529 type = "info", 530 header = blue(" # ") 531 ) 532 mytxt = "%s: %s" % (red(_("Database EAPI")),darkgreen(str(self.dbformat_eapi)),) 533 self.Entropy.updateProgress( 534 mytxt, 535 importance = 0, 536 type = "info", 537 header = blue(" # ") 538 )

539

540 - def get_eapi3_local_database(self, repo):

541 542 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 543 mydbconn = None 544 try: 545 mydbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 546 mydbconn.validateDatabase() 547 except ( 548 self.Entropy.dbapi2.OperationalError, 549 self.Entropy.dbapi2.IntegrityError, 550 SystemDatabaseError, 551 IOError, 552 OSError,): 553 mydbconn = None 554 return mydbconn

555

556 - def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, session):

557 558 product = self.Entropy.SystemSettings['repositories']['product'] 559 data = eapi3_interface.CmdInterface.differential_packages_comparison( 560 session, idpackages, repo, etpConst['currentarch'], product 561 ) 562 if isinstance(data,bool): # then it's probably == False 563 return False,False,False 564 elif not isinstance(data,dict): 565 return None,None,None 566 elif not data.has_key('added') or \ 567 not data.has_key('removed') or \ 568 not data.has_key('checksum'): 569 return None,None,None 570 return data['added'],data['removed'],data['checksum']

571

572 - def get_eapi3_repository_metadata(self, eapi3_interface, repo, session):

573 product = self.Entropy.SystemSettings['repositories']['product'] 574 self.socket.setdefaulttimeout(self.big_socket_timeout) 575 data = eapi3_interface.CmdInterface.get_repository_metadata( 576 session, repo, etpConst['currentarch'], product 577 ) 578 if not isinstance(data,dict): return {} 579 return data

580

581 - def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12):

582 583 def prepare_exit(mysock, session = None): 584 try: 585 if session != None: 586 mysock.close_session(session) 587 mysock.disconnect() 588 except (self.socket.error,): 589 pass

590 591 eapi3_interface = self.get_eapi3_connection(repo) 592 if eapi3_interface == None: return False 593 594 session = eapi3_interface.open_session() 595 596 # AttributeError because mydbconn can be == None 597 try: 598 mydbconn = self.get_eapi3_local_database(repo) 599 myidpackages = mydbconn.listAllIdpackages() 600 except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,AttributeError,): 601 prepare_exit(eapi3_interface, session) 602 return False 603 604 added_ids, removed_ids, checksum = self.get_eapi3_database_differences( 605 eapi3_interface, repo, 606 myidpackages, session 607 ) 608 if (None in (added_ids,removed_ids,checksum)) or \ 609 (not added_ids and not removed_ids and self.forceUpdate): 610 mydbconn.closeDB() 611 prepare_exit(eapi3_interface, session) 612 return False 613 614 elif not checksum: # {added_ids, removed_ids, checksum} == False 615 mydbconn.closeDB() 616 prepare_exit(eapi3_interface, session) 617 mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("remote database suddenly locked")),) 618 self.Entropy.updateProgress( 619 mytxt, 620 importance = 0, 621 type = "info", 622 header = blue(" # "), 623 ) 624 return None 625 626 # is it worth it? 627 if len(added_ids) > threshold: 628 mytxt = "%s: %s (%s: %s/%s)" % ( 629 blue(_("EAPI3 Service")), darkred(_("skipping differential sync")), 630 brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)), 631 ) 632 self.Entropy.updateProgress( 633 mytxt, 634 importance = 0, 635 type = "info", 636 header = blue(" # "), 637 ) 638 mydbconn.closeDB() 639 prepare_exit(eapi3_interface, session) 640 return False 641 642 count = 0 643 added_segments = [] 644 mytmp = set() 645 646 for idpackage in added_ids: 647 count += 1 648 mytmp.add(idpackage) 649 if count % chunk_size == 0: 650 added_segments.append(list(mytmp)) 651 mytmp.clear() 652 if mytmp: added_segments.append(list(mytmp)) 653 del mytmp 654 655 # fetch and store 656 count = 0 657 maxcount = len(added_segments) 658 product = self.Entropy.SystemSettings['repositories']['product'] 659 for segment in added_segments: 660 661 count += 1 662 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) 663 self.Entropy.updateProgress( 664 mytxt, importance = 0, type = "info", 665 header = "\t", back = True, count = (count,maxcount,) 666 ) 667 fetch_count = 0 668 max_fetch_count = 5 669 670 while 1: 671 672 # anti loop protection 673 if fetch_count > max_fetch_count: 674 mydbconn.closeDB() 675 prepare_exit(eapi3_interface, session) 676 return False 677 678 fetch_count += 1 679 pkgdata = eapi3_interface.CmdInterface.get_strict_package_information( 680 session, segment, repo, etpConst['currentarch'], product 681 ) 682 if pkgdata == None: 683 mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), 684 darkred(str(segment)),) 685 self.Entropy.updateProgress( 686 mytxt, importance = 1, type = "warning", 687 header = "\t", count = (count,maxcount,) 688 ) 689 continue 690 elif not pkgdata: # pkgdata == False 691 mytxt = "%s: %s" % ( 692 blue(_("Service status")), 693 darkred("remote database suddenly locked"), 694 ) 695 self.Entropy.updateProgress( 696 mytxt, importance = 1, type = "info", 697 header = "\t", count = (count,maxcount,) 698 ) 699 mydbconn.closeDB() 700 prepare_exit(eapi3_interface, session) 701 return None 702 elif isinstance(pkgdata,tuple): 703 mytxt = "%s: %s, %s. %s" % ( 704 blue(_("Service status")), 705 pkgdata[0], pkgdata[1], 706 darkred("Error processing the command"), 707 ) 708 self.Entropy.updateProgress( 709 mytxt, importance = 1, type = "info", 710 header = "\t", count = (count,maxcount,) 711 ) 712 mydbconn.closeDB() 713 prepare_exit(eapi3_interface, session) 714 return None 715 716 try: 717 for idpackage in pkgdata: 718 self.dumpTools.dumpobj( 719 "%s%s" % (etpCache['eapi3_fetch'],idpackage,), 720 pkgdata[idpackage], 721 ignore_exceptions = False 722 ) 723 except (IOError,EOFError,OSError,), e: 724 mytxt = "%s: %s: %s." % ( 725 blue(_("Local status")), 726 darkred("Error storing data"), 727 e, 728 ) 729 self.Entropy.updateProgress( 730 mytxt, importance = 1, type = "info", 731 header = "\t", count = (count,maxcount,) 732 ) 733 mydbconn.closeDB() 734 prepare_exit(eapi3_interface, session) 735 return None 736 737 break 738 739 del added_segments 740 741 repo_metadata = self.get_eapi3_repository_metadata(eapi3_interface, 742 repo, session) 743 metadata_elements = ("sets", "treeupdates_actions", 744 "treeupdates_digest", "library_idpackages",) 745 for elem in metadata_elements: 746 if elem not in repo_metadata: 747 mydbconn.closeDB() 748 prepare_exit(eapi3_interface, session) 749 mytxt = "%s: %s" % ( 750 blue(_("EAPI3 Service status")), 751 darkred(_("cannot fetch repository metadata")), 752 ) 753 self.Entropy.updateProgress( 754 mytxt, 755 importance = 0, 756 type = "info", 757 header = blue(" # "), 758 ) 759 return None 760 761 # update treeupdates 762 try: 763 mydbconn.setRepositoryUpdatesDigest(repo, 764 repo_metadata['treeupdates_digest']) 765 mydbconn.bumpTreeUpdatesActions( 766 repo_metadata['treeupdates_actions']) 767 except (self.dbapi2.Error,): 768 mydbconn.closeDB() 769 prepare_exit(eapi3_interface, session) 770 mytxt = "%s: %s" % ( 771 blue(_("EAPI3 Service status")), 772 darkred(_("cannot update treeupdates data")), 773 ) 774 self.Entropy.updateProgress( 775 mytxt, 776 importance = 0, 777 type = "info", 778 header = blue(" # "), 779 ) 780 return None 781 782 # update package sets 783 try: 784 mydbconn.clearPackageSets() 785 mydbconn.insertPackageSets(repo_metadata['sets']) 786 except (self.dbapi2.Error,): 787 mydbconn.closeDB() 788 prepare_exit(eapi3_interface, session) 789 mytxt = "%s: %s" % ( 790 blue(_("EAPI3 Service status")), 791 darkred(_("cannot update package sets data")), 792 ) 793 self.Entropy.updateProgress( 794 mytxt, 795 importance = 0, 796 type = "info", 797 header = blue(" # "), 798 ) 799 return None 800 801 # update libraries <=> idpackages map 802 try: 803 mydbconn.clearNeededLibraryIdpackages() 804 mydbconn.setNeededLibraryIdpackages( 805 repo_metadata['library_idpackages']) 806 except (self.dbapi2.Error,): 807 mydbconn.closeDB() 808 prepare_exit(eapi3_interface, session) 809 mytxt = "%s: %s" % ( 810 blue(_("EAPI3 Service status")), 811 darkred(_("cannot update library data")), 812 ) 813 self.Entropy.updateProgress( 814 mytxt, 815 importance = 0, 816 type = "info", 817 header = blue(" # "), 818 ) 819 return None 820 821 # I don't need you anymore 822 # disconnect socket 823 prepare_exit(eapi3_interface, session) 824 825 # now that we have all stored, add 826 count = 0 827 maxcount = len(added_ids) 828 for idpackage in added_ids: 829 count += 1 830 mydata = self.Cacher.pop("%s%s" % (etpCache['eapi3_fetch'],idpackage,)) 831 if mydata == None: 832 mytxt = "%s: %s" % ( 833 blue(_("Fetch error on segment while adding")), 834 darkred(str(segment)), 835 ) 836 self.Entropy.updateProgress( 837 mytxt, importance = 1, type = "warning", 838 header = "\t", count = (count,maxcount,) 839 ) 840 mydbconn.closeDB() 841 return False 842 843 mytxt = "%s %s" % (blue(_("Injecting package")), darkgreen(mydata['atom']),) 844 self.Entropy.updateProgress( 845 mytxt, importance = 0, type = "info", 846 header = "\t", back = True, count = (count,maxcount,) 847 ) 848 try: 849 mydbconn.addPackage( 850 mydata, revision = mydata['revision'], 851 idpackage = idpackage, do_commit = False, 852 formatted_content = True 853 ) 854 except (self.dbapi2.Error,): 855 self.Entropy.updateProgress( 856 blue(_("repository error while adding packages")), 857 importance = 1, type = "warning", 858 header = "\t", count = (count, maxcount,) 859 ) 860 mydbconn.closeDB() 861 return False 862 863 self.Entropy.updateProgress( 864 blue(_("Packages injection complete")), importance = 0, 865 type = "info", header = "\t", 866 ) 867 868 # now remove 869 maxcount = len(removed_ids) 870 count = 0 871 for idpackage in removed_ids: 872 myatom = mydbconn.retrieveAtom(idpackage) 873 count += 1 874 mytxt = "%s: %s" % (blue(_("Removing package")), darkred(str(myatom)),) 875 self.Entropy.updateProgress( 876 mytxt, importance = 0, type = "info", 877 header = "\t", back = True, count = (count,maxcount,) 878 ) 879 try: 880 mydbconn.removePackage(idpackage, do_cleanup = False, 881 do_commit = False) 882 except (self.dbapi2.Error,): 883 self.Entropy.updateProgress( 884 blue(_("repository error while removing packages")), 885 importance = 1, type = "warning", 886 header = "\t", count = (count, maxcount,) 887 ) 888 mydbconn.closeDB() 889 return False 890 891 self.Entropy.updateProgress( 892 blue(_("Packages removal complete")), 893 importance = 0, type = "info", 894 header = "\t", 895 ) 896 897 mydbconn.commitChanges() 898 mydbconn.clearCache() 899 # now verify if both checksums match 900 result = False 901 mychecksum = mydbconn.checksum(do_order = True, 902 strict = False, strings = True) 903 if checksum == mychecksum: 904 result = True 905 else: 906 self.Entropy.updateProgress( 907 blue(_("Database checksum doesn't match remote.")), 908 importance = 0, type = "info", header = "\t", 909 ) 910 mytxt = "%s: %s" % (_('local'), mychecksum,) 911 self.Entropy.updateProgress( 912 mytxt, importance = 0, 913 type = "info", header = "\t", 914 ) 915 mytxt = "%s: %s" % (_('remote'), checksum,) 916 self.Entropy.updateProgress( 917 mytxt, importance = 0, 918 type = "info", header = "\t", 919 ) 920 921 mydbconn.closeDB() 922 return result 923

924 - def run_sync(self):

925 926 self.dbupdated = False 927 repocount = 0 928 repolength = len(self.reponames) 929 for repo in self.reponames: 930 931 repocount += 1 932 self.reset_dbformat_eapi(repo) 933 self.show_repository_information(repo, (repocount,repolength)) 934 935 if not self.forceUpdate: 936 updated = self.handle_repository_update(repo) 937 if updated: 938 self.Entropy.cycleDone() 939 self.alreadyUpdated += 1 940 continue 941 942 locked = self.handle_repository_lock(repo) 943 if locked: 944 self.notAvailable += 1 945 self.Entropy.cycleDone() 946 continue 947 948 # clear database interface cache belonging to this repository 949 self.clear_repository_cache(repo) 950 self.__ensure_repository_path(repo) 951 952 # dealing with EAPI 953 # setting some vars 954 do_skip = False 955 skip_this_repo = False 956 db_checksum_down_status = False 957 do_db_update_transfer = False 958 rc = 0 959 # some variables 960 dumpfile = os.path.join( 961 self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'], 962 etpConst['etpdatabasedumplight']) 963 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 964 dbfile_old = dbfile+".sync" 965 cmethod = self.__validate_compression_method(repo) 966 967 while 1: 968 969 if do_skip: 970 break 971 972 if self.dbformat_eapi < 3: 973 974 down_status = self.handle_database_download(repo, cmethod) 975 if not down_status: 976 self.Entropy.cycleDone() 977 self.notAvailable += 1 978 do_skip = True 979 skip_this_repo = True 980 continue 981 db_checksum_down_status = self.handle_database_checksum_download( 982 repo, cmethod) 983 break 984 985 elif self.dbformat_eapi == 3 and not \ 986 (os.path.isfile(dbfile) and os.access(dbfile,os.W_OK)): 987 988 do_db_update_transfer = None 989 self.dbformat_eapi -= 1 990 continue 991 992 elif self.dbformat_eapi == 3: 993 994 status = False 995 try: 996 status = self.handle_eapi3_database_sync(repo) 997 except self.socket.error, e: 998 mytxt = "%s: %s" % ( 999 blue(_("EAPI3 Service error")), 1000 darkred(unicode(e)), 1001 ) 1002 self.Entropy.updateProgress( 1003 mytxt, 1004 importance = 0, 1005 type = "info", 1006 header = blue(" # "), 1007 ) 1008 except: 1009 # avoid broken entries, deal with every exception 1010 self.__remove_repository_files(repo, cmethod) 1011 raise 1012 1013 if status == None: # remote db not available anymore ? 1014 time.sleep(5) 1015 locked = self.handle_repository_lock(repo) 1016 if locked: 1017 self.Entropy.cycleDone() 1018 self.notAvailable += 1 1019 do_skip = True 1020 skip_this_repo = True 1021 else: # ah, well... dunno then... 1022 do_db_update_transfer = None 1023 self.dbformat_eapi -= 1 1024 continue 1025 elif not status: # (status == False) 1026 # set to none and completely skip database alignment 1027 do_db_update_transfer = None 1028 self.dbformat_eapi -= 1 1029 continue 1030 1031 break 1032 1033 if skip_this_repo: 1034 continue 1035 1036 if self.dbformat_eapi in (1, 2,): 1037 1038 # new policy, always deny repository if 1039 # its database checksum cannot be fetched 1040 if not db_checksum_down_status: 1041 # delete all 1042 self.__remove_repository_files(repo, cmethod) 1043 self.syncErrors = True 1044 self.Entropy.cycleDone() 1045 continue 1046 1047 rc = self.check_downloaded_database(repo, cmethod) 1048 if rc != 0: 1049 # delete all 1050 self.__remove_repository_files(repo, cmethod) 1051 self.syncErrors = True 1052 self.Entropy.cycleDone() 1053 continue 1054 1055 # if do_db_update_transfer == False and not None 1056 if (do_db_update_transfer is not None) and not \ 1057 do_db_update_transfer: 1058 1059 if os.access(dbfile, os.R_OK | os.W_OK | os.F_OK): 1060 try: 1061 os.rename(dbfile, dbfile_old) 1062 do_db_update_transfer = True 1063 except OSError: 1064 do_db_update_transfer = False 1065 1066 unpack_status = self.handle_downloaded_database_unpack(repo, 1067 cmethod) 1068 1069 if not unpack_status: 1070 # delete all 1071 self.__remove_repository_files(repo, cmethod) 1072 self.syncErrors = True 1073 self.Entropy.cycleDone() 1074 continue 1075 1076 # re-validate 1077 if not os.path.isfile(dbfile): 1078 do_db_update_transfer = False 1079 1080 elif os.path.isfile(dbfile) and not do_db_update_transfer and \ 1081 (self.dbformat_eapi != 1): 1082 os.remove(dbfile) 1083 1084 if self.dbformat_eapi == 2: 1085 rc = self.do_eapi2_inject_downloaded_dump(dumpfile, 1086 dbfile, cmethod) 1087 1088 if do_db_update_transfer: 1089 self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old) 1090 1091 if self.dbformat_eapi == 2: 1092 # remove the dump 1093 os.remove(dumpfile) 1094 1095 if rc != 0: 1096 # delete all 1097 self.__remove_repository_files(repo, cmethod) 1098 self.syncErrors = True 1099 self.Entropy.cycleDone() 1100 if os.path.isfile(dbfile_old): 1101 os.remove(dbfile_old) 1102 continue 1103 1104 if os.path.isfile(dbfile) and os.access(dbfile,os.W_OK): 1105 try: 1106 self.Entropy.setup_default_file_perms(dbfile) 1107 except OSError: # notification applet 1108 pass 1109 1110 # database is going to be updated 1111 self.dbupdated = True 1112 self.do_standard_items_download(repo) 1113 self.Entropy.update_repository_revision(repo) 1114 if self.Entropy.indexing: 1115 self.do_database_indexing(repo) 1116 def_repoid = self.Entropy.SystemSettings['repositories']['default_repository'] 1117 if repo == def_repoid: 1118 try: 1119 self.run_config_files_updates(repo) 1120 except Exception, e: 1121 self.entropyTools.print_traceback() 1122 mytxt = "%s: %s" % ( 1123 blue(_("Configuration files update error, not critical, continuing")), 1124 darkred(unicode(e)), 1125 ) 1126 self.Entropy.updateProgress(mytxt, importance = 0, 1127 type = "info", header = blue(" # "),) 1128 self.updated_repos.add(repo) 1129 self.Entropy.cycleDone() 1130 1131 # remove garbage 1132 if os.access(dbfile_old, os.R_OK | os.F_OK): 1133 os.remove(dbfile_old) 1134 1135 # keep them closed 1136 self.Entropy.close_all_repositories() 1137 self.Entropy.validate_repositories() 1138 self.Entropy.close_all_repositories() 1139 1140 # clean caches, fetch security 1141 if self.dbupdated: 1142 self.Entropy.purge_cache(client_purge = False) 1143 if self.fetchSecurity: 1144 self.do_update_security_advisories() 1145 # do treeupdates 1146 if isinstance(self.Entropy.clientDbconn, EntropyRepository) and \ 1147 self.entropyTools.is_root(): # only as root due to Portage 1148 for repo in self.reponames: 1149 try: 1150 dbc = self.Entropy.open_repository(repo) 1151 except RepositoryError: 1152 # download failed and repo is not available, skip! 1153 continue 1154 self.Entropy.repository_packages_spm_sync(repo, dbc) 1155 self.Entropy.close_all_repositories() 1156 1157 if self.syncErrors: 1158 self.Entropy.updateProgress( 1159 red(_("Something bad happened. Please have a look.")), 1160 importance = 1, 1161 type = "warning", 1162 header = darkred(" @@ ") 1163 ) 1164 self.syncErrors = True 1165 self.Entropy.resources_remove_lock() 1166 return 128 1167 1168 if not self.noEquoCheck: 1169 self.check_entropy_updates() 1170 1171 return 0

1172

1173 - def run_config_files_updates(self, repo):

1174 1175 # are we root? 1176 if etpConst['uid'] != 0: 1177 self.Entropy.updateProgress( 1178 brown(_("Skipping configuration files update, you are not root.")), 1179 importance = 1, 1180 type = "info", 1181 header = blue(" @@ ") 1182 ) 1183 return 1184 1185 # make.conf 1186 self._config_updates_make_conf(repo) 1187 self._config_updates_make_profile(repo)

1188 1189

1190 - def _config_updates_make_conf(self, repo):

1191 1192 ## WARNING: it doesn't handle multi-line variables, yet. remember this. 1193 url, repo_make_conf = self._construct_paths("make.conf", repo, None) 1194 system_make_conf = etpConst['spm']['global_make_conf'] 1195 make_conf_variables_check = ["CHOST"] 1196 1197 if os.path.isfile(repo_make_conf) and os.access(repo_make_conf,os.R_OK): 1198 1199 if not os.path.isfile(system_make_conf): 1200 self.Entropy.updateProgress( 1201 "%s %s. %s." % (red(system_make_conf),blue(_("does not exist")),blue(_("Overwriting")),), 1202 importance = 1, 1203 type = "info", 1204 header = blue(" @@ ") 1205 ) 1206 if os.path.lexists(system_make_conf): 1207 shutil.move( 1208 system_make_conf, 1209 "%s.backup_%s" % (system_make_conf,self.entropyTools.get_random_number(),) 1210 ) 1211 shutil.copy2(repo_make_conf,system_make_conf) 1212 1213 elif os.access(system_make_conf,os.W_OK): 1214 1215 repo_f = open(repo_make_conf,"r") 1216 sys_f = open(system_make_conf,"r") 1217 repo_make_c = [x.strip() for x in repo_f.readlines()] 1218 sys_make_c = [x.strip() for x in sys_f.readlines()] 1219 repo_f.close() 1220 sys_f.close() 1221 1222 # read repository settings 1223 repo_data = {} 1224 for setting in make_conf_variables_check: 1225 for line in repo_make_c: 1226 if line.startswith(setting+"="): 1227 # there can't be bash vars with a space after its name on declaration 1228 repo_data[setting] = line 1229 # I don't break, because there might be other overlapping settings 1230 1231 differences = {} 1232 # update make.conf data in memory 1233 for setting in repo_data: 1234 for idx in range(len(sys_make_c)): 1235 line = sys_make_c[idx] 1236 if line.startswith(setting+"=") and (line != repo_data[setting]): 1237 # there can't be bash vars with a space after its name on declaration 1238 self.Entropy.updateProgress( 1239 "%s: %s %s. %s." % ( 1240 red(system_make_conf), bold(unicode(setting)), 1241 blue(_("variable differs")), red(_("Updating")), 1242 ), 1243 importance = 1, 1244 type = "info", 1245 header = blue(" @@ ") 1246 ) 1247 differences[setting] = repo_data[setting] 1248 line = repo_data[setting] 1249 sys_make_c[idx] = line 1250 1251 if differences: 1252 1253 self.Entropy.updateProgress( 1254 "%s: %s." % (red(system_make_conf), blue(_("updating critical variables")),), 1255 importance = 1, 1256 type = "info", 1257 header = blue(" @@ ") 1258 ) 1259 # backup user make.conf 1260 shutil.copy2(system_make_conf,"%s.entropy_backup" % (system_make_conf,)) 1261 1262 self.Entropy.updateProgress( 1263 "%s: %s." % ( 1264 red(system_make_conf), darkgreen("writing changes to disk"), 1265 ), 1266 importance = 1, 1267 type = "info", 1268 header = blue(" @@ ") 1269 ) 1270 # write to disk, safely 1271 tmp_make_conf = "%s.entropy_write" % (system_make_conf,) 1272 f = open(tmp_make_conf,"w") 1273 for line in sys_make_c: f.write(line+"\n") 1274 f.flush() 1275 f.close() 1276 shutil.move(tmp_make_conf,system_make_conf) 1277 1278 # update environment 1279 for var in differences: 1280 try: 1281 myval = '='.join(differences[var].strip().split("=")[1:]) 1282 if myval: 1283 if myval[0] in ("'",'"',): myval = myval[1:] 1284 if myval[-1] in ("'",'"',): myval = myval[:-1] 1285 except IndexError: 1286 myval = '' 1287 os.environ[var] = myval

1288

1289 - def _config_updates_make_profile(self, repo):

1290 url, repo_make_profile = self._construct_paths("profile.link", 1291 repo, None) 1292 system_make_profile = etpConst['spm']['global_make_profile'] 1293 if not (os.path.isfile(repo_make_profile) and \ 1294 os.access(repo_make_profile,os.R_OK)): 1295 return 1296 1297 f = open(repo_make_profile,"r") 1298 repo_profile_link_data = f.readline().strip() 1299 f.close() 1300 current_profile_link = '' 1301 if os.path.islink(system_make_profile) and \ 1302 os.access(system_make_profile,os.R_OK): 1303 1304 current_profile_link = os.readlink(system_make_profile) 1305 1306 if (repo_profile_link_data != current_profile_link) and \ 1307 repo_profile_link_data: 1308 1309 self.Entropy.updateProgress( 1310 "%s: %s %s. %s." % ( 1311 red(system_make_profile), blue("link"), 1312 blue(_("differs")), red(_("Updating")), 1313 ), 1314 importance = 1, 1315 type = "info", 1316 header = blue(" @@ ") 1317 ) 1318 merge_sfx = ".entropy_merge" 1319 os.symlink(repo_profile_link_data,system_make_profile+merge_sfx) 1320 if self.entropyTools.is_valid_path(system_make_profile+merge_sfx): 1321 os.rename(system_make_profile+merge_sfx,system_make_profile) 1322 else: 1323 # revert change, link does not exist yet 1324 self.Entropy.updateProgress( 1325 "%s: %s %s. %s." % ( 1326 red(system_make_profile), blue("new link"), 1327 blue(_("does not exist")), red(_("Reverting")), 1328 ), 1329 importance = 1, 1330 type = "info", 1331 header = blue(" @@ ") 1332 ) 1333 os.remove(system_make_profile+merge_sfx)

1334 1335

1336 - def check_entropy_updates(self):

1337 rc = False 1338 if not self.noEquoCheck: 1339 try: 1340 rc, pkg_match = self.Entropy.check_package_update("sys-apps/entropy", deep = True) 1341 except: 1342 pass 1343 if rc: 1344 self.newEquo = True 1345 mytxt = "%s: %s. %s." % ( 1346 bold("Equo/Entropy"), 1347 blue(_("a new release is available")), 1348 darkred(_("Mind to install it before any other package")), 1349 ) 1350 self.Entropy.updateProgress( 1351 mytxt, 1352 importance = 1, 1353 type = "info", 1354 header = bold(" !!! ") 1355 )

1356

1357 - def handle_downloaded_database_unpack(self, repo, cmethod):

1358 1359 file_to_unpack = etpConst['etpdatabasedump'] 1360 if self.dbformat_eapi == 1: 1361 file_to_unpack = etpConst['etpdatabasefile'] 1362 mytxt = "%s %s %s" % (red(_("Unpacking database to")), 1363 darkgreen(file_to_unpack), red("..."),) 1364 self.Entropy.updateProgress( 1365 mytxt, 1366 importance = 0, 1367 type = "info", 1368 header = "\t" 1369 ) 1370 1371 myrc = self.__unpack_downloaded_database(repo, cmethod) 1372 if myrc != 0: 1373 mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")), 1374 red(_("Skipping repository")),) 1375 self.Entropy.updateProgress( 1376 mytxt, 1377 importance = 1, 1378 type = "warning", 1379 header = "\t" 1380 ) 1381 return False 1382 return True

1383 1384

1385 - def handle_database_checksum_download(self, repo, cmethod):

1386 1387 hashfile = etpConst[cmethod[8]] 1388 downitem = 'cklight' 1389 if self.dbformat_eapi == 2: # EAPI = 2 1390 hashfile = etpConst[cmethod[6]] 1391 downitem = 'dbdumplightck' 1392 1393 mytxt = "%s %s %s" % ( 1394 red(_("Downloading checksum")), 1395 darkgreen(hashfile), 1396 red("..."), 1397 ) 1398 # download checksum 1399 self.Entropy.updateProgress( 1400 mytxt, 1401 importance = 0, 1402 type = "info", 1403 header = "\t" 1404 ) 1405 1406 db_down_status = self.download_item(downitem, repo, cmethod, 1407 disallow_redirect = True) 1408 if not db_down_status and (downitem != 'cklight'): 1409 # fallback to old method, deprecated 1410 db_down_status = self.download_item('cklight', repo, cmethod, 1411 disallow_redirect = True) 1412 1413 if not db_down_status: 1414 mytxt = "%s %s !" % ( 1415 red(_("Cannot fetch checksum")), 1416 red(_("Cannot verify database integrity")), 1417 ) 1418 self.Entropy.updateProgress( 1419 mytxt, 1420 importance = 1, 1421 type = "warning", 1422 header = "\t" 1423 ) 1424 return db_down_status

1425

1426 - def load_background_repository_lock_check(self, repo):

1427 # kill previous 1428 self.current_repository_got_locked = False 1429 self.kill_previous_repository_lock_scanner() 1430 self.LockScanner = TimeScheduled(5, self.repository_lock_scanner, repo) 1431 self.LockScanner.start()

1432

1433 - def kill_previous_repository_lock_scanner(self):

1434 if self.LockScanner != None: 1435 self.LockScanner.kill()

1436

1437 - def repository_lock_scanner(self, repo):

1438 locked = self.handle_repository_lock(repo) 1439 if locked: 1440 self.current_repository_got_locked = True

1441

1442 - def repository_lock_scanner_status(self):

1443 # raise an exception if repo got suddenly locked 1444 if self.current_repository_got_locked: 1445 mytxt = _("Current repository got suddenly locked. Download aborted.") 1446 raise RepositoryError('RepositoryError %s' % (mytxt,))

1447

1448 - def handle_database_download(self, repo, cmethod):

1449 1450 def show_repo_locked_message(): 1451 mytxt = "%s: %s." % ( 1452 bold(_("Attention")), 1453 red(_("remote database got suddenly locked")), 1454 ) 1455 self.Entropy.updateProgress( 1456 mytxt, 1457 importance = 1, 1458 type = "warning", 1459 header = "\t" 1460 )

1461 1462 # starting to download 1463 mytxt = "%s ..." % (red(_("Downloading repository database")),) 1464 self.Entropy.updateProgress( 1465 mytxt, 1466 importance = 1, 1467 type = "info", 1468 header = "\t" 1469 ) 1470 1471 down_status = False 1472 if self.dbformat_eapi == 2: 1473 # start a check in background 1474 self.load_background_repository_lock_check(repo) 1475 down_status = self.download_item("dbdumplight", repo, cmethod, 1476 lock_status_func = self.repository_lock_scanner_status, 1477 disallow_redirect = True) 1478 if self.current_repository_got_locked: 1479 self.kill_previous_repository_lock_scanner() 1480 show_repo_locked_message() 1481 return False 1482 if not down_status: # fallback to old db 1483 # start a check in background 1484 self.load_background_repository_lock_check(repo) 1485 self.dbformat_eapi = 1 1486 down_status = self.download_item("dblight", repo, cmethod, 1487 lock_status_func = self.repository_lock_scanner_status, 1488 disallow_redirect = True) 1489 if self.current_repository_got_locked: 1490 self.kill_previous_repository_lock_scanner() 1491 show_repo_locked_message() 1492 return False 1493 1494 if not down_status: 1495 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database does not exist online")),) 1496 self.Entropy.updateProgress( 1497 mytxt, 1498 importance = 1, 1499 type = "warning", 1500 header = "\t" 1501 ) 1502 1503 self.kill_previous_repository_lock_scanner() 1504 return down_status 1505

1506 - def handle_repository_update(self, repo):

1507 # check if database is already updated to the latest revision 1508 update = self.is_repository_updatable(repo) 1509 if not update: 1510 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database is already up to date")),) 1511 self.Entropy.updateProgress( 1512 mytxt, 1513 importance = 1, 1514 type = "info", 1515 header = "\t" 1516 ) 1517 return True 1518 # also check for eapi3 lock 1519 if self.dbformat_eapi == 3: 1520 locked = self.is_repository_eapi3_locked(repo) 1521 if locked: 1522 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database will be ready soon")),) 1523 self.Entropy.updateProgress( 1524 mytxt, 1525 importance = 1, 1526 type = "info", 1527 header = "\t" 1528 ) 1529 return True 1530 return False

1531

1532 - def handle_repository_lock(self, repo):

1533 # get database lock 1534 unlocked = self.is_repository_unlocked(repo) 1535 if not unlocked: 1536 mytxt = "%s: %s. %s." % ( 1537 bold(_("Attention")), 1538 red(_("Repository is being updated")), 1539 red(_("Try again in a few minutes")), 1540 ) 1541 self.Entropy.updateProgress( 1542 mytxt, 1543 importance = 1, 1544 type = "warning", 1545 header = "\t" 1546 ) 1547 return True 1548 return False

1549

1550 - def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

1551 1552 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, 1553 indexing_override = False) 1554 old_dbconn = self.Entropy.open_generic_database(dbfile_old, 1555 xcache = False, indexing_override = False) 1556 upd_rc = 0 1557 try: 1558 upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") 1559 except (self.dbapi2.OperationalError, self.dbapi2.IntegrityError, 1560 self.dbapi2.DatabaseError,): 1561 pass 1562 old_dbconn.closeDB() 1563 dbconn.closeDB() 1564 if upd_rc > 0: 1565 # -1 means no changes, == force used 1566 # 0 means too much hassle 1567 os.rename(dbfile_old, dbfile) 1568 return upd_rc

1569

1570 - def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

1571 1572 # load the dump into database 1573 mytxt = "%s %s, %s %s" % ( 1574 red(_("Injecting downloaded dump")), 1575 darkgreen(etpConst[cmethod[5]]), 1576 red(_("please wait")), 1577 red("..."), 1578 ) 1579 self.Entropy.updateProgress( 1580 mytxt, 1581 importance = 0, 1582 type = "info", 1583 header = "\t" 1584 ) 1585 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 1586 rc = dbconn.doDatabaseImport(dumpfile, dbfile) 1587 dbconn.closeDB() 1588 return rc

1589 1590

1591 - def do_update_security_advisories(self):

1592 # update Security Advisories 1593 try: 1594 securityConn = self.Entropy.Security() 1595 securityConn.fetch_advisories() 1596 except Exception, e: 1597 self.entropyTools.print_traceback(f = self.Entropy.clientLog) 1598 mytxt = "%s: %s" % (red(_("Advisories fetch error")),e,) 1599 self.Entropy.updateProgress( 1600 mytxt, 1601 importance = 1, 1602 type = "warning", 1603 header = darkred(" @@ ") 1604 )

1605

1606 - def do_standard_items_download(self, repo):

1607 1608 g_make_conf = os.path.basename(etpConst['spm']['global_make_conf']) 1609 pkg_unmask = os.path.basename(etpConst['spm']['global_package_unmask']) 1610 pkg_keywords = os.path.basename(etpConst['spm']['global_package_keywords']) 1611 pkg_use = os.path.basename(etpConst['spm']['global_package_use']) 1612 profile_link = etpConst['spm']['global_make_profile_link_name'] 1613 notice_board = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['local_notice_board']) 1614 1615 objects_to_unpack = ("meta_file",) 1616 1617 download_items = [ 1618 ( 1619 "meta_file", 1620 etpConst['etpdatabasemetafilesfile'], 1621 True, 1622 "%s %s %s" % ( 1623 red(_("Downloading repository metafile")), 1624 darkgreen(etpConst['etpdatabasemetafilesfile']), 1625 red("..."), 1626 ) 1627 ), 1628 ( 1629 "ca.cert", 1630 etpConst['etpdatabasecacertfile'], 1631 True, 1632 "%s %s %s" % ( 1633 red(_("Downloading SSL CA certificate")), 1634 darkgreen(etpConst['etpdatabasecacertfile']), 1635 red("..."), 1636 ) 1637 ), 1638 ( 1639 "server.cert", 1640 etpConst['etpdatabaseservercertfile'], 1641 True, 1642 "%s %s %s" % ( 1643 red(_("Downloading SSL Server certificate")), 1644 darkgreen(etpConst['etpdatabaseservercertfile']), 1645 red("..."), 1646 ) 1647 ), 1648 ( 1649 "mask", 1650 etpConst['etpdatabasemaskfile'], 1651 True, 1652 "%s %s %s" % ( 1653 red(_("Downloading package mask")), 1654 darkgreen(etpConst['etpdatabasemaskfile']), 1655 red("..."), 1656 ) 1657 ), 1658 ( 1659 "system_mask", 1660 etpConst['etpdatabasesytemmaskfile'], 1661 True, 1662 "%s %s %s" % ( 1663 red(_("Downloading packages system mask")), 1664 darkgreen(etpConst['etpdatabasesytemmaskfile']), 1665 red("..."), 1666 ) 1667 ), 1668 ( 1669 "keywords", 1670 etpConst['etpdatabasekeywordsfile'], 1671 True, 1672 "%s %s %s" % ( 1673 red(_("Downloading repository provided package keywords")), 1674 darkgreen(etpConst['etpdatabasekeywordsfile']), 1675 red("..."), 1676 ) 1677 ), 1678 ( 1679 "conflicting_tagged", 1680 etpConst['etpdatabaseconflictingtaggedfile'], 1681 True, 1682 "%s %s %s" % ( 1683 red(_("Downloading conflicting tagged packages file")), 1684 darkgreen(etpConst['etpdatabaseconflictingtaggedfile']), 1685 red("..."), 1686 ) 1687 ), 1688 ( 1689 "critical_updates", 1690 etpConst['etpdatabasecriticalfile'], 1691 True, 1692 "%s %s %s" % ( 1693 red(_("Downloading critical updates file")), 1694 darkgreen(etpConst['etpdatabasecriticalfile']), 1695 red("..."), 1696 ) 1697 ), 1698 ( 1699 "lic_whitelist", 1700 etpConst['etpdatabaselicwhitelistfile'], 1701 True, 1702 "%s %s %s" % ( 1703 red(_("Downloading license whitelist")), 1704 darkgreen(etpConst['etpdatabaselicwhitelistfile']), 1705 red("..."), 1706 ) 1707 ), 1708 ( 1709 "rev", 1710 etpConst['etpdatabaserevisionfile'], 1711 False, 1712 "%s %s %s" % ( 1713 red(_("Downloading revision")), 1714 darkgreen(etpConst['etpdatabaserevisionfile']), 1715 red("..."), 1716 ) 1717 ), 1718 ( 1719 "make.conf", 1720 g_make_conf, 1721 True, 1722 "%s %s %s" % ( 1723 red(_("Downloading SPM global configuration")), 1724 darkgreen(g_make_conf), 1725 red("..."), 1726 ) 1727 ), 1728 ( 1729 "package.unmask", 1730 pkg_unmask, 1731 True, 1732 "%s %s %s" % ( 1733 red(_("Downloading SPM package unmasking configuration")), 1734 darkgreen(pkg_unmask), 1735 red("..."), 1736 ) 1737 ), 1738 ( 1739 "package.keywords", 1740 pkg_keywords, 1741 True, 1742 "%s %s %s" % ( 1743 red(_("Downloading SPM package keywording configuration")), 1744 darkgreen(pkg_keywords), 1745 red("..."), 1746 ) 1747 ), 1748 ( 1749 "package.use", 1750 pkg_use, 1751 True, 1752 "%s %s %s" % ( 1753 red(_("Downloading SPM package USE flags configuration")), 1754 darkgreen(pkg_use), 1755 red("..."), 1756 ) 1757 ), 1758 ( 1759 "profile.link", 1760 profile_link, 1761 True, 1762 "%s %s %s" % ( 1763 red(_("Downloading SPM Profile configuration")), 1764 darkgreen(profile_link), 1765 red("..."), 1766 ) 1767 ), 1768 ( 1769 "notice_board", 1770 notice_board, 1771 True, 1772 "%s %s %s" % ( 1773 red(_("Downloading Notice Board")), 1774 darkgreen(notice_board), 1775 red("..."), 1776 ) 1777 ) 1778 ] 1779 1780 def my_show_info(txt): 1781 self.Entropy.updateProgress( 1782 txt, 1783 importance = 0, 1784 type = "info", 1785 header = "\t", 1786 back = True 1787 )

1788 1789 def my_show_down_status(message, mytype): 1790 self.Entropy.updateProgress( 1791 message, 1792 importance = 0, 1793 type = mytype, 1794 header = "\t" 1795 ) 1796 1797 def my_show_file_unpack(fp): 1798 self.Entropy.updateProgress( 1799 "%s: %s" % (darkgreen(_("unpacked meta file")),brown(fp),), 1800 header = blue(u"\t << ") 1801 ) 1802 1803 downloaded_by_unpack = set() 1804 for item, myfile, ignorable, mytxt in download_items: 1805 1806 # if it's been already downloaded, skip 1807 if myfile in downloaded_by_unpack: continue 1808 1809 my_show_info(mytxt) 1810 mystatus = self.download_item(item, repo, disallow_redirect = True) 1811 mytype = 'info' 1812 1813 # download failed, is it critical? 1814 if not mystatus: 1815 if ignorable: 1816 message = "%s: %s." % (blue(myfile),red(_("not available, it's ok"))) 1817 else: 1818 mytype = 'warning' 1819 message = "%s: %s." % (blue(myfile),darkred(_("not available, not much ok!"))) 1820 my_show_down_status(message, mytype) 1821 continue 1822 1823 myurl, mypath = self._construct_paths(item, repo, None) 1824 message = "%s: %s." % (blue(myfile),darkgreen(_("available, w00t!"))) 1825 my_show_down_status(message, mytype) 1826 if item not in objects_to_unpack: continue 1827 if not (os.path.isfile(mypath) and os.access(mypath,os.R_OK)): continue 1828 1829 while 1: 1830 tmpdir = os.path.join(os.path.dirname(mypath),"meta_unpack_%s" % (random.randint(1,10000),)) 1831 if not os.path.lexists(tmpdir): break 1832 os.makedirs(tmpdir,0775) 1833 1834 repo_dir = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 1835 try: 1836 done = self.entropyTools.universal_uncompress(mypath, tmpdir, catch_empty = True) 1837 if not done: continue 1838 myfiles_to_move = set(os.listdir(tmpdir)) 1839 1840 # exclude files not available by default 1841 files_not_found_file = etpConst['etpdatabasemetafilesnotfound'] 1842 if files_not_found_file in myfiles_to_move: 1843 myfiles_to_move.remove(files_not_found_file) 1844 try: 1845 with open(os.path.join(tmpdir,files_not_found_file),"r") as f: 1846 f_nf = [x.strip() for x in f.readlines()] 1847 downloaded_by_unpack |= set(f_nf) 1848 except IOError: 1849 pass 1850 1851 for myfile in sorted(myfiles_to_move): 1852 from_mypath = os.path.join(tmpdir,myfile) 1853 to_mypath = os.path.join(repo_dir,myfile) 1854 try: 1855 os.rename(from_mypath,to_mypath) 1856 downloaded_by_unpack.add(myfile) 1857 my_show_file_unpack(myfile) 1858 except OSError: 1859 continue 1860 1861 finally: 1862 1863 shutil.rmtree(tmpdir,True) 1864 try: os.rmdir(tmpdir) 1865 except OSError: pass 1866 1867 1868 mytxt = "%s: %s" % ( 1869 red(_("Repository revision")), 1870 bold(str(self.Entropy.get_repository_revision(repo))), 1871 ) 1872 self.Entropy.updateProgress( 1873 mytxt, 1874 importance = 1, 1875 type = "info", 1876 header = "\t" 1877 ) 1878 1879 1880

1881 - def do_database_indexing(self, repo):

1882 1883 # renice a bit, to avoid eating resources 1884 old_prio = self.Entropy.set_priority(15) 1885 mytxt = red("%s ...") % (_("Indexing Repository metadata"),) 1886 self.Entropy.updateProgress( 1887 mytxt, 1888 importance = 1, 1889 type = "info", 1890 header = "\t", 1891 back = True 1892 ) 1893 dbconn = self.Entropy.open_repository(repo) 1894 dbconn.createAllIndexes() 1895 dbconn.commitChanges(force = True) 1896 # get list of indexes 1897 repo_indexes = dbconn.listAllIndexes() 1898 if self.Entropy.clientDbconn != None: 1899 try: # client db can be absent 1900 client_indexes = self.Entropy.clientDbconn.listAllIndexes() 1901 if repo_indexes != client_indexes: 1902 self.Entropy.clientDbconn.createAllIndexes() 1903 except: 1904 pass 1905 self.Entropy.set_priority(old_prio)

1906 1907

1908 - def sync(self):

1909 1910 # close them 1911 self.Entropy.close_all_repositories() 1912 1913 # let's dance! 1914 mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),) 1915 self.Entropy.updateProgress( 1916 mytxt, 1917 importance = 2, 1918 type = "info", 1919 header = darkred(" @@ ") 1920 ) 1921 1922 gave_up = self.Entropy.lock_check(self.Entropy.resources_check_lock) 1923 if gave_up: 1924 return 3 1925 1926 locked = self.Entropy.application_lock_check() 1927 if locked: 1928 return 4 1929 1930 # lock 1931 acquired = self.Entropy.resources_create_lock() 1932 if not acquired: 1933 return 4 # app locked during lock acquire 1934 try: 1935 rc = self.run_sync() 1936 finally: 1937 self.Entropy.resources_remove_lock() 1938 if rc: 1939 return rc 1940 1941 # remove lock 1942 self.Entropy.resources_remove_lock() 1943 1944 if (self.notAvailable >= len(self.reponames)): 1945 return 2 1946 elif (self.notAvailable > 0): 1947 return 1 1948 1949 return 0

1950

Source Code for Module entropy.client.interfaces.repository