entropy.client.interfaces.repository

1 # -*- coding: utf-8 -*- 2 ''' 3 # DESCRIPTION: 4 # Entropy Object Oriented Interface 5 6 Copyright (C) 2007-2009 Fabio Erculiani 7 8 This program is free software; you can redistribute it and/or modify 9 it under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 2 of the License, or 11 (at your option) any later version. 12 13 This program is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 GNU General Public License for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with this program; if not, write to the Free Software 20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 ''' 22 23 from __future__ import with_statement 24 import os 25 import time 26 import shutil 27 import subprocess 28 import random 29 from entropy.i18n import _ 30 from entropy.db import dbapi2, EntropyRepository 31 from entropy.misc import TimeScheduled 32 from entropy.const import * 33 from entropy.exceptions import * 34 from entropy.output import * 35

36 -class Repository:

37 38 import entropy.dump as dumpTools 39 import entropy.tools as entropyTools 40 import socket

41 - def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):

42 43 self.LockScanner = None 44 from entropy.client.interfaces import Client 45 if not isinstance(EquoInstance,Client): 46 mytxt = _("A valid Equo instance or subclass is needed") 47 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 48 49 self.supported_download_items = ( 50 "db", "dblight", "rev", "ck", "cklight", "compck", 51 "lock","mask", "system_mask", "dbdump", "conflicting_tagged", 52 "dbdumplight", "dbdumplightck", "dbdumpck", "lic_whitelist", 53 "make.conf", "package.mask", "package.unmask", "package.keywords", 54 "profile.link", "package.use", "server.cert", "ca.cert", 55 "meta_file", "notice_board" 56 ) 57 self.big_socket_timeout = 25 58 self.Entropy = EquoInstance 59 from entropy.cache import EntropyCacher 60 self.Cacher = EntropyCacher() 61 self.dbapi2 = dbapi2 62 self.reponames = reponames 63 self.forceUpdate = forceUpdate 64 self.syncErrors = False 65 self.dbupdated = False 66 self.newEquo = False 67 self.fetchSecurity = fetchSecurity 68 self.noEquoCheck = noEquoCheck 69 self.alreadyUpdated = 0 70 self.notAvailable = 0 71 self.valid_eapis = [1,2,3] 72 self.reset_dbformat_eapi(None) 73 self.current_repository_got_locked = False 74 self.updated_repos = set() 75 76 # check self.Entropy.SystemSettings['repositories']['available'] 77 if not self.Entropy.SystemSettings['repositories']['available']: 78 mytxt = _("No repositories specified in %s") % (etpConst['repositoriesconf'],) 79 raise MissingParameter("MissingParameter: %s" % (mytxt,)) 80 81 if not self.reponames: 82 self.reponames.extend(self.Entropy.SystemSettings['repositories']['available'].keys()[:])

83

84 - def __del__(self):

85 if self.LockScanner != None: 86 self.LockScanner.kill()

87

88 - def get_eapi3_connection(self, repository):

89 # get database url 90 dburl = self.Entropy.SystemSettings['repositories']['available'][repository]['plain_database'] 91 if dburl.startswith("file://"): 92 return None 93 try: 94 dburl = dburl.split("/")[2] 95 except IndexError: 96 return None 97 port = self.Entropy.SystemSettings['repositories']['available'][repository]['service_port'] 98 try: 99 from entropy.services.ugc.interfaces import Client 100 from entropy.client.services.ugc.commands import Client as CommandsClient 101 eapi3_socket = Client(self.Entropy, CommandsClient, output_header = "\t") 102 eapi3_socket.socket_timeout = self.big_socket_timeout 103 eapi3_socket.connect(dburl, port) 104 return eapi3_socket 105 except (ConnectionError,self.socket.error,): 106 return None

107

108 - def check_eapi3_availability(self, repository):

109 conn = self.get_eapi3_connection(repository) 110 if conn == None: return False 111 try: 112 conn.disconnect() 113 except (self.socket.error,AttributeError,): 114 return False 115 return True

116

117 - def reset_dbformat_eapi(self, repository):

118 119 self.dbformat_eapi = 2 120 if repository != None: 121 eapi_avail = self.check_eapi3_availability(repository) 122 if eapi_avail: 123 self.dbformat_eapi = 3 124 125 # FIXME, find a way to do that without needing sqlite3 exec. 126 if not os.access("/usr/bin/sqlite3",os.X_OK) or self.entropyTools.islive(): 127 self.dbformat_eapi = 1 128 else: 129 rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True) 130 if rc != 0: self.dbformat_eapi = 1 131 132 eapi_env = os.getenv("FORCE_EAPI") 133 if eapi_env != None: 134 try: 135 myeapi = int(eapi_env) 136 except (ValueError,TypeError,): 137 return 138 if myeapi in self.valid_eapis: 139 self.dbformat_eapi = myeapi

140 141

142 - def __validate_repository_id(self, repoid):

143 if repoid not in self.reponames: 144 mytxt = _("Repository is not listed in self.reponames") 145 raise InvalidData("InvalidData: %s" % (mytxt,))

146

147 - def __validate_compression_method(self, repo):

148 149 self.__validate_repository_id(repo) 150 151 repo_settings = self.Entropy.SystemSettings['repositories'] 152 dbc_format = repo_settings['available'][repo]['dbcformat'] 153 cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format) 154 if cmethod == None: 155 mytxt = _("Wrong database compression method") 156 raise InvalidDataType("InvalidDataType: %s" % (mytxt,)) 157 158 return cmethod

159

160 - def __ensure_repository_path(self, repo):

161 162 self.__validate_repository_id(repo) 163 164 # create dir if it doesn't exist 165 if not os.path.isdir(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']): 166 os.makedirs(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],0775) 167 168 const_setup_perms(etpConst['etpdatabaseclientdir'],etpConst['entropygid'])

169

170 - def _construct_paths(self, item, repo, cmethod):

171 172 if item not in self.supported_download_items: 173 mytxt = _("Supported items: %s") % (self.supported_download_items,) 174 raise InvalidData("InvalidData: %s" % (mytxt,)) 175 176 items_needing_cmethod = ( 177 "db", "dblight", "cklight", "dbdump", "dbdumpck", 178 "dbdumplight", "dbdumplightck", "compck", 179 ) 180 if (item in items_needing_cmethod) and (cmethod == None): 181 mytxt = _("For %s, cmethod can't be None") % (item,) 182 raise InvalidData("InvalidData: %s" % (mytxt,)) 183 184 repo_db = self.Entropy.SystemSettings['repositories']['available'][repo]['database'] 185 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 186 ec_rev = etpConst['etpdatabaserevisionfile'] 187 ec_hash = etpConst['etpdatabasehashfile'] 188 ec_maskfile = etpConst['etpdatabasemaskfile'] 189 ec_sysmaskfile = etpConst['etpdatabasesytemmaskfile'] 190 ec_confl_taged = etpConst['etpdatabaseconflictingtaggedfile'] 191 make_conf_file = os.path.basename(etpConst['spm']['global_make_conf']) 192 pkg_mask_file = os.path.basename(etpConst['spm']['global_package_mask']) 193 pkg_unmask_file = os.path.basename(etpConst['spm']['global_package_unmask']) 194 pkg_keywords_file = os.path.basename(etpConst['spm']['global_package_keywords']) 195 pkg_use_file = os.path.basename(etpConst['spm']['global_package_use']) 196 sys_profile_lnk = etpConst['spm']['global_make_profile_link_name'] 197 pkg_lic_wl_file = etpConst['etpdatabaselicwhitelistfile'] 198 repo_lock_file = etpConst['etpdatabasedownloadlockfile'] 199 ca_cert_file = etpConst['etpdatabasecacertfile'] 200 server_cert_file = etpConst['etpdatabaseservercertfile'] 201 notice_board_filename = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board']) 202 meta_file = etpConst['etpdatabasemetafilesfile'] 203 md5_ext = etpConst['packagesmd5fileext'] 204 ec_cm2 = None 205 ec_cm3 = None 206 ec_cm4 = None 207 ec_cm5 = None 208 ec_cm6 = None 209 ec_cm7 = None 210 ec_cm8 = None 211 if cmethod != None: 212 ec_cm2 = etpConst[cmethod[2]] 213 ec_cm3 = etpConst[cmethod[3]] 214 ec_cm4 = etpConst[cmethod[4]] 215 ec_cm5 = etpConst[cmethod[5]] 216 ec_cm6 = etpConst[cmethod[6]] 217 ec_cm7 = etpConst[cmethod[7]] 218 ec_cm8 = etpConst[cmethod[8]] 219 220 mymap = { 221 'db': ("%s/%s" % (repo_db,ec_cm2,),"%s/%s" % (repo_dbpath,ec_cm2,),), 222 'dblight': ("%s/%s" % (repo_db,ec_cm7,),"%s/%s" % (repo_dbpath,ec_cm7,),), 223 'dbdump': ("%s/%s" % (repo_db,ec_cm3,),"%s/%s" % (repo_dbpath,ec_cm3,),), 224 'dbdumplight': ("%s/%s" % (repo_db,ec_cm5,),"%s/%s" % (repo_dbpath,ec_cm5,),), 225 'rev': ("%s/%s" % (repo_db,ec_rev,),"%s/%s" % (repo_dbpath,ec_rev,),), 226 'ck': ("%s/%s" % (repo_db,ec_hash,),"%s/%s" % (repo_dbpath,ec_hash,),), 227 'cklight': ("%s/%s" % (repo_db,ec_cm8,),"%s/%s" % (repo_dbpath,ec_cm8,),), 228 'compck': ("%s/%s%s" % (repo_db, ec_cm2, md5_ext,),"%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),), 229 'dbdumpck': ("%s/%s" % (repo_db,ec_cm4,),"%s/%s" % (repo_dbpath,ec_cm4,),), 230 'dbdumplightck': ("%s/%s" % (repo_db,ec_cm6,),"%s/%s" % (repo_dbpath,ec_cm6,),), 231 'mask': ("%s/%s" % (repo_db,ec_maskfile,),"%s/%s" % (repo_dbpath,ec_maskfile,),), 232 'system_mask': ("%s/%s" % (repo_db,ec_sysmaskfile,),"%s/%s" % (repo_dbpath,ec_sysmaskfile,),), 233 'conflicting_tagged': ("%s/%s" % (repo_db,ec_confl_taged,),"%s/%s" % (repo_dbpath,ec_confl_taged,),), 234 'make.conf': ("%s/%s" % (repo_db,make_conf_file,),"%s/%s" % (repo_dbpath,make_conf_file,),), 235 'package.mask': ("%s/%s" % (repo_db,pkg_mask_file,),"%s/%s" % (repo_dbpath,pkg_mask_file,),), 236 'package.unmask': ("%s/%s" % (repo_db,pkg_unmask_file,),"%s/%s" % (repo_dbpath,pkg_unmask_file,),), 237 'package.keywords': ("%s/%s" % (repo_db,pkg_keywords_file,),"%s/%s" % (repo_dbpath,pkg_keywords_file,),), 238 'package.use': ("%s/%s" % (repo_db,pkg_use_file,),"%s/%s" % (repo_dbpath,pkg_use_file,),), 239 'profile.link': ("%s/%s" % (repo_db,sys_profile_lnk,),"%s/%s" % (repo_dbpath,sys_profile_lnk,),), 240 'lic_whitelist': ("%s/%s" % (repo_db,pkg_lic_wl_file,),"%s/%s" % (repo_dbpath,pkg_lic_wl_file,),), 241 'lock': ("%s/%s" % (repo_db,repo_lock_file,),"%s/%s" % (repo_dbpath,repo_lock_file,),), 242 'server.cert': ("%s/%s" % (repo_db,server_cert_file,),"%s/%s" % (repo_dbpath,server_cert_file,),), 243 'ca.cert': ("%s/%s" % (repo_db,ca_cert_file,),"%s/%s" % (repo_dbpath,ca_cert_file,),), 244 'notice_board': (self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board'],"%s/%s" % (repo_dbpath,notice_board_filename,),), 245 'meta_file': ("%s/%s" % (repo_db,meta_file,),"%s/%s" % (repo_dbpath,meta_file,),), 246 } 247 248 return mymap.get(item)

249

250 - def __remove_repository_files(self, repo, cmethod):

251 252 dbfilenameid = cmethod[2] 253 dblightfilenameid = cmethod[7] 254 self.__validate_repository_id(repo) 255 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 256 257 def remove_eapi1(): 258 259 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabasehashfile']): 260 os.remove(repo_dbpath+"/"+etpConst['etpdatabasehashfile']) 261 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 262 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 263 264 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid]): 265 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid]) 266 if os.path.isfile(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 267 etpConst['packagesmd5fileext']): 268 os.remove(repo_dbpath + "/" + etpConst[dblightfilenameid] + \ 269 etpConst['packagesmd5fileext']) 270 271 if os.path.isfile(repo_dbpath+"/"+etpConst[dbfilenameid]): 272 os.remove(repo_dbpath+"/"+etpConst[dbfilenameid]) 273 if os.path.isfile(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 274 etpConst['packagesmd5fileext']): 275 os.remove(repo_dbpath + "/" + etpConst[dbfilenameid] + \ 276 etpConst['packagesmd5fileext'])

277 278 if self.dbformat_eapi == 1: 279 remove_eapi1() 280 elif self.dbformat_eapi in (2,3,): 281 remove_eapi1() 282 if os.path.isfile(repo_dbpath+"/"+cmethod[6]): 283 os.remove(repo_dbpath+"/"+cmethod[6]) 284 if os.path.isfile(repo_dbpath+"/"+etpConst[cmethod[5]]): 285 os.remove(repo_dbpath+"/"+etpConst[cmethod[5]]) 286 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 287 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 288 else: 289 mytxt = _("self.dbformat_eapi must be in (1,2,3,)") 290 raise InvalidData('InvalidData: %s' % (mytxt,))

291

292 - def __unpack_downloaded_database(self, repo, cmethod):

293 294 self.__validate_repository_id(repo) 295 rc = 0 296 path = None 297 sys_set_repos = self.Entropy.SystemSettings['repositories']['available'] 298 repo_data = sys_set_repos[repo] 299 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[7]] 300 if self.dbformat_eapi == 2: 301 myfile = repo_data['dbpath'] + "/" + etpConst[cmethod[5]] 302 303 304 if self.dbformat_eapi in (1, 2,): 305 try: 306 307 myfunc = getattr(self.entropyTools, cmethod[1]) 308 path = myfunc(myfile) 309 # rename path correctly 310 if self.dbformat_eapi == 1: 311 new_path = os.path.join(os.path.dirname(path), 312 etpConst['etpdatabasefile']) 313 os.rename(path, new_path) 314 path = new_path 315 316 except (OSError, EOFError): 317 rc = 1 318 if os.path.isfile(myfile): 319 os.remove(myfile) 320 321 else: 322 mytxt = _("self.dbformat_eapi must be in (1,2)") 323 raise InvalidData('InvalidData: %s' % (mytxt,)) 324 325 if rc == 0: 326 self.Entropy.setup_default_file_perms(path) 327 328 return rc

329

330 - def _verify_file_checksum(self, file_path, md5_checksum_path):

331 ck_f = open(md5_checksum_path, "r") 332 md5hash = ck_f.readline().strip() 333 md5hash = md5hash.split()[0] 334 ck_f.close() 335 return self.entropyTools.compare_md5(file_path, md5hash)

336

337 - def __verify_database_checksum(self, repo, cmethod = None):

338 339 self.__validate_repository_id(repo) 340 sys_settings_repos = self.Entropy.SystemSettings['repositories'] 341 avail_config = sys_settings_repos['available'][repo] 342 343 if self.dbformat_eapi == 1: 344 dbfile = avail_config['dbpath'] + "/" + etpConst[cmethod[7]] 345 md5file = avail_config['dbpath'] + "/" + etpConst[cmethod[8]] 346 347 elif self.dbformat_eapi == 2: 348 dbfile = avail_config['dbpath'] + "/" + etpConst[cmethod[5]] 349 md5file = avail_config['dbpath'] + "/" + etpConst[cmethod[6]] 350 351 else: 352 mytxt = _("self.dbformat_eapi must be in (1,2)") 353 raise InvalidData('InvalidData: %s' % (mytxt,)) 354 355 if not os.access(md5file, os.F_OK | os.R_OK): 356 return -1 357 358 return self._verify_file_checksum(dbfile, md5file)

359 360 # @returns -1 if the file is not available 361 # @returns int>0 if the revision has been retrieved

362 - def get_online_repository_revision(self, repo):

363 364 self.__validate_repository_id(repo) 365 366 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaserevisionfile'] 367 status = self.entropyTools.get_remote_data(url) 368 if (status): 369 status = status[0].strip() 370 try: 371 status = int(status) 372 except ValueError: 373 status = -1 374 return status 375 else: 376 return -1

377

378 - def get_online_eapi3_lock(self, repo):

379 self.__validate_repository_id(repo) 380 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaseeapi3lockfile'] 381 data = self.entropyTools.get_remote_data(url) 382 if not data: 383 return False 384 return True

385

386 - def is_repository_eapi3_locked(self, repo):

387 self.__validate_repository_id(repo) 388 return self.get_online_eapi3_lock(repo)

389

390 - def is_repository_updatable(self, repo):

391 392 self.__validate_repository_id(repo) 393 394 onlinestatus = self.get_online_repository_revision(repo) 395 if (onlinestatus != -1): 396 localstatus = self.Entropy.get_repository_revision(repo) 397 if (localstatus == onlinestatus) and (not self.forceUpdate): 398 return False 399 return True

400

401 - def is_repository_unlocked(self, repo):

402 403 self.__validate_repository_id(repo) 404 405 rc = self.download_item("lock", repo, disallow_redirect = True) 406 if rc: # cannot download database 407 self.syncErrors = True 408 return False 409 return True

410

411 - def clear_repository_cache(self, repo):

412 self.__validate_repository_id(repo) 413 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repo,)) 414 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repo,))

415 416 # this function can be reimplemented

417 - def download_item(self, item, repo, cmethod = None, lock_status_func = None, disallow_redirect = True):

418 419 self.__validate_repository_id(repo) 420 url, filepath = self._construct_paths(item, repo, cmethod) 421 422 # to avoid having permissions issues 423 # it's better to remove the file before, 424 # otherwise new permissions won't be written 425 if os.path.isfile(filepath): 426 os.remove(filepath) 427 filepath_dir = os.path.dirname(filepath) 428 if not os.path.isdir(filepath_dir) and not os.path.lexists(filepath_dir): 429 os.makedirs(filepath_dir,0775) 430 const_setup_perms(filepath_dir, etpConst['entropygid']) 431 432 fetchConn = self.Entropy.urlFetcher( 433 url, 434 filepath, 435 resume = False, 436 abort_check_func = lock_status_func, 437 disallow_redirect = disallow_redirect 438 ) 439 fetchConn.progress = self.Entropy.progress 440 441 rc = fetchConn.download() 442 del fetchConn 443 if rc in ("-1","-2","-3","-4"): 444 return False 445 self.Entropy.setup_default_file_perms(filepath) 446 return True

447

448 - def check_downloaded_database(self, repo, cmethod):

449 450 dbfilename = etpConst[cmethod[7]] 451 if self.dbformat_eapi == 2: 452 dbfilename = etpConst[cmethod[5]] 453 # verify checksum 454 mytxt = "%s %s %s" % ( 455 red(_("Checking downloaded database")), 456 darkgreen(dbfilename), 457 red("..."), 458 ) 459 self.Entropy.updateProgress( 460 mytxt, 461 importance = 0, 462 back = True, 463 type = "info", 464 header = "\t" 465 ) 466 db_status = self.__verify_database_checksum(repo, cmethod) 467 if db_status == -1: 468 mytxt = "%s. %s !" % ( 469 red(_("Cannot open digest")), 470 red(_("Cannot verify database integrity")), 471 ) 472 self.Entropy.updateProgress( 473 mytxt, 474 importance = 1, 475 type = "warning", 476 header = "\t" 477 ) 478 elif db_status: 479 mytxt = "%s: %s" % ( 480 red(_("Downloaded database status")), 481 bold(_("OK")), 482 ) 483 self.Entropy.updateProgress( 484 mytxt, 485 importance = 1, 486 type = "info", 487 header = "\t" 488 ) 489 else: 490 mytxt = "%s: %s" % ( 491 red(_("Downloaded database status")), 492 darkred(_("ERROR")), 493 ) 494 self.Entropy.updateProgress( 495 mytxt, 496 importance = 1, 497 type = "error", 498 header = "\t" 499 ) 500 mytxt = "%s. %s" % ( 501 red(_("An error occured while checking database integrity")), 502 red(_("Giving up")), 503 ) 504 self.Entropy.updateProgress( 505 mytxt, 506 importance = 1, 507 type = "error", 508 header = "\t" 509 ) 510 return 1 511 return 0

512 513

514 - def show_repository_information(self, repo, count_info):

515 516 self.Entropy.updateProgress( 517 bold("%s") % ( self.Entropy.SystemSettings['repositories']['available'][repo]['description'] ), 518 importance = 2, 519 type = "info", 520 count = count_info, 521 header = blue(" # ") 522 ) 523 mytxt = "%s: %s" % (red(_("Database URL")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['database']),) 524 self.Entropy.updateProgress( 525 mytxt, 526 importance = 1, 527 type = "info", 528 header = blue(" # ") 529 ) 530 mytxt = "%s: %s" % (red(_("Database local path")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']),) 531 self.Entropy.updateProgress( 532 mytxt, 533 importance = 0, 534 type = "info", 535 header = blue(" # ") 536 ) 537 mytxt = "%s: %s" % (red(_("Database EAPI")),darkgreen(str(self.dbformat_eapi)),) 538 self.Entropy.updateProgress( 539 mytxt, 540 importance = 0, 541 type = "info", 542 header = blue(" # ") 543 )

544

545 - def get_eapi3_local_database(self, repo):

546 547 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 548 mydbconn = None 549 try: 550 mydbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 551 mydbconn.validateDatabase() 552 except ( 553 self.Entropy.dbapi2.OperationalError, 554 self.Entropy.dbapi2.IntegrityError, 555 SystemDatabaseError, 556 IOError, 557 OSError,): 558 mydbconn = None 559 return mydbconn

560

561 - def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, session):

562 563 product = self.Entropy.SystemSettings['repositories']['product'] 564 data = eapi3_interface.CmdInterface.differential_packages_comparison( 565 session, idpackages, repo, etpConst['currentarch'], product 566 ) 567 if isinstance(data,bool): # then it's probably == False 568 return False,False,False 569 elif not isinstance(data,dict): 570 return None,None,None 571 elif not data.has_key('added') or \ 572 not data.has_key('removed') or \ 573 not data.has_key('checksum'): 574 return None,None,None 575 return data['added'],data['removed'],data['checksum']

576

577 - def get_eapi3_repository_metadata(self, eapi3_interface, repo, session):

578 product = self.Entropy.SystemSettings['repositories']['product'] 579 self.socket.setdefaulttimeout(self.big_socket_timeout) 580 data = eapi3_interface.CmdInterface.get_repository_metadata( 581 session, repo, etpConst['currentarch'], product 582 ) 583 if not isinstance(data,dict): return {} 584 return data

585

586 - def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12):

587 588 def prepare_exit(mysock, session = None): 589 try: 590 if session != None: 591 mysock.close_session(session) 592 mysock.disconnect() 593 except (self.socket.error,): 594 pass

595 596 eapi3_interface = self.get_eapi3_connection(repo) 597 if eapi3_interface == None: return False 598 599 session = eapi3_interface.open_session() 600 601 # AttributeError because mydbconn can be == None 602 try: 603 mydbconn = self.get_eapi3_local_database(repo) 604 myidpackages = mydbconn.listAllIdpackages() 605 except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,AttributeError,): 606 prepare_exit(eapi3_interface, session) 607 return False 608 609 added_ids, removed_ids, checksum = self.get_eapi3_database_differences( 610 eapi3_interface, repo, 611 myidpackages, session 612 ) 613 if (None in (added_ids,removed_ids,checksum)) or \ 614 (not added_ids and not removed_ids and self.forceUpdate): 615 mydbconn.closeDB() 616 prepare_exit(eapi3_interface, session) 617 return False 618 619 elif not checksum: # {added_ids, removed_ids, checksum} == False 620 mydbconn.closeDB() 621 prepare_exit(eapi3_interface, session) 622 mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("remote database suddenly locked")),) 623 self.Entropy.updateProgress( 624 mytxt, 625 importance = 0, 626 type = "info", 627 header = blue(" # "), 628 ) 629 return None 630 631 # is it worth it? 632 if len(added_ids) > threshold: 633 mytxt = "%s: %s (%s: %s/%s)" % ( 634 blue(_("EAPI3 Service")), darkred(_("skipping differential sync")), 635 brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)), 636 ) 637 self.Entropy.updateProgress( 638 mytxt, 639 importance = 0, 640 type = "info", 641 header = blue(" # "), 642 ) 643 mydbconn.closeDB() 644 prepare_exit(eapi3_interface, session) 645 return False 646 647 count = 0 648 added_segments = [] 649 mytmp = set() 650 651 for idpackage in added_ids: 652 count += 1 653 mytmp.add(idpackage) 654 if count % chunk_size == 0: 655 added_segments.append(list(mytmp)) 656 mytmp.clear() 657 if mytmp: added_segments.append(list(mytmp)) 658 del mytmp 659 660 # fetch and store 661 count = 0 662 maxcount = len(added_segments) 663 product = self.Entropy.SystemSettings['repositories']['product'] 664 for segment in added_segments: 665 666 count += 1 667 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) 668 self.Entropy.updateProgress( 669 mytxt, importance = 0, type = "info", 670 header = "\t", back = True, count = (count,maxcount,) 671 ) 672 fetch_count = 0 673 max_fetch_count = 5 674 675 while 1: 676 677 # anti loop protection 678 if fetch_count > max_fetch_count: 679 mydbconn.closeDB() 680 prepare_exit(eapi3_interface, session) 681 return False 682 683 fetch_count += 1 684 pkgdata = eapi3_interface.CmdInterface.get_strict_package_information( 685 session, segment, repo, etpConst['currentarch'], product 686 ) 687 if pkgdata == None: 688 mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), 689 darkred(str(segment)),) 690 self.Entropy.updateProgress( 691 mytxt, importance = 1, type = "warning", 692 header = "\t", count = (count,maxcount,) 693 ) 694 continue 695 elif not pkgdata: # pkgdata == False 696 mytxt = "%s: %s" % ( 697 blue(_("Service status")), 698 darkred("remote database suddenly locked"), 699 ) 700 self.Entropy.updateProgress( 701 mytxt, importance = 1, type = "info", 702 header = "\t", count = (count,maxcount,) 703 ) 704 mydbconn.closeDB() 705 prepare_exit(eapi3_interface, session) 706 return None 707 elif isinstance(pkgdata,tuple): 708 mytxt = "%s: %s, %s. %s" % ( 709 blue(_("Service status")), 710 pkgdata[0], pkgdata[1], 711 darkred("Error processing the command"), 712 ) 713 self.Entropy.updateProgress( 714 mytxt, importance = 1, type = "info", 715 header = "\t", count = (count,maxcount,) 716 ) 717 mydbconn.closeDB() 718 prepare_exit(eapi3_interface, session) 719 return None 720 721 try: 722 for idpackage in pkgdata: 723 self.dumpTools.dumpobj( 724 "%s%s" % (etpCache['eapi3_fetch'],idpackage,), 725 pkgdata[idpackage], 726 ignore_exceptions = False 727 ) 728 except (IOError,EOFError,OSError,), e: 729 mytxt = "%s: %s: %s." % ( 730 blue(_("Local status")), 731 darkred("Error storing data"), 732 e, 733 ) 734 self.Entropy.updateProgress( 735 mytxt, importance = 1, type = "info", 736 header = "\t", count = (count,maxcount,) 737 ) 738 mydbconn.closeDB() 739 prepare_exit(eapi3_interface, session) 740 return None 741 742 break 743 744 del added_segments 745 746 repo_metadata = self.get_eapi3_repository_metadata(eapi3_interface, 747 repo, session) 748 metadata_elements = ("sets", "treeupdates_actions", 749 "treeupdates_digest", "library_idpackages",) 750 for elem in metadata_elements: 751 if elem not in repo_metadata: 752 mydbconn.closeDB() 753 prepare_exit(eapi3_interface, session) 754 mytxt = "%s: %s" % ( 755 blue(_("EAPI3 Service status")), 756 darkred(_("cannot fetch repository metadata")), 757 ) 758 self.Entropy.updateProgress( 759 mytxt, 760 importance = 0, 761 type = "info", 762 header = blue(" # "), 763 ) 764 return None 765 766 # update treeupdates 767 try: 768 mydbconn.setRepositoryUpdatesDigest(repo, 769 repo_metadata['treeupdates_digest']) 770 mydbconn.bumpTreeUpdatesActions( 771 repo_metadata['treeupdates_actions']) 772 except (self.dbapi2.Error,): 773 mydbconn.closeDB() 774 prepare_exit(eapi3_interface, session) 775 mytxt = "%s: %s" % ( 776 blue(_("EAPI3 Service status")), 777 darkred(_("cannot update treeupdates data")), 778 ) 779 self.Entropy.updateProgress( 780 mytxt, 781 importance = 0, 782 type = "info", 783 header = blue(" # "), 784 ) 785 return None 786 787 # update package sets 788 try: 789 mydbconn.clearPackageSets() 790 mydbconn.insertPackageSets(repo_metadata['sets']) 791 except (self.dbapi2.Error,): 792 mydbconn.closeDB() 793 prepare_exit(eapi3_interface, session) 794 mytxt = "%s: %s" % ( 795 blue(_("EAPI3 Service status")), 796 darkred(_("cannot update package sets data")), 797 ) 798 self.Entropy.updateProgress( 799 mytxt, 800 importance = 0, 801 type = "info", 802 header = blue(" # "), 803 ) 804 return None 805 806 # update libraries <=> idpackages map 807 try: 808 mydbconn.clearNeededLibraryIdpackages() 809 mydbconn.setNeededLibraryIdpackages( 810 repo_metadata['library_idpackages']) 811 except (self.dbapi2.Error,): 812 mydbconn.closeDB() 813 prepare_exit(eapi3_interface, session) 814 mytxt = "%s: %s" % ( 815 blue(_("EAPI3 Service status")), 816 darkred(_("cannot update library data")), 817 ) 818 self.Entropy.updateProgress( 819 mytxt, 820 importance = 0, 821 type = "info", 822 header = blue(" # "), 823 ) 824 return None 825 826 # I don't need you anymore 827 # disconnect socket 828 prepare_exit(eapi3_interface, session) 829 830 # now that we have all stored, add 831 count = 0 832 maxcount = len(added_ids) 833 for idpackage in added_ids: 834 count += 1 835 mydata = self.Cacher.pop("%s%s" % (etpCache['eapi3_fetch'],idpackage,)) 836 if mydata == None: 837 mytxt = "%s: %s" % ( 838 blue(_("Fetch error on segment while adding")), 839 darkred(str(segment)), 840 ) 841 self.Entropy.updateProgress( 842 mytxt, importance = 1, type = "warning", 843 header = "\t", count = (count,maxcount,) 844 ) 845 mydbconn.closeDB() 846 return False 847 848 mytxt = "%s %s" % (blue(_("Injecting package")), darkgreen(mydata['atom']),) 849 self.Entropy.updateProgress( 850 mytxt, importance = 0, type = "info", 851 header = "\t", back = True, count = (count,maxcount,) 852 ) 853 try: 854 mydbconn.addPackage( 855 mydata, revision = mydata['revision'], 856 idpackage = idpackage, do_commit = False, 857 formatted_content = True 858 ) 859 except (self.dbapi2.Error,): 860 self.Entropy.updateProgress( 861 blue(_("repository error while adding packages")), 862 importance = 1, type = "warning", 863 header = "\t", count = (count, maxcount,) 864 ) 865 mydbconn.closeDB() 866 return False 867 868 self.Entropy.updateProgress( 869 blue(_("Packages injection complete")), importance = 0, 870 type = "info", header = "\t", 871 ) 872 873 # now remove 874 maxcount = len(removed_ids) 875 count = 0 876 for idpackage in removed_ids: 877 myatom = mydbconn.retrieveAtom(idpackage) 878 count += 1 879 mytxt = "%s: %s" % (blue(_("Removing package")), darkred(str(myatom)),) 880 self.Entropy.updateProgress( 881 mytxt, importance = 0, type = "info", 882 header = "\t", back = True, count = (count,maxcount,) 883 ) 884 try: 885 mydbconn.removePackage(idpackage, do_cleanup = False, 886 do_commit = False) 887 except (self.dbapi2.Error,): 888 self.Entropy.updateProgress( 889 blue(_("repository error while removing packages")), 890 importance = 1, type = "warning", 891 header = "\t", count = (count, maxcount,) 892 ) 893 mydbconn.closeDB() 894 return False 895 896 self.Entropy.updateProgress( 897 blue(_("Packages removal complete")), 898 importance = 0, type = "info", 899 header = "\t", 900 ) 901 902 mydbconn.commitChanges() 903 mydbconn.clearCache() 904 # now verify if both checksums match 905 result = False 906 mychecksum = mydbconn.database_checksum(do_order = True, 907 strict = False, strings = True) 908 if checksum == mychecksum: 909 result = True 910 else: 911 self.Entropy.updateProgress( 912 blue(_("Database checksum doesn't match remote.")), 913 importance = 0, type = "info", header = "\t", 914 ) 915 mytxt = "%s: %s" % (_('local'), mychecksum,) 916 self.Entropy.updateProgress( 917 mytxt, importance = 0, 918 type = "info", header = "\t", 919 ) 920 mytxt = "%s: %s" % (_('remote'), checksum,) 921 self.Entropy.updateProgress( 922 mytxt, importance = 0, 923 type = "info", header = "\t", 924 ) 925 926 mydbconn.closeDB() 927 return result 928

929 - def run_sync(self):

930 931 self.dbupdated = False 932 repocount = 0 933 repolength = len(self.reponames) 934 for repo in self.reponames: 935 936 repocount += 1 937 self.reset_dbformat_eapi(repo) 938 self.show_repository_information(repo, (repocount,repolength)) 939 940 if not self.forceUpdate: 941 updated = self.handle_repository_update(repo) 942 if updated: 943 self.Entropy.cycleDone() 944 self.alreadyUpdated += 1 945 continue 946 947 locked = self.handle_repository_lock(repo) 948 if locked: 949 self.notAvailable += 1 950 self.Entropy.cycleDone() 951 continue 952 953 # clear database interface cache belonging to this repository 954 self.clear_repository_cache(repo) 955 self.__ensure_repository_path(repo) 956 957 # dealing with EAPI 958 # setting some vars 959 do_skip = False 960 skip_this_repo = False 961 db_checksum_down_status = False 962 do_db_update_transfer = False 963 rc = 0 964 # some variables 965 dumpfile = os.path.join( 966 self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'], 967 etpConst['etpdatabasedumplight']) 968 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 969 dbfile_old = dbfile+".sync" 970 cmethod = self.__validate_compression_method(repo) 971 972 while 1: 973 974 if do_skip: 975 break 976 977 if self.dbformat_eapi < 3: 978 979 down_status = self.handle_database_download(repo, cmethod) 980 if not down_status: 981 self.Entropy.cycleDone() 982 self.notAvailable += 1 983 do_skip = True 984 skip_this_repo = True 985 continue 986 db_checksum_down_status = self.handle_database_checksum_download( 987 repo, cmethod) 988 break 989 990 elif self.dbformat_eapi == 3 and not \ 991 (os.path.isfile(dbfile) and os.access(dbfile,os.W_OK)): 992 993 do_db_update_transfer = None 994 self.dbformat_eapi -= 1 995 continue 996 997 elif self.dbformat_eapi == 3: 998 999 status = False 1000 try: 1001 status = self.handle_eapi3_database_sync(repo) 1002 except self.socket.error, e: 1003 mytxt = "%s: %s" % ( 1004 blue(_("EAPI3 Service error")), 1005 darkred(unicode(e)), 1006 ) 1007 self.Entropy.updateProgress( 1008 mytxt, 1009 importance = 0, 1010 type = "info", 1011 header = blue(" # "), 1012 ) 1013 except: 1014 # avoid broken entries, deal with every exception 1015 self.__remove_repository_files(repo, cmethod) 1016 raise 1017 1018 if status == None: # remote db not available anymore ? 1019 time.sleep(5) 1020 locked = self.handle_repository_lock(repo) 1021 if locked: 1022 self.Entropy.cycleDone() 1023 self.notAvailable += 1 1024 do_skip = True 1025 skip_this_repo = True 1026 else: # ah, well... dunno then... 1027 do_db_update_transfer = None 1028 self.dbformat_eapi -= 1 1029 continue 1030 elif not status: # (status == False) 1031 # set to none and completely skip database alignment 1032 do_db_update_transfer = None 1033 self.dbformat_eapi -= 1 1034 continue 1035 1036 break 1037 1038 if skip_this_repo: 1039 continue 1040 1041 if self.dbformat_eapi in (1, 2,): 1042 1043 # new policy, always deny repository if 1044 # its database checksum cannot be fetched 1045 if not db_checksum_down_status: 1046 # delete all 1047 self.__remove_repository_files(repo, cmethod) 1048 self.syncErrors = True 1049 self.Entropy.cycleDone() 1050 continue 1051 1052 rc = self.check_downloaded_database(repo, cmethod) 1053 if rc != 0: 1054 # delete all 1055 self.__remove_repository_files(repo, cmethod) 1056 self.syncErrors = True 1057 self.Entropy.cycleDone() 1058 continue 1059 1060 # if do_db_update_transfer == False and not None 1061 if (do_db_update_transfer is not None) and not \ 1062 do_db_update_transfer: 1063 1064 if os.access(dbfile, os.R_OK | os.R_OK | os.F_OK): 1065 try: 1066 os.rename(dbfile, dbfile_old) 1067 do_db_update_transfer = True 1068 except OSError: 1069 do_db_update_transfer = False 1070 1071 unpack_status = self.handle_downloaded_database_unpack(repo, 1072 cmethod) 1073 1074 if not unpack_status: 1075 # delete all 1076 self.__remove_repository_files(repo, cmethod) 1077 self.syncErrors = True 1078 self.Entropy.cycleDone() 1079 continue 1080 1081 # re-validate 1082 if not os.path.isfile(dbfile): 1083 do_db_update_transfer = False 1084 1085 elif os.path.isfile(dbfile) and not do_db_update_transfer and \ 1086 (self.dbformat_eapi != 1): 1087 os.remove(dbfile) 1088 1089 if self.dbformat_eapi == 2: 1090 rc = self.do_eapi2_inject_downloaded_dump(dumpfile, 1091 dbfile, cmethod) 1092 1093 if do_db_update_transfer: 1094 self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old) 1095 1096 if self.dbformat_eapi == 2: 1097 # remove the dump 1098 os.remove(dumpfile) 1099 1100 if rc != 0: 1101 # delete all 1102 self.__remove_repository_files(repo, cmethod) 1103 self.syncErrors = True 1104 self.Entropy.cycleDone() 1105 if os.path.isfile(dbfile_old): 1106 os.remove(dbfile_old) 1107 continue 1108 1109 if os.path.isfile(dbfile) and os.access(dbfile,os.W_OK): 1110 try: 1111 self.Entropy.setup_default_file_perms(dbfile) 1112 except OSError: # notification applet 1113 pass 1114 1115 # database is going to be updated 1116 self.dbupdated = True 1117 self.do_standard_items_download(repo) 1118 self.Entropy.update_repository_revision(repo) 1119 if self.Entropy.indexing: 1120 self.do_database_indexing(repo) 1121 def_repoid = self.Entropy.SystemSettings['repositories']['default_repository'] 1122 if repo == def_repoid: 1123 try: 1124 self.run_config_files_updates(repo) 1125 except Exception, e: 1126 self.entropyTools.print_traceback() 1127 mytxt = "%s: %s" % ( 1128 blue(_("Configuration files update error, not critical, continuing")), 1129 darkred(unicode(e)), 1130 ) 1131 self.Entropy.updateProgress(mytxt, importance = 0, 1132 type = "info", header = blue(" # "),) 1133 self.updated_repos.add(repo) 1134 self.Entropy.cycleDone() 1135 1136 # remove garbage 1137 if os.path.isfile(dbfile_old): 1138 os.remove(dbfile_old) 1139 1140 # keep them closed 1141 self.Entropy.close_all_repositories() 1142 self.Entropy.validate_repositories() 1143 self.Entropy.close_all_repositories() 1144 1145 # clean caches, fetch security 1146 if self.dbupdated: 1147 self.Entropy.purge_cache(client_purge = False) 1148 if self.fetchSecurity: 1149 self.do_update_security_advisories() 1150 # do treeupdates 1151 if isinstance(self.Entropy.clientDbconn, EntropyRepository) and \ 1152 self.entropyTools.is_root(): # only as root due to Portage 1153 for repo in self.reponames: 1154 dbc = self.Entropy.open_repository(repo) 1155 self.Entropy.repository_packages_spm_sync(repo, dbc) 1156 self.Entropy.close_all_repositories() 1157 1158 if self.syncErrors: 1159 self.Entropy.updateProgress( 1160 red(_("Something bad happened. Please have a look.")), 1161 importance = 1, 1162 type = "warning", 1163 header = darkred(" @@ ") 1164 ) 1165 self.syncErrors = True 1166 self.Entropy.resources_remove_lock() 1167 return 128 1168 1169 if not self.noEquoCheck: 1170 self.check_entropy_updates() 1171 1172 return 0

1173

1174 - def run_config_files_updates(self, repo):

1175 1176 # are we root? 1177 if etpConst['uid'] != 0: 1178 self.Entropy.updateProgress( 1179 brown(_("Skipping configuration files update, you are not root.")), 1180 importance = 1, 1181 type = "info", 1182 header = blue(" @@ ") 1183 ) 1184 return 1185 1186 # make.conf 1187 self._config_updates_make_conf(repo) 1188 self._config_updates_make_profile(repo)

1189 1190

1191 - def _config_updates_make_conf(self, repo):

1192 1193 ## WARNING: it doesn't handle multi-line variables, yet. remember this. 1194 url, repo_make_conf = self._construct_paths("make.conf", repo, None) 1195 system_make_conf = etpConst['spm']['global_make_conf'] 1196 make_conf_variables_check = ["CHOST"] 1197 1198 if os.path.isfile(repo_make_conf) and os.access(repo_make_conf,os.R_OK): 1199 1200 if not os.path.isfile(system_make_conf): 1201 self.Entropy.updateProgress( 1202 "%s %s. %s." % (red(system_make_conf),blue(_("does not exist")),blue(_("Overwriting")),), 1203 importance = 1, 1204 type = "info", 1205 header = blue(" @@ ") 1206 ) 1207 if os.path.lexists(system_make_conf): 1208 shutil.move( 1209 system_make_conf, 1210 "%s.backup_%s" % (system_make_conf,self.entropyTools.get_random_number(),) 1211 ) 1212 shutil.copy2(repo_make_conf,system_make_conf) 1213 1214 elif os.access(system_make_conf,os.W_OK): 1215 1216 repo_f = open(repo_make_conf,"r") 1217 sys_f = open(system_make_conf,"r") 1218 repo_make_c = [x.strip() for x in repo_f.readlines()] 1219 sys_make_c = [x.strip() for x in sys_f.readlines()] 1220 repo_f.close() 1221 sys_f.close() 1222 1223 # read repository settings 1224 repo_data = {} 1225 for setting in make_conf_variables_check: 1226 for line in repo_make_c: 1227 if line.startswith(setting+"="): 1228 # there can't be bash vars with a space after its name on declaration 1229 repo_data[setting] = line 1230 # I don't break, because there might be other overlapping settings 1231 1232 differences = {} 1233 # update make.conf data in memory 1234 for setting in repo_data: 1235 for idx in range(len(sys_make_c)): 1236 line = sys_make_c[idx] 1237 if line.startswith(setting+"=") and (line != repo_data[setting]): 1238 # there can't be bash vars with a space after its name on declaration 1239 self.Entropy.updateProgress( 1240 "%s: %s %s. %s." % ( 1241 red(system_make_conf), bold(unicode(setting)), 1242 blue(_("variable differs")), red(_("Updating")), 1243 ), 1244 importance = 1, 1245 type = "info", 1246 header = blue(" @@ ") 1247 ) 1248 differences[setting] = repo_data[setting] 1249 line = repo_data[setting] 1250 sys_make_c[idx] = line 1251 1252 if differences: 1253 1254 self.Entropy.updateProgress( 1255 "%s: %s." % (red(system_make_conf), blue(_("updating critical variables")),), 1256 importance = 1, 1257 type = "info", 1258 header = blue(" @@ ") 1259 ) 1260 # backup user make.conf 1261 shutil.copy2(system_make_conf,"%s.entropy_backup" % (system_make_conf,)) 1262 1263 self.Entropy.updateProgress( 1264 "%s: %s." % ( 1265 red(system_make_conf), darkgreen("writing changes to disk"), 1266 ), 1267 importance = 1, 1268 type = "info", 1269 header = blue(" @@ ") 1270 ) 1271 # write to disk, safely 1272 tmp_make_conf = "%s.entropy_write" % (system_make_conf,) 1273 f = open(tmp_make_conf,"w") 1274 for line in sys_make_c: f.write(line+"\n") 1275 f.flush() 1276 f.close() 1277 shutil.move(tmp_make_conf,system_make_conf) 1278 1279 # update environment 1280 for var in differences: 1281 try: 1282 myval = '='.join(differences[var].strip().split("=")[1:]) 1283 if myval: 1284 if myval[0] in ("'",'"',): myval = myval[1:] 1285 if myval[-1] in ("'",'"',): myval = myval[:-1] 1286 except IndexError: 1287 myval = '' 1288 os.environ[var] = myval

1289

1290 - def _config_updates_make_profile(self, repo):

1291 url, repo_make_profile = self._construct_paths("profile.link", 1292 repo, None) 1293 system_make_profile = etpConst['spm']['global_make_profile'] 1294 if not (os.path.isfile(repo_make_profile) and \ 1295 os.access(repo_make_profile,os.R_OK)): 1296 return 1297 1298 f = open(repo_make_profile,"r") 1299 repo_profile_link_data = f.readline().strip() 1300 f.close() 1301 current_profile_link = '' 1302 if os.path.islink(system_make_profile) and \ 1303 os.access(system_make_profile,os.R_OK): 1304 1305 current_profile_link = os.readlink(system_make_profile) 1306 1307 if (repo_profile_link_data != current_profile_link) and \ 1308 repo_profile_link_data: 1309 1310 self.Entropy.updateProgress( 1311 "%s: %s %s. %s." % ( 1312 red(system_make_profile), blue("link"), 1313 blue(_("differs")), red(_("Updating")), 1314 ), 1315 importance = 1, 1316 type = "info", 1317 header = blue(" @@ ") 1318 ) 1319 merge_sfx = ".entropy_merge" 1320 os.symlink(repo_profile_link_data,system_make_profile+merge_sfx) 1321 if self.entropyTools.is_valid_path(system_make_profile+merge_sfx): 1322 os.rename(system_make_profile+merge_sfx,system_make_profile) 1323 else: 1324 # revert change, link does not exist yet 1325 self.Entropy.updateProgress( 1326 "%s: %s %s. %s." % ( 1327 red(system_make_profile), blue("new link"), 1328 blue(_("does not exist")), red(_("Reverting")), 1329 ), 1330 importance = 1, 1331 type = "info", 1332 header = blue(" @@ ") 1333 ) 1334 os.remove(system_make_profile+merge_sfx)

1335 1336

1337 - def check_entropy_updates(self):

1338 rc = False 1339 if not self.noEquoCheck: 1340 try: 1341 rc, pkg_match = self.Entropy.check_package_update("sys-apps/entropy", deep = True) 1342 except: 1343 pass 1344 if rc: 1345 self.newEquo = True 1346 mytxt = "%s: %s. %s." % ( 1347 bold("Equo/Entropy"), 1348 blue(_("a new release is available")), 1349 darkred(_("Mind to install it before any other package")), 1350 ) 1351 self.Entropy.updateProgress( 1352 mytxt, 1353 importance = 1, 1354 type = "info", 1355 header = bold(" !!! ") 1356 )

1357

1358 - def handle_downloaded_database_unpack(self, repo, cmethod):

1359 1360 file_to_unpack = etpConst['etpdatabasedump'] 1361 if self.dbformat_eapi == 1: 1362 file_to_unpack = etpConst['etpdatabasefile'] 1363 mytxt = "%s %s %s" % (red(_("Unpacking database to")), 1364 darkgreen(file_to_unpack), red("..."),) 1365 self.Entropy.updateProgress( 1366 mytxt, 1367 importance = 0, 1368 type = "info", 1369 header = "\t" 1370 ) 1371 1372 myrc = self.__unpack_downloaded_database(repo, cmethod) 1373 if myrc != 0: 1374 mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")), 1375 red(_("Skipping repository")),) 1376 self.Entropy.updateProgress( 1377 mytxt, 1378 importance = 1, 1379 type = "warning", 1380 header = "\t" 1381 ) 1382 return False 1383 return True

1384 1385

1386 - def handle_database_checksum_download(self, repo, cmethod):

1387 1388 hashfile = etpConst[cmethod[8]] 1389 downitem = 'cklight' 1390 if self.dbformat_eapi == 2: # EAPI = 2 1391 hashfile = etpConst[cmethod[6]] 1392 downitem = 'dbdumplightck' 1393 1394 mytxt = "%s %s %s" % ( 1395 red(_("Downloading checksum")), 1396 darkgreen(hashfile), 1397 red("..."), 1398 ) 1399 # download checksum 1400 self.Entropy.updateProgress( 1401 mytxt, 1402 importance = 0, 1403 type = "info", 1404 header = "\t" 1405 ) 1406 1407 db_down_status = self.download_item(downitem, repo, cmethod, 1408 disallow_redirect = True) 1409 if not db_down_status and (downitem != 'cklight'): 1410 # fallback to old method, deprecated 1411 db_down_status = self.download_item('cklight', repo, cmethod, 1412 disallow_redirect = True) 1413 1414 if not db_down_status: 1415 mytxt = "%s %s !" % ( 1416 red(_("Cannot fetch checksum")), 1417 red(_("Cannot verify database integrity")), 1418 ) 1419 self.Entropy.updateProgress( 1420 mytxt, 1421 importance = 1, 1422 type = "warning", 1423 header = "\t" 1424 ) 1425 return db_down_status

1426

1427 - def load_background_repository_lock_check(self, repo):

1428 # kill previous 1429 self.current_repository_got_locked = False 1430 self.kill_previous_repository_lock_scanner() 1431 self.LockScanner = TimeScheduled(5, self.repository_lock_scanner, repo) 1432 self.LockScanner.start()

1433

1434 - def kill_previous_repository_lock_scanner(self):

1435 if self.LockScanner != None: 1436 self.LockScanner.kill()

1437

1438 - def repository_lock_scanner(self, repo):

1439 locked = self.handle_repository_lock(repo) 1440 if locked: 1441 self.current_repository_got_locked = True

1442

1443 - def repository_lock_scanner_status(self):

1444 # raise an exception if repo got suddenly locked 1445 if self.current_repository_got_locked: 1446 mytxt = _("Current repository got suddenly locked. Download aborted.") 1447 raise RepositoryError('RepositoryError %s' % (mytxt,))

1448

1449 - def handle_database_download(self, repo, cmethod):

1450 1451 def show_repo_locked_message(): 1452 mytxt = "%s: %s." % ( 1453 bold(_("Attention")), 1454 red(_("remote database got suddenly locked")), 1455 ) 1456 self.Entropy.updateProgress( 1457 mytxt, 1458 importance = 1, 1459 type = "warning", 1460 header = "\t" 1461 )

1462 1463 # starting to download 1464 mytxt = "%s ..." % (red(_("Downloading repository database")),) 1465 self.Entropy.updateProgress( 1466 mytxt, 1467 importance = 1, 1468 type = "info", 1469 header = "\t" 1470 ) 1471 1472 down_status = False 1473 if self.dbformat_eapi == 2: 1474 # start a check in background 1475 self.load_background_repository_lock_check(repo) 1476 down_status = self.download_item("dbdumplight", repo, cmethod, 1477 lock_status_func = self.repository_lock_scanner_status, 1478 disallow_redirect = True) 1479 if self.current_repository_got_locked: 1480 self.kill_previous_repository_lock_scanner() 1481 show_repo_locked_message() 1482 return False 1483 if not down_status: # fallback to old db 1484 # start a check in background 1485 self.load_background_repository_lock_check(repo) 1486 self.dbformat_eapi = 1 1487 down_status = self.download_item("dblight", repo, cmethod, 1488 lock_status_func = self.repository_lock_scanner_status, 1489 disallow_redirect = True) 1490 if self.current_repository_got_locked: 1491 self.kill_previous_repository_lock_scanner() 1492 show_repo_locked_message() 1493 return False 1494 1495 if not down_status: 1496 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database does not exist online")),) 1497 self.Entropy.updateProgress( 1498 mytxt, 1499 importance = 1, 1500 type = "warning", 1501 header = "\t" 1502 ) 1503 1504 self.kill_previous_repository_lock_scanner() 1505 return down_status 1506

1507 - def handle_repository_update(self, repo):

1508 # check if database is already updated to the latest revision 1509 update = self.is_repository_updatable(repo) 1510 if not update: 1511 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database is already up to date")),) 1512 self.Entropy.updateProgress( 1513 mytxt, 1514 importance = 1, 1515 type = "info", 1516 header = "\t" 1517 ) 1518 return True 1519 # also check for eapi3 lock 1520 if self.dbformat_eapi == 3: 1521 locked = self.is_repository_eapi3_locked(repo) 1522 if locked: 1523 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database will be ready soon")),) 1524 self.Entropy.updateProgress( 1525 mytxt, 1526 importance = 1, 1527 type = "info", 1528 header = "\t" 1529 ) 1530 return True 1531 return False

1532

1533 - def handle_repository_lock(self, repo):

1534 # get database lock 1535 unlocked = self.is_repository_unlocked(repo) 1536 if not unlocked: 1537 mytxt = "%s: %s. %s." % ( 1538 bold(_("Attention")), 1539 red(_("Repository is being updated")), 1540 red(_("Try again in a few minutes")), 1541 ) 1542 self.Entropy.updateProgress( 1543 mytxt, 1544 importance = 1, 1545 type = "warning", 1546 header = "\t" 1547 ) 1548 return True 1549 return False

1550

1551 - def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

1552 1553 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, 1554 indexing_override = False) 1555 old_dbconn = self.Entropy.open_generic_database(dbfile_old, 1556 xcache = False, indexing_override = False) 1557 upd_rc = 0 1558 try: 1559 upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") 1560 except (self.dbapi2.OperationalError, self.dbapi2.IntegrityError, 1561 self.dbapi2.DatabaseError,): 1562 pass 1563 old_dbconn.closeDB() 1564 dbconn.closeDB() 1565 if upd_rc > 0: 1566 # -1 means no changes, == force used 1567 # 0 means too much hassle 1568 os.rename(dbfile_old, dbfile) 1569 return upd_rc

1570

1571 - def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

1572 1573 # load the dump into database 1574 mytxt = "%s %s, %s %s" % ( 1575 red(_("Injecting downloaded dump")), 1576 darkgreen(etpConst[cmethod[5]]), 1577 red(_("please wait")), 1578 red("..."), 1579 ) 1580 self.Entropy.updateProgress( 1581 mytxt, 1582 importance = 0, 1583 type = "info", 1584 header = "\t" 1585 ) 1586 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 1587 rc = dbconn.doDatabaseImport(dumpfile, dbfile) 1588 dbconn.closeDB() 1589 return rc

1590 1591

1592 - def do_update_security_advisories(self):

1593 # update Security Advisories 1594 try: 1595 securityConn = self.Entropy.Security() 1596 securityConn.fetch_advisories() 1597 except Exception, e: 1598 self.entropyTools.print_traceback(f = self.Entropy.clientLog) 1599 mytxt = "%s: %s" % (red(_("Advisories fetch error")),e,) 1600 self.Entropy.updateProgress( 1601 mytxt, 1602 importance = 1, 1603 type = "warning", 1604 header = darkred(" @@ ") 1605 )

1606

1607 - def do_standard_items_download(self, repo):

1608 1609 g_make_conf = os.path.basename(etpConst['spm']['global_make_conf']) 1610 pkg_unmask = os.path.basename(etpConst['spm']['global_package_unmask']) 1611 pkg_keywords = os.path.basename(etpConst['spm']['global_package_keywords']) 1612 pkg_use = os.path.basename(etpConst['spm']['global_package_use']) 1613 profile_link = etpConst['spm']['global_make_profile_link_name'] 1614 notice_board = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['local_notice_board']) 1615 1616 objects_to_unpack = ("meta_file",) 1617 1618 download_items = [ 1619 ( 1620 "meta_file", 1621 etpConst['etpdatabasemetafilesfile'], 1622 True, 1623 "%s %s %s" % ( 1624 red(_("Downloading repository metafile")), 1625 darkgreen(etpConst['etpdatabasemetafilesfile']), 1626 red("..."), 1627 ) 1628 ), 1629 ( 1630 "ca.cert", 1631 etpConst['etpdatabasecacertfile'], 1632 True, 1633 "%s %s %s" % ( 1634 red(_("Downloading SSL CA certificate")), 1635 darkgreen(etpConst['etpdatabasecacertfile']), 1636 red("..."), 1637 ) 1638 ), 1639 ( 1640 "server.cert", 1641 etpConst['etpdatabaseservercertfile'], 1642 True, 1643 "%s %s %s" % ( 1644 red(_("Downloading SSL Server certificate")), 1645 darkgreen(etpConst['etpdatabaseservercertfile']), 1646 red("..."), 1647 ) 1648 ), 1649 ( 1650 "mask", 1651 etpConst['etpdatabasemaskfile'], 1652 True, 1653 "%s %s %s" % ( 1654 red(_("Downloading package mask")), 1655 darkgreen(etpConst['etpdatabasemaskfile']), 1656 red("..."), 1657 ) 1658 ), 1659 ( 1660 "system_mask", 1661 etpConst['etpdatabasesytemmaskfile'], 1662 True, 1663 "%s %s %s" % ( 1664 red(_("Downloading packages system mask")), 1665 darkgreen(etpConst['etpdatabasesytemmaskfile']), 1666 red("..."), 1667 ) 1668 ), 1669 ( 1670 "conflicting_tagged", 1671 etpConst['etpdatabaseconflictingtaggedfile'], 1672 True, 1673 "%s %s %s" % ( 1674 red(_("Downloading conflicting tagged packages file")), 1675 darkgreen(etpConst['etpdatabaseconflictingtaggedfile']), 1676 red("..."), 1677 ) 1678 ), 1679 ( 1680 "lic_whitelist", 1681 etpConst['etpdatabaselicwhitelistfile'], 1682 True, 1683 "%s %s %s" % ( 1684 red(_("Downloading license whitelist")), 1685 darkgreen(etpConst['etpdatabaselicwhitelistfile']), 1686 red("..."), 1687 ) 1688 ), 1689 ( 1690 "rev", 1691 etpConst['etpdatabaserevisionfile'], 1692 False, 1693 "%s %s %s" % ( 1694 red(_("Downloading revision")), 1695 darkgreen(etpConst['etpdatabaserevisionfile']), 1696 red("..."), 1697 ) 1698 ), 1699 ( 1700 "make.conf", 1701 g_make_conf, 1702 True, 1703 "%s %s %s" % ( 1704 red(_("Downloading SPM global configuration")), 1705 darkgreen(g_make_conf), 1706 red("..."), 1707 ) 1708 ), 1709 ( 1710 "package.unmask", 1711 pkg_unmask, 1712 True, 1713 "%s %s %s" % ( 1714 red(_("Downloading SPM package unmasking configuration")), 1715 darkgreen(pkg_unmask), 1716 red("..."), 1717 ) 1718 ), 1719 ( 1720 "package.keywords", 1721 pkg_keywords, 1722 True, 1723 "%s %s %s" % ( 1724 red(_("Downloading SPM package keywording configuration")), 1725 darkgreen(pkg_keywords), 1726 red("..."), 1727 ) 1728 ), 1729 ( 1730 "package.use", 1731 pkg_use, 1732 True, 1733 "%s %s %s" % ( 1734 red(_("Downloading SPM package USE flags configuration")), 1735 darkgreen(pkg_use), 1736 red("..."), 1737 ) 1738 ), 1739 ( 1740 "profile.link", 1741 profile_link, 1742 True, 1743 "%s %s %s" % ( 1744 red(_("Downloading SPM Profile configuration")), 1745 darkgreen(profile_link), 1746 red("..."), 1747 ) 1748 ), 1749 ( 1750 "notice_board", 1751 notice_board, 1752 True, 1753 "%s %s %s" % ( 1754 red(_("Downloading Notice Board")), 1755 darkgreen(notice_board), 1756 red("..."), 1757 ) 1758 ) 1759 ] 1760 1761 def my_show_info(txt): 1762 self.Entropy.updateProgress( 1763 txt, 1764 importance = 0, 1765 type = "info", 1766 header = "\t", 1767 back = True 1768 )

1769 1770 def my_show_down_status(message, mytype): 1771 self.Entropy.updateProgress( 1772 message, 1773 importance = 0, 1774 type = mytype, 1775 header = "\t" 1776 ) 1777 1778 def my_show_file_unpack(fp): 1779 self.Entropy.updateProgress( 1780 "%s: %s" % (darkgreen(_("unpacked meta file")),brown(fp),), 1781 header = blue(u"\t << ") 1782 ) 1783 1784 downloaded_by_unpack = set() 1785 for item, myfile, ignorable, mytxt in download_items: 1786 1787 # if it's been already downloaded, skip 1788 if myfile in downloaded_by_unpack: continue 1789 1790 my_show_info(mytxt) 1791 mystatus = self.download_item(item, repo, disallow_redirect = True) 1792 mytype = 'info' 1793 1794 # download failed, is it critical? 1795 if not mystatus: 1796 if ignorable: 1797 message = "%s: %s." % (blue(myfile),red(_("not available, it's ok"))) 1798 else: 1799 mytype = 'warning' 1800 message = "%s: %s." % (blue(myfile),darkred(_("not available, not much ok!"))) 1801 my_show_down_status(message, mytype) 1802 continue 1803 1804 myurl, mypath = self._construct_paths(item, repo, None) 1805 message = "%s: %s." % (blue(myfile),darkgreen(_("available, w00t!"))) 1806 my_show_down_status(message, mytype) 1807 if item not in objects_to_unpack: continue 1808 if not (os.path.isfile(mypath) and os.access(mypath,os.R_OK)): continue 1809 1810 while 1: 1811 tmpdir = os.path.join(os.path.dirname(mypath),"meta_unpack_%s" % (random.randint(1,10000),)) 1812 if not os.path.lexists(tmpdir): break 1813 os.makedirs(tmpdir,0775) 1814 1815 repo_dir = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 1816 try: 1817 done = self.entropyTools.universal_uncompress(mypath, tmpdir, catch_empty = True) 1818 if not done: continue 1819 myfiles_to_move = set(os.listdir(tmpdir)) 1820 1821 # exclude files not available by default 1822 files_not_found_file = etpConst['etpdatabasemetafilesnotfound'] 1823 if files_not_found_file in myfiles_to_move: 1824 myfiles_to_move.remove(files_not_found_file) 1825 try: 1826 with open(os.path.join(tmpdir,files_not_found_file),"r") as f: 1827 f_nf = [x.strip() for x in f.readlines()] 1828 downloaded_by_unpack |= set(f_nf) 1829 except IOError: 1830 pass 1831 1832 for myfile in sorted(myfiles_to_move): 1833 from_mypath = os.path.join(tmpdir,myfile) 1834 to_mypath = os.path.join(repo_dir,myfile) 1835 try: 1836 os.rename(from_mypath,to_mypath) 1837 downloaded_by_unpack.add(myfile) 1838 my_show_file_unpack(myfile) 1839 except OSError: 1840 continue 1841 1842 finally: 1843 1844 shutil.rmtree(tmpdir,True) 1845 try: os.rmdir(tmpdir) 1846 except OSError: pass 1847 1848 1849 mytxt = "%s: %s" % ( 1850 red(_("Repository revision")), 1851 bold(str(self.Entropy.get_repository_revision(repo))), 1852 ) 1853 self.Entropy.updateProgress( 1854 mytxt, 1855 importance = 1, 1856 type = "info", 1857 header = "\t" 1858 ) 1859 1860 1861

1862 - def do_database_indexing(self, repo):

1863 1864 # renice a bit, to avoid eating resources 1865 old_prio = self.Entropy.set_priority(15) 1866 mytxt = red("%s ...") % (_("Indexing Repository metadata"),) 1867 self.Entropy.updateProgress( 1868 mytxt, 1869 importance = 1, 1870 type = "info", 1871 header = "\t", 1872 back = True 1873 ) 1874 dbconn = self.Entropy.open_repository(repo) 1875 dbconn.createAllIndexes() 1876 # get list of indexes 1877 repo_indexes = dbconn.listAllIndexes() 1878 if self.Entropy.clientDbconn != None: 1879 try: # client db can be absent 1880 client_indexes = self.Entropy.clientDbconn.listAllIndexes() 1881 if repo_indexes != client_indexes: 1882 self.Entropy.clientDbconn.createAllIndexes() 1883 except: 1884 pass 1885 self.Entropy.set_priority(old_prio)

1886 1887

1888 - def sync(self):

1889 1890 # close them 1891 self.Entropy.close_all_repositories() 1892 1893 # let's dance! 1894 mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),) 1895 self.Entropy.updateProgress( 1896 mytxt, 1897 importance = 2, 1898 type = "info", 1899 header = darkred(" @@ ") 1900 ) 1901 1902 gave_up = self.Entropy.lock_check(self.Entropy.resources_check_lock) 1903 if gave_up: 1904 return 3 1905 1906 locked = self.Entropy.application_lock_check() 1907 if locked: 1908 self.Entropy.resources_remove_lock() 1909 return 4 1910 1911 # lock 1912 self.Entropy.resources_create_lock() 1913 try: 1914 rc = self.run_sync() 1915 except: 1916 self.Entropy.resources_remove_lock() 1917 raise 1918 if rc: return rc 1919 1920 # remove lock 1921 self.Entropy.resources_remove_lock() 1922 1923 if (self.notAvailable >= len(self.reponames)): 1924 return 2 1925 elif (self.notAvailable > 0): 1926 return 1 1927 1928 return 0

1929

Source Code for Module entropy.client.interfaces.repository