entropy.client.interfaces.repository

1 # -*- coding: utf-8 -*- 2 ''' 3 # DESCRIPTION: 4 # Entropy Object Oriented Interface 5 6 Copyright (C) 2007-2009 Fabio Erculiani 7 8 This program is free software; you can redistribute it and/or modify 9 it under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 2 of the License, or 11 (at your option) any later version. 12 13 This program is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 GNU General Public License for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with this program; if not, write to the Free Software 20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 ''' 22 23 from __future__ import with_statement 24 import os 25 import time 26 import shutil 27 import subprocess 28 import random 29 from entropy.i18n import _ 30 from entropy.db import dbapi2, LocalRepository 31 from entropy.misc import TimeScheduled 32 from entropy.const import * 33 from entropy.exceptions import * 34 from entropy.output import * 35

36 -class Repository:

37 38 import entropy.dump as dumpTools 39 import entropy.tools as entropyTools 40 import socket

41 - def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):

42 43 self.LockScanner = None 44 from entropy.client.interfaces import Client 45 if not isinstance(EquoInstance,Client): 46 mytxt = _("A valid Equo instance or subclass is needed") 47 raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) 48 49 self.supported_download_items = ( 50 "db","rev","ck", 51 "lock","mask", "system_mask", "dbdump", "conflicting_tagged", 52 "dbdumplight", "dbdumplightck", "dbdumpck", "lic_whitelist", 53 "make.conf", "package.mask", "package.unmask", "package.keywords", 54 "profile.link", "package.use", "server.cert", "ca.cert", 55 "meta_file", "notice_board" 56 ) 57 self.big_socket_timeout = 25 58 self.Entropy = EquoInstance 59 from entropy.cache import EntropyCacher 60 self.Cacher = EntropyCacher() 61 self.dbapi2 = dbapi2 62 self.reponames = reponames 63 self.forceUpdate = forceUpdate 64 self.syncErrors = False 65 self.dbupdated = False 66 self.newEquo = False 67 self.fetchSecurity = fetchSecurity 68 self.noEquoCheck = noEquoCheck 69 self.alreadyUpdated = 0 70 self.notAvailable = 0 71 self.valid_eapis = [1,2,3] 72 self.reset_dbformat_eapi(None) 73 self.current_repository_got_locked = False 74 self.updated_repos = set() 75 76 # check self.Entropy.SystemSettings['repositories']['available'] 77 if not self.Entropy.SystemSettings['repositories']['available']: 78 mytxt = _("No repositories specified in %s") % (etpConst['repositoriesconf'],) 79 raise MissingParameter("MissingParameter: %s" % (mytxt,)) 80 81 if not self.reponames: 82 self.reponames.extend(self.Entropy.SystemSettings['repositories']['available'].keys()[:])

83

84 - def __del__(self):

85 if self.LockScanner != None: 86 self.LockScanner.kill()

87

88 - def get_eapi3_connection(self, repository):

89 # get database url 90 dburl = self.Entropy.SystemSettings['repositories']['available'][repository]['plain_database'] 91 if dburl.startswith("file://"): 92 return None 93 try: 94 dburl = dburl.split("/")[2] 95 except IndexError: 96 return None 97 port = self.Entropy.SystemSettings['repositories']['available'][repository]['service_port'] 98 try: 99 from entropy.services.ugc.interfaces import Client 100 from entropy.client.services.ugc.commands import Client as CommandsClient 101 eapi3_socket = Client(self.Entropy, CommandsClient, output_header = "\t") 102 eapi3_socket.socket_timeout = self.big_socket_timeout 103 eapi3_socket.connect(dburl, port) 104 return eapi3_socket 105 except (ConnectionError,self.socket.error,): 106 return None

107

108 - def check_eapi3_availability(self, repository):

109 conn = self.get_eapi3_connection(repository) 110 if conn == None: return False 111 try: 112 conn.disconnect() 113 except (self.socket.error,AttributeError,): 114 return False 115 return True

116

117 - def reset_dbformat_eapi(self, repository):

118 119 self.dbformat_eapi = 2 120 if repository != None: 121 eapi_avail = self.check_eapi3_availability(repository) 122 if eapi_avail: 123 self.dbformat_eapi = 3 124 125 # FIXME, find a way to do that without needing sqlite3 exec. 126 if not os.access("/usr/bin/sqlite3",os.X_OK) or self.entropyTools.islive(): 127 self.dbformat_eapi = 1 128 else: 129 rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True) 130 if rc != 0: self.dbformat_eapi = 1 131 132 eapi_env = os.getenv("FORCE_EAPI") 133 if eapi_env != None: 134 try: 135 myeapi = int(eapi_env) 136 except (ValueError,TypeError,): 137 return 138 if myeapi in self.valid_eapis: 139 self.dbformat_eapi = myeapi

140 141

142 - def __validate_repository_id(self, repoid):

143 if repoid not in self.reponames: 144 mytxt = _("Repository is not listed in self.reponames") 145 raise InvalidData("InvalidData: %s" % (mytxt,))

146

147 - def __validate_compression_method(self, repo):

148 149 self.__validate_repository_id(repo) 150 151 repo_settings = self.Entropy.SystemSettings['repositories'] 152 dbc_format = repo_settings['available'][repo]['dbcformat'] 153 cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format) 154 if cmethod == None: 155 mytxt = _("Wrong database compression method") 156 raise InvalidDataType("InvalidDataType: %s" % (mytxt,)) 157 158 return cmethod

159

160 - def __ensure_repository_path(self, repo):

161 162 self.__validate_repository_id(repo) 163 164 # create dir if it doesn't exist 165 if not os.path.isdir(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']): 166 os.makedirs(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],0775) 167 168 const_setup_perms(etpConst['etpdatabaseclientdir'],etpConst['entropygid'])

169

170 - def _construct_paths(self, item, repo, cmethod):

171 172 if item not in self.supported_download_items: 173 mytxt = _("Supported items: %s") % (self.supported_download_items,) 174 raise InvalidData("InvalidData: %s" % (mytxt,)) 175 if (item in ("db","dbdump", "dbdumpck",)) and (cmethod == None): 176 mytxt = _("For %s, cmethod can't be None") % (item,) 177 raise InvalidData("InvalidData: %s" % (mytxt,)) 178 179 repo_db = self.Entropy.SystemSettings['repositories']['available'][repo]['database'] 180 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 181 ec_rev = etpConst['etpdatabaserevisionfile'] 182 ec_hash = etpConst['etpdatabasehashfile'] 183 ec_maskfile = etpConst['etpdatabasemaskfile'] 184 ec_sysmaskfile = etpConst['etpdatabasesytemmaskfile'] 185 ec_confl_taged = etpConst['etpdatabaseconflictingtaggedfile'] 186 make_conf_file = os.path.basename(etpConst['spm']['global_make_conf']) 187 pkg_mask_file = os.path.basename(etpConst['spm']['global_package_mask']) 188 pkg_unmask_file = os.path.basename(etpConst['spm']['global_package_unmask']) 189 pkg_keywords_file = os.path.basename(etpConst['spm']['global_package_keywords']) 190 pkg_use_file = os.path.basename(etpConst['spm']['global_package_use']) 191 sys_profile_lnk = etpConst['spm']['global_make_profile_link_name'] 192 pkg_lic_wl_file = etpConst['etpdatabaselicwhitelistfile'] 193 repo_lock_file = etpConst['etpdatabasedownloadlockfile'] 194 ca_cert_file = etpConst['etpdatabasecacertfile'] 195 server_cert_file = etpConst['etpdatabaseservercertfile'] 196 notice_board_filename = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board']) 197 meta_file = etpConst['etpdatabasemetafilesfile'] 198 ec_cm2 = None 199 ec_cm3 = None 200 ec_cm4 = None 201 ec_cm5 = None 202 ec_cm6 = None 203 if cmethod != None: 204 ec_cm2 = etpConst[cmethod[2]] 205 ec_cm3 = etpConst[cmethod[3]] 206 ec_cm4 = etpConst[cmethod[4]] 207 ec_cm5 = etpConst[cmethod[5]] 208 ec_cm6 = etpConst[cmethod[6]] 209 210 mymap = { 211 'db': ("%s/%s" % (repo_db,ec_cm2,),"%s/%s" % (repo_dbpath,ec_cm2,),), 212 'dbdump': ("%s/%s" % (repo_db,ec_cm3,),"%s/%s" % (repo_dbpath,ec_cm3,),), 213 'dbdumplight': ("%s/%s" % (repo_db,ec_cm5,),"%s/%s" % (repo_dbpath,ec_cm5,),), 214 'rev': ("%s/%s" % (repo_db,ec_rev,),"%s/%s" % (repo_dbpath,ec_rev,),), 215 'ck': ("%s/%s" % (repo_db,ec_hash,),"%s/%s" % (repo_dbpath,ec_hash,),), 216 'dbdumpck': ("%s/%s" % (repo_db,ec_cm4,),"%s/%s" % (repo_dbpath,ec_cm4,),), 217 'dbdumplightck': ("%s/%s" % (repo_db,ec_cm6,),"%s/%s" % (repo_dbpath,ec_cm6,),), 218 'mask': ("%s/%s" % (repo_db,ec_maskfile,),"%s/%s" % (repo_dbpath,ec_maskfile,),), 219 'system_mask': ("%s/%s" % (repo_db,ec_sysmaskfile,),"%s/%s" % (repo_dbpath,ec_sysmaskfile,),), 220 'conflicting_tagged': ("%s/%s" % (repo_db,ec_confl_taged,),"%s/%s" % (repo_dbpath,ec_confl_taged,),), 221 'make.conf': ("%s/%s" % (repo_db,make_conf_file,),"%s/%s" % (repo_dbpath,make_conf_file,),), 222 'package.mask': ("%s/%s" % (repo_db,pkg_mask_file,),"%s/%s" % (repo_dbpath,pkg_mask_file,),), 223 'package.unmask': ("%s/%s" % (repo_db,pkg_unmask_file,),"%s/%s" % (repo_dbpath,pkg_unmask_file,),), 224 'package.keywords': ("%s/%s" % (repo_db,pkg_keywords_file,),"%s/%s" % (repo_dbpath,pkg_keywords_file,),), 225 'package.use': ("%s/%s" % (repo_db,pkg_use_file,),"%s/%s" % (repo_dbpath,pkg_use_file,),), 226 'profile.link': ("%s/%s" % (repo_db,sys_profile_lnk,),"%s/%s" % (repo_dbpath,sys_profile_lnk,),), 227 'lic_whitelist': ("%s/%s" % (repo_db,pkg_lic_wl_file,),"%s/%s" % (repo_dbpath,pkg_lic_wl_file,),), 228 'lock': ("%s/%s" % (repo_db,repo_lock_file,),"%s/%s" % (repo_dbpath,repo_lock_file,),), 229 'server.cert': ("%s/%s" % (repo_db,server_cert_file,),"%s/%s" % (repo_dbpath,server_cert_file,),), 230 'ca.cert': ("%s/%s" % (repo_db,ca_cert_file,),"%s/%s" % (repo_dbpath,ca_cert_file,),), 231 'notice_board': (self.Entropy.SystemSettings['repositories']['available'][repo]['notice_board'],"%s/%s" % (repo_dbpath,notice_board_filename,),), 232 'meta_file': ("%s/%s" % (repo_db,meta_file,),"%s/%s" % (repo_dbpath,meta_file,),), 233 } 234 235 return mymap.get(item)

236

237 - def __remove_repository_files(self, repo, cmethod):

238 239 dbfilenameid = cmethod[2] 240 self.__validate_repository_id(repo) 241 repo_dbpath = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 242 243 def remove_eapi1(repo_dbpath, dbfilenameid): 244 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabasehashfile']): 245 os.remove(repo_dbpath+"/"+etpConst['etpdatabasehashfile']) 246 if os.path.isfile(repo_dbpath+"/"+etpConst[dbfilenameid]): 247 os.remove(repo_dbpath+"/"+etpConst[dbfilenameid]) 248 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 249 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile'])

250 251 if self.dbformat_eapi == 1: 252 remove_eapi1(repo_dbpath, dbfilenameid) 253 elif self.dbformat_eapi in (2,3,): 254 remove_eapi1(repo_dbpath, dbfilenameid) 255 if os.path.isfile(repo_dbpath+"/"+cmethod[6]): 256 os.remove(repo_dbpath+"/"+cmethod[6]) 257 if os.path.isfile(repo_dbpath+"/"+etpConst[cmethod[5]]): 258 os.remove(repo_dbpath+"/"+etpConst[cmethod[5]]) 259 if os.path.isfile(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']): 260 os.remove(repo_dbpath+"/"+etpConst['etpdatabaserevisionfile']) 261 else: 262 mytxt = _("self.dbformat_eapi must be in (1,2,3,)") 263 raise InvalidData('InvalidData: %s' % (mytxt,))

264

265 - def __unpack_downloaded_database(self, repo, cmethod):

266 267 self.__validate_repository_id(repo) 268 rc = 0 269 path = None 270 271 if self.dbformat_eapi == 1: 272 myfile = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']+"/"+etpConst[cmethod[2]] 273 try: 274 myfunc = getattr(self.entropyTools, cmethod[1]) 275 path = myfunc(myfile) 276 except EOFError: 277 rc = 1 278 if os.path.isfile(myfile): 279 os.remove(myfile) 280 elif self.dbformat_eapi == 2: 281 myfile = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']+"/"+etpConst[cmethod[5]] 282 try: 283 mycall = getattr(self.entropyTools,cmethod[1]) 284 path = mycall(myfile) 285 except EOFError: 286 rc = 1 287 if os.path.isfile(myfile): 288 os.remove(myfile) 289 else: 290 mytxt = _("self.dbformat_eapi must be in (1,2)") 291 raise InvalidData('InvalidData: %s' % (mytxt,)) 292 293 if rc == 0: 294 self.Entropy.setup_default_file_perms(path) 295 296 return rc

297

298 - def __verify_database_checksum(self, repo, cmethod = None):

299 300 self.__validate_repository_id(repo) 301 302 if self.dbformat_eapi == 1: 303 dbfile = etpConst['etpdatabasefile'] 304 try: 305 f = open(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") 306 md5hash = f.readline().strip() 307 md5hash = md5hash.split()[0] 308 f.close() 309 except: 310 return -1 311 elif self.dbformat_eapi == 2: 312 dbfile = etpConst[cmethod[5]] 313 try: 314 f = open(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']+"/"+etpConst[cmethod[6]],"r") 315 md5hash = f.readline().strip() 316 md5hash = md5hash.split()[0] 317 f.close() 318 except: 319 return -1 320 else: 321 mytxt = _("self.dbformat_eapi must be in (1,2)") 322 raise InvalidData('InvalidData: %s' % (mytxt,)) 323 324 rc = self.entropyTools.compare_md5(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']+"/"+dbfile,md5hash) 325 return rc

326 327 # @returns -1 if the file is not available 328 # @returns int>0 if the revision has been retrieved

329 - def get_online_repository_revision(self, repo):

330 331 self.__validate_repository_id(repo) 332 333 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaserevisionfile'] 334 status = self.entropyTools.get_remote_data(url) 335 if (status): 336 status = status[0].strip() 337 try: 338 status = int(status) 339 except ValueError: 340 status = -1 341 return status 342 else: 343 return -1

344

345 - def get_online_eapi3_lock(self, repo):

346 self.__validate_repository_id(repo) 347 url = self.Entropy.SystemSettings['repositories']['available'][repo]['database']+"/"+etpConst['etpdatabaseeapi3lockfile'] 348 data = self.entropyTools.get_remote_data(url) 349 if not data: 350 return False 351 return True

352

353 - def is_repository_eapi3_locked(self, repo):

354 self.__validate_repository_id(repo) 355 return self.get_online_eapi3_lock(repo)

356

357 - def is_repository_updatable(self, repo):

358 359 self.__validate_repository_id(repo) 360 361 onlinestatus = self.get_online_repository_revision(repo) 362 if (onlinestatus != -1): 363 localstatus = self.Entropy.get_repository_revision(repo) 364 if (localstatus == onlinestatus) and (not self.forceUpdate): 365 return False 366 return True

367

368 - def is_repository_unlocked(self, repo):

369 370 self.__validate_repository_id(repo) 371 372 rc = self.download_item("lock", repo, disallow_redirect = True) 373 if rc: # cannot download database 374 self.syncErrors = True 375 return False 376 return True

377

378 - def clear_repository_cache(self, repo):

379 self.__validate_repository_id(repo) 380 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repo,)) 381 self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repo,))

382 383 # this function can be reimplemented

384 - def download_item(self, item, repo, cmethod = None, lock_status_func = None, disallow_redirect = True):

385 386 self.__validate_repository_id(repo) 387 url, filepath = self._construct_paths(item, repo, cmethod) 388 389 # to avoid having permissions issues 390 # it's better to remove the file before, 391 # otherwise new permissions won't be written 392 if os.path.isfile(filepath): 393 os.remove(filepath) 394 filepath_dir = os.path.dirname(filepath) 395 if not os.path.isdir(filepath_dir) and not os.path.lexists(filepath_dir): 396 os.makedirs(filepath_dir,0775) 397 const_setup_perms(filepath_dir, etpConst['entropygid']) 398 399 fetchConn = self.Entropy.urlFetcher( 400 url, 401 filepath, 402 resume = False, 403 abort_check_func = lock_status_func, 404 disallow_redirect = disallow_redirect 405 ) 406 fetchConn.progress = self.Entropy.progress 407 408 rc = fetchConn.download() 409 del fetchConn 410 if rc in ("-1","-2","-3","-4"): 411 return False 412 self.Entropy.setup_default_file_perms(filepath) 413 return True

414

415 - def check_downloaded_database(self, repo, cmethod):

416 dbfilename = etpConst['etpdatabasefile'] 417 if self.dbformat_eapi == 2: 418 dbfilename = etpConst[cmethod[5]] 419 # verify checksum 420 mytxt = "%s %s %s" % ( 421 red(_("Checking downloaded database")), 422 darkgreen(dbfilename), 423 red("..."), 424 ) 425 self.Entropy.updateProgress( 426 mytxt, 427 importance = 0, 428 back = True, 429 type = "info", 430 header = "\t" 431 ) 432 db_status = self.__verify_database_checksum(repo, cmethod) 433 if db_status == -1: 434 mytxt = "%s. %s !" % (red(_("Cannot open digest")),red(_("Cannot verify database integrity")),) 435 self.Entropy.updateProgress( 436 mytxt, 437 importance = 1, 438 type = "warning", 439 header = "\t" 440 ) 441 elif db_status: 442 mytxt = "%s: %s" % (red(_("Downloaded database status")),bold(_("OK")),) 443 self.Entropy.updateProgress( 444 mytxt, 445 importance = 1, 446 type = "info", 447 header = "\t" 448 ) 449 else: 450 mytxt = "%s: %s" % (red(_("Downloaded database status")),darkred(_("ERROR")),) 451 self.Entropy.updateProgress( 452 mytxt, 453 importance = 1, 454 type = "error", 455 header = "\t" 456 ) 457 mytxt = "%s. %s" % (red(_("An error occured while checking database integrity")),red(_("Giving up")),) 458 self.Entropy.updateProgress( 459 mytxt, 460 importance = 1, 461 type = "error", 462 header = "\t" 463 ) 464 return 1 465 return 0

466 467

468 - def show_repository_information(self, repo, count_info):

469 470 self.Entropy.updateProgress( 471 bold("%s") % ( self.Entropy.SystemSettings['repositories']['available'][repo]['description'] ), 472 importance = 2, 473 type = "info", 474 count = count_info, 475 header = blue(" # ") 476 ) 477 mytxt = "%s: %s" % (red(_("Database URL")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['database']),) 478 self.Entropy.updateProgress( 479 mytxt, 480 importance = 1, 481 type = "info", 482 header = blue(" # ") 483 ) 484 mytxt = "%s: %s" % (red(_("Database local path")),darkgreen(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath']),) 485 self.Entropy.updateProgress( 486 mytxt, 487 importance = 0, 488 type = "info", 489 header = blue(" # ") 490 ) 491 mytxt = "%s: %s" % (red(_("Database EAPI")),darkgreen(str(self.dbformat_eapi)),) 492 self.Entropy.updateProgress( 493 mytxt, 494 importance = 0, 495 type = "info", 496 header = blue(" # ") 497 )

498

499 - def get_eapi3_local_database(self, repo):

500 501 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 502 mydbconn = None 503 try: 504 mydbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 505 mydbconn.validateDatabase() 506 except ( 507 self.Entropy.dbapi2.OperationalError, 508 self.Entropy.dbapi2.IntegrityError, 509 SystemDatabaseError, 510 IOError, 511 OSError,): 512 mydbconn = None 513 return mydbconn

514

515 - def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, session):

516 517 product = self.Entropy.SystemSettings['repositories']['product'] 518 data = eapi3_interface.CmdInterface.differential_packages_comparison( 519 session, idpackages, repo, etpConst['currentarch'], product 520 ) 521 if isinstance(data,bool): # then it's probably == False 522 return False,False,False 523 elif not isinstance(data,dict): 524 return None,None,None 525 elif not data.has_key('added') or \ 526 not data.has_key('removed') or \ 527 not data.has_key('checksum'): 528 return None,None,None 529 return data['added'],data['removed'],data['checksum']

530

531 - def get_eapi3_repository_metadata(self, eapi3_interface, repo, session):

532 product = self.Entropy.SystemSettings['repositories']['product'] 533 self.socket.setdefaulttimeout(self.big_socket_timeout) 534 data = eapi3_interface.CmdInterface.get_repository_metadata( 535 session, repo, etpConst['currentarch'], product 536 ) 537 if not isinstance(data,dict): return {} 538 return data

539

540 - def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12):

541 542 def prepare_exit(mysock, session = None): 543 try: 544 if session != None: 545 mysock.close_session(session) 546 mysock.disconnect() 547 except (self.socket.error,): 548 pass

549 550 eapi3_interface = self.get_eapi3_connection(repo) 551 if eapi3_interface == None: return False 552 553 session = eapi3_interface.open_session() 554 555 # AttributeError because mydbconn can be == None 556 try: 557 mydbconn = self.get_eapi3_local_database(repo) 558 myidpackages = mydbconn.listAllIdpackages() 559 except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,AttributeError,): 560 prepare_exit(eapi3_interface, session) 561 return False 562 563 added_ids, removed_ids, checksum = self.get_eapi3_database_differences( 564 eapi3_interface, repo, 565 myidpackages, session 566 ) 567 if (None in (added_ids,removed_ids,checksum)) or \ 568 (not added_ids and not removed_ids and self.forceUpdate): 569 mydbconn.closeDB() 570 prepare_exit(eapi3_interface, session) 571 return False 572 573 elif not checksum: # {added_ids, removed_ids, checksum} == False 574 mydbconn.closeDB() 575 prepare_exit(eapi3_interface, session) 576 mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("remote database suddenly locked")),) 577 self.Entropy.updateProgress( 578 mytxt, 579 importance = 0, 580 type = "info", 581 header = blue(" # "), 582 ) 583 return None 584 585 # is it worth it? 586 if len(added_ids) > threshold: 587 mytxt = "%s: %s (%s: %s/%s)" % ( 588 blue(_("EAPI3 Service")), darkred(_("skipping differential sync")), 589 brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)), 590 ) 591 self.Entropy.updateProgress( 592 mytxt, 593 importance = 0, 594 type = "info", 595 header = blue(" # "), 596 ) 597 mydbconn.closeDB() 598 prepare_exit(eapi3_interface, session) 599 return False 600 601 count = 0 602 added_segments = [] 603 mytmp = set() 604 605 for idpackage in added_ids: 606 count += 1 607 mytmp.add(idpackage) 608 if count % chunk_size == 0: 609 added_segments.append(list(mytmp)) 610 mytmp.clear() 611 if mytmp: added_segments.append(list(mytmp)) 612 del mytmp 613 614 # fetch and store 615 count = 0 616 maxcount = len(added_segments) 617 product = self.Entropy.SystemSettings['repositories']['product'] 618 for segment in added_segments: 619 620 count += 1 621 mytxt = "%s %s" % (blue(_("Fetching segments")), "...",) 622 self.Entropy.updateProgress( 623 mytxt, importance = 0, type = "info", 624 header = "\t", back = True, count = (count,maxcount,) 625 ) 626 fetch_count = 0 627 max_fetch_count = 5 628 629 while 1: 630 631 # anti loop protection 632 if fetch_count > max_fetch_count: 633 mydbconn.closeDB() 634 prepare_exit(eapi3_interface, session) 635 return False 636 637 fetch_count += 1 638 pkgdata = eapi3_interface.CmdInterface.get_strict_package_information( 639 session, segment, repo, etpConst['currentarch'], product 640 ) 641 if pkgdata == None: 642 mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), 643 darkred(str(segment)),) 644 self.Entropy.updateProgress( 645 mytxt, importance = 1, type = "warning", 646 header = "\t", count = (count,maxcount,) 647 ) 648 continue 649 elif not pkgdata: # pkgdata == False 650 mytxt = "%s: %s" % ( 651 blue(_("Service status")), 652 darkred("remote database suddenly locked"), 653 ) 654 self.Entropy.updateProgress( 655 mytxt, importance = 1, type = "info", 656 header = "\t", count = (count,maxcount,) 657 ) 658 mydbconn.closeDB() 659 prepare_exit(eapi3_interface, session) 660 return None 661 elif isinstance(pkgdata,tuple): 662 mytxt = "%s: %s, %s. %s" % ( 663 blue(_("Service status")), 664 pkgdata[0], pkgdata[1], 665 darkred("Error processing the command"), 666 ) 667 self.Entropy.updateProgress( 668 mytxt, importance = 1, type = "info", 669 header = "\t", count = (count,maxcount,) 670 ) 671 mydbconn.closeDB() 672 prepare_exit(eapi3_interface, session) 673 return None 674 675 try: 676 for idpackage in pkgdata: 677 self.dumpTools.dumpobj( 678 "%s%s" % (etpCache['eapi3_fetch'],idpackage,), 679 pkgdata[idpackage], 680 ignore_exceptions = False 681 ) 682 except (IOError,EOFError,OSError,), e: 683 mytxt = "%s: %s: %s." % ( 684 blue(_("Local status")), 685 darkred("Error storing data"), 686 e, 687 ) 688 self.Entropy.updateProgress( 689 mytxt, importance = 1, type = "info", 690 header = "\t", count = (count,maxcount,) 691 ) 692 mydbconn.closeDB() 693 prepare_exit(eapi3_interface, session) 694 return None 695 696 break 697 698 del added_segments 699 700 repo_metadata = self.get_eapi3_repository_metadata(eapi3_interface, 701 repo, session) 702 metadata_elements = ("sets", "treeupdates_actions", 703 "treeupdates_digest", "library_idpackages",) 704 for elem in metadata_elements: 705 if elem not in repo_metadata: 706 mydbconn.closeDB() 707 prepare_exit(eapi3_interface, session) 708 mytxt = "%s: %s" % ( 709 blue(_("EAPI3 Service status")), 710 darkred(_("cannot fetch repository metadata")), 711 ) 712 self.Entropy.updateProgress( 713 mytxt, 714 importance = 0, 715 type = "info", 716 header = blue(" # "), 717 ) 718 return None 719 720 # update treeupdates 721 try: 722 mydbconn.setRepositoryUpdatesDigest(repo, 723 repo_metadata['treeupdates_digest']) 724 mydbconn.bumpTreeUpdatesActions( 725 repo_metadata['treeupdates_actions']) 726 except (self.dbapi2.Error,): 727 mydbconn.closeDB() 728 prepare_exit(eapi3_interface, session) 729 mytxt = "%s: %s" % ( 730 blue(_("EAPI3 Service status")), 731 darkred(_("cannot update treeupdates data")), 732 ) 733 self.Entropy.updateProgress( 734 mytxt, 735 importance = 0, 736 type = "info", 737 header = blue(" # "), 738 ) 739 return None 740 741 # update package sets 742 try: 743 mydbconn.clearPackageSets() 744 mydbconn.insertPackageSets(repo_metadata['sets']) 745 except (self.dbapi2.Error,): 746 mydbconn.closeDB() 747 prepare_exit(eapi3_interface, session) 748 mytxt = "%s: %s" % ( 749 blue(_("EAPI3 Service status")), 750 darkred(_("cannot update package sets data")), 751 ) 752 self.Entropy.updateProgress( 753 mytxt, 754 importance = 0, 755 type = "info", 756 header = blue(" # "), 757 ) 758 return None 759 760 # update libraries <=> idpackages map 761 try: 762 mydbconn.clearNeededLibraryIdpackages() 763 mydbconn.setNeededLibraryIdpackages( 764 repo_metadata['library_idpackages']) 765 except (self.dbapi2.Error,): 766 mydbconn.closeDB() 767 prepare_exit(eapi3_interface, session) 768 mytxt = "%s: %s" % ( 769 blue(_("EAPI3 Service status")), 770 darkred(_("cannot update library data")), 771 ) 772 self.Entropy.updateProgress( 773 mytxt, 774 importance = 0, 775 type = "info", 776 header = blue(" # "), 777 ) 778 return None 779 780 # I don't need you anymore 781 # disconnect socket 782 prepare_exit(eapi3_interface, session) 783 784 # now that we have all stored, add 785 count = 0 786 maxcount = len(added_ids) 787 for idpackage in added_ids: 788 count += 1 789 mydata = self.Cacher.pop("%s%s" % (etpCache['eapi3_fetch'],idpackage,)) 790 if mydata == None: 791 mytxt = "%s: %s" % ( 792 blue(_("Fetch error on segment while adding")), 793 darkred(str(segment)), 794 ) 795 self.Entropy.updateProgress( 796 mytxt, importance = 1, type = "warning", 797 header = "\t", count = (count,maxcount,) 798 ) 799 mydbconn.closeDB() 800 return False 801 802 mytxt = "%s %s" % (blue(_("Injecting package")), darkgreen(mydata['atom']),) 803 self.Entropy.updateProgress( 804 mytxt, importance = 0, type = "info", 805 header = "\t", back = True, count = (count,maxcount,) 806 ) 807 mydbconn.addPackage( 808 mydata, revision = mydata['revision'], 809 idpackage = idpackage, do_remove = False, 810 do_commit = False, formatted_content = True 811 ) 812 813 self.Entropy.updateProgress( 814 blue(_("Packages injection complete")), importance = 0, 815 type = "info", header = "\t", 816 ) 817 818 # now remove 819 maxcount = len(removed_ids) 820 count = 0 821 for idpackage in removed_ids: 822 myatom = mydbconn.retrieveAtom(idpackage) 823 count += 1 824 mytxt = "%s: %s" % (blue(_("Removing package")), darkred(str(myatom)),) 825 self.Entropy.updateProgress( 826 mytxt, importance = 0, type = "info", 827 header = "\t", back = True, count = (count,maxcount,) 828 ) 829 mydbconn.removePackage(idpackage, do_cleanup = False, do_commit = False) 830 831 self.Entropy.updateProgress( 832 blue(_("Packages removal complete")), 833 importance = 0, type = "info", 834 header = "\t", 835 ) 836 837 mydbconn.commitChanges() 838 mydbconn.clearCache() 839 # now verify if both checksums match 840 result = False 841 mychecksum = mydbconn.database_checksum(do_order = True, strict = False, strings = True) 842 if checksum == mychecksum: 843 result = True 844 else: 845 mytxt = "%s: %s: %s | %s: %s" % ( 846 blue(_("Database checksum doesn't match remote.")), 847 darkgreen(_("local")), mychecksum, 848 darkred(_("remote")), checksum, 849 ) 850 self.Entropy.updateProgress( 851 mytxt, importance = 0, 852 type = "info", header = "\t", 853 ) 854 855 mydbconn.closeDB() 856 return result 857

858 - def run_sync(self):

859 860 self.dbupdated = False 861 repocount = 0 862 repolength = len(self.reponames) 863 for repo in self.reponames: 864 865 repocount += 1 866 self.reset_dbformat_eapi(repo) 867 self.show_repository_information(repo, (repocount,repolength)) 868 869 if not self.forceUpdate: 870 updated = self.handle_repository_update(repo) 871 if updated: 872 self.Entropy.cycleDone() 873 self.alreadyUpdated += 1 874 continue 875 876 locked = self.handle_repository_lock(repo) 877 if locked: 878 self.notAvailable += 1 879 self.Entropy.cycleDone() 880 continue 881 882 # clear database interface cache belonging to this repository 883 self.clear_repository_cache(repo) 884 self.__ensure_repository_path(repo) 885 886 # dealing with EAPI 887 # setting some vars 888 do_skip = False 889 skip_this_repo = False 890 db_down_status = False 891 do_db_update_transfer = False 892 rc = 0 893 # some variables 894 dumpfile = os.path.join( 895 self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'], 896 etpConst['etpdatabasedumplight']) 897 dbfile = os.path.join(self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'],etpConst['etpdatabasefile']) 898 dbfile_old = dbfile+".sync" 899 cmethod = self.__validate_compression_method(repo) 900 901 while 1: 902 903 if do_skip: 904 break 905 906 if self.dbformat_eapi < 3: 907 908 down_status = self.handle_database_download(repo, cmethod) 909 if not down_status: 910 self.Entropy.cycleDone() 911 self.notAvailable += 1 912 do_skip = True 913 skip_this_repo = True 914 continue 915 db_down_status = self.handle_database_checksum_download(repo, cmethod) 916 break 917 918 elif self.dbformat_eapi == 3 and not (os.path.isfile(dbfile) and os.access(dbfile,os.W_OK)): 919 920 do_db_update_transfer = None 921 self.dbformat_eapi -= 1 922 continue 923 924 elif self.dbformat_eapi == 3: 925 926 status = False 927 try: 928 status = self.handle_eapi3_database_sync(repo) 929 except self.socket.error, e: 930 mytxt = "%s: %s" % ( 931 blue(_("EAPI3 Service error")), 932 darkred(unicode(e)), 933 ) 934 self.Entropy.updateProgress( 935 mytxt, 936 importance = 0, 937 type = "info", 938 header = blue(" # "), 939 ) 940 except: 941 # avoid broken entries, deal with every exception 942 self.__remove_repository_files(repo, cmethod) 943 raise 944 945 if status == None: # remote db not available anymore ? 946 time.sleep(5) 947 locked = self.handle_repository_lock(repo) 948 if locked: 949 self.Entropy.cycleDone() 950 self.notAvailable += 1 951 do_skip = True 952 skip_this_repo = True 953 else: # ah, well... dunno then... 954 do_db_update_transfer = None 955 self.dbformat_eapi -= 1 956 continue 957 elif not status: # (status == False) 958 # set to none and completely skip database alignment 959 do_db_update_transfer = None 960 self.dbformat_eapi -= 1 961 continue 962 963 break 964 965 if skip_this_repo: 966 continue 967 968 if self.dbformat_eapi in (1,2,): 969 970 if self.dbformat_eapi == 2 and db_down_status: 971 rc = self.check_downloaded_database(repo, cmethod) 972 if rc != 0: 973 # delete all 974 self.__remove_repository_files(repo, cmethod) 975 self.syncErrors = True 976 self.Entropy.cycleDone() 977 continue 978 979 if isinstance(do_db_update_transfer,bool) and not do_db_update_transfer: 980 if os.path.isfile(dbfile): 981 try: 982 shutil.move(dbfile,dbfile_old) 983 do_db_update_transfer = True 984 except: 985 pass 986 987 # unpack database 988 unpack_status = self.handle_downloaded_database_unpack(repo, cmethod) 989 if not unpack_status: 990 # delete all 991 self.__remove_repository_files(repo, cmethod) 992 self.syncErrors = True 993 self.Entropy.cycleDone() 994 continue 995 996 if self.dbformat_eapi == 1 and db_down_status: 997 rc = self.check_downloaded_database(repo, cmethod) 998 if rc != 0: 999 # delete all 1000 self.__remove_repository_files(repo, cmethod) 1001 self.syncErrors = True 1002 self.Entropy.cycleDone() 1003 if os.path.isfile(dbfile_old): 1004 os.remove(dbfile_old) 1005 continue 1006 1007 # re-validate 1008 if not os.path.isfile(dbfile): 1009 do_db_update_transfer = False 1010 elif os.path.isfile(dbfile) and not do_db_update_transfer and (self.dbformat_eapi != 1): 1011 os.remove(dbfile) 1012 1013 if self.dbformat_eapi == 2: 1014 rc = self.do_eapi2_inject_downloaded_dump(dumpfile, dbfile, cmethod) 1015 1016 if do_db_update_transfer: 1017 self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old) 1018 if self.dbformat_eapi == 2: 1019 # remove the dump 1020 os.remove(dumpfile) 1021 1022 if rc != 0: 1023 # delete all 1024 self.__remove_repository_files(repo, cmethod) 1025 self.syncErrors = True 1026 self.Entropy.cycleDone() 1027 if os.path.isfile(dbfile_old): 1028 os.remove(dbfile_old) 1029 continue 1030 1031 if os.path.isfile(dbfile) and os.access(dbfile,os.W_OK): 1032 try: 1033 self.Entropy.setup_default_file_perms(dbfile) 1034 except OSError: # notification applet 1035 pass 1036 1037 # database is going to be updated 1038 self.dbupdated = True 1039 self.do_standard_items_download(repo) 1040 self.Entropy.update_repository_revision(repo) 1041 if self.Entropy.indexing: 1042 self.do_database_indexing(repo) 1043 def_repoid = self.Entropy.SystemSettings['repositories']['default_repository'] 1044 if repo == def_repoid: 1045 try: 1046 self.run_config_files_updates(repo) 1047 except Exception, e: 1048 self.entropyTools.print_traceback() 1049 mytxt = "%s: %s" % ( 1050 blue(_("Configuration files update error, not critical, continuing")), 1051 darkred(unicode(e)), 1052 ) 1053 self.Entropy.updateProgress(mytxt, importance = 0, type = "info", header = blue(" # "),) 1054 self.updated_repos.add(repo) 1055 self.Entropy.cycleDone() 1056 1057 # remove garbage 1058 if os.path.isfile(dbfile_old): 1059 os.remove(dbfile_old) 1060 1061 # keep them closed 1062 self.Entropy.close_all_repositories() 1063 self.Entropy.validate_repositories() 1064 self.Entropy.close_all_repositories() 1065 1066 # clean caches, fetch security 1067 if self.dbupdated: 1068 self.Entropy.generate_cache( 1069 depcache = self.Entropy.xcache, 1070 configcache = False, 1071 client_purge = False, 1072 install_queue = False 1073 ) 1074 if self.fetchSecurity: 1075 self.do_update_security_advisories() 1076 # do treeupdates 1077 if isinstance(self.Entropy.clientDbconn, LocalRepository) and \ 1078 self.entropyTools.is_root(): # only as root due to Portage 1079 for repo in self.reponames: 1080 dbc = self.Entropy.open_repository(repo) 1081 dbc.clientUpdatePackagesData(self.Entropy.clientDbconn) 1082 self.Entropy.close_all_repositories() 1083 1084 if self.syncErrors: 1085 self.Entropy.updateProgress( 1086 red(_("Something bad happened. Please have a look.")), 1087 importance = 1, 1088 type = "warning", 1089 header = darkred(" @@ ") 1090 ) 1091 self.syncErrors = True 1092 self.Entropy.resources_remove_lock() 1093 return 128 1094 1095 if not self.noEquoCheck: 1096 self.check_entropy_updates() 1097 1098 return 0

1099

1100 - def run_config_files_updates(self, repo):

1101 1102 # are we root? 1103 if etpConst['uid'] != 0: 1104 self.Entropy.updateProgress( 1105 brown(_("Skipping configuration files update, you are not root.")), 1106 importance = 1, 1107 type = "info", 1108 header = blue(" @@ ") 1109 ) 1110 return 1111 1112 # make.conf 1113 self._config_updates_make_conf(repo) 1114 self._config_updates_make_profile(repo)

1115 1116

1117 - def _config_updates_make_conf(self, repo):

1118 1119 ## WARNING: it doesn't handle multi-line variables, yet. remember this. 1120 url, repo_make_conf = self._construct_paths("make.conf", repo, None) 1121 system_make_conf = etpConst['spm']['global_make_conf'] 1122 make_conf_variables_check = ["CHOST"] 1123 1124 if os.path.isfile(repo_make_conf) and os.access(repo_make_conf,os.R_OK): 1125 1126 if not os.path.isfile(system_make_conf): 1127 self.Entropy.updateProgress( 1128 "%s %s. %s." % (red(system_make_conf),blue(_("does not exist")),blue(_("Overwriting")),), 1129 importance = 1, 1130 type = "info", 1131 header = blue(" @@ ") 1132 ) 1133 if os.path.lexists(system_make_conf): 1134 shutil.move( 1135 system_make_conf, 1136 "%s.backup_%s" % (system_make_conf,self.entropyTools.get_random_number(),) 1137 ) 1138 shutil.copy2(repo_make_conf,system_make_conf) 1139 1140 elif os.access(system_make_conf,os.W_OK): 1141 1142 repo_f = open(repo_make_conf,"r") 1143 sys_f = open(system_make_conf,"r") 1144 repo_make_c = [x.strip() for x in repo_f.readlines()] 1145 sys_make_c = [x.strip() for x in sys_f.readlines()] 1146 repo_f.close() 1147 sys_f.close() 1148 1149 # read repository settings 1150 repo_data = {} 1151 for setting in make_conf_variables_check: 1152 for line in repo_make_c: 1153 if line.startswith(setting+"="): 1154 # there can't be bash vars with a space after its name on declaration 1155 repo_data[setting] = line 1156 # I don't break, because there might be other overlapping settings 1157 1158 differences = {} 1159 # update make.conf data in memory 1160 for setting in repo_data: 1161 for idx in range(len(sys_make_c)): 1162 line = sys_make_c[idx] 1163 if line.startswith(setting+"=") and (line != repo_data[setting]): 1164 # there can't be bash vars with a space after its name on declaration 1165 self.Entropy.updateProgress( 1166 "%s: %s %s. %s." % ( 1167 red(system_make_conf), bold(unicode(setting)), 1168 blue(_("variable differs")), red(_("Updating")), 1169 ), 1170 importance = 1, 1171 type = "info", 1172 header = blue(" @@ ") 1173 ) 1174 differences[setting] = repo_data[setting] 1175 line = repo_data[setting] 1176 sys_make_c[idx] = line 1177 1178 if differences: 1179 1180 self.Entropy.updateProgress( 1181 "%s: %s." % (red(system_make_conf), blue(_("updating critical variables")),), 1182 importance = 1, 1183 type = "info", 1184 header = blue(" @@ ") 1185 ) 1186 # backup user make.conf 1187 shutil.copy2(system_make_conf,"%s.entropy_backup" % (system_make_conf,)) 1188 1189 self.Entropy.updateProgress( 1190 "%s: %s." % ( 1191 red(system_make_conf), darkgreen("writing changes to disk"), 1192 ), 1193 importance = 1, 1194 type = "info", 1195 header = blue(" @@ ") 1196 ) 1197 # write to disk, safely 1198 tmp_make_conf = "%s.entropy_write" % (system_make_conf,) 1199 f = open(tmp_make_conf,"w") 1200 for line in sys_make_c: f.write(line+"\n") 1201 f.flush() 1202 f.close() 1203 shutil.move(tmp_make_conf,system_make_conf) 1204 1205 # update environment 1206 for var in differences: 1207 try: 1208 myval = '='.join(differences[var].strip().split("=")[1:]) 1209 if myval: 1210 if myval[0] in ("'",'"',): myval = myval[1:] 1211 if myval[-1] in ("'",'"',): myval = myval[:-1] 1212 except IndexError: 1213 myval = '' 1214 os.environ[var] = myval

1215

1216 - def _config_updates_make_profile(self, repo):

1217 url, repo_make_profile = self._construct_paths("profile.link", 1218 repo, None) 1219 system_make_profile = etpConst['spm']['global_make_profile'] 1220 if not (os.path.isfile(repo_make_profile) and \ 1221 os.access(repo_make_profile,os.R_OK)): 1222 return 1223 1224 f = open(repo_make_profile,"r") 1225 repo_profile_link_data = f.readline().strip() 1226 f.close() 1227 current_profile_link = '' 1228 if os.path.islink(system_make_profile) and \ 1229 os.access(system_make_profile,os.R_OK): 1230 1231 current_profile_link = os.readlink(system_make_profile) 1232 1233 if (repo_profile_link_data != current_profile_link) and \ 1234 repo_profile_link_data: 1235 1236 self.Entropy.updateProgress( 1237 "%s: %s %s. %s." % ( 1238 red(system_make_profile), blue("link"), 1239 blue(_("differs")), red(_("Updating")), 1240 ), 1241 importance = 1, 1242 type = "info", 1243 header = blue(" @@ ") 1244 ) 1245 merge_sfx = ".entropy_merge" 1246 os.symlink(repo_profile_link_data,system_make_profile+merge_sfx) 1247 if self.entropyTools.is_valid_path(system_make_profile+merge_sfx): 1248 os.rename(system_make_profile+merge_sfx,system_make_profile) 1249 else: 1250 # revert change, link does not exist yet 1251 self.Entropy.updateProgress( 1252 "%s: %s %s. %s." % ( 1253 red(system_make_profile), blue("new link"), 1254 blue(_("does not exist")), red(_("Reverting")), 1255 ), 1256 importance = 1, 1257 type = "info", 1258 header = blue(" @@ ") 1259 ) 1260 os.remove(system_make_profile+merge_sfx)

1261 1262

1263 - def check_entropy_updates(self):

1264 rc = False 1265 if not self.noEquoCheck: 1266 try: 1267 rc, pkg_match = self.Entropy.check_package_update("sys-apps/entropy", deep = True) 1268 except: 1269 pass 1270 if rc: 1271 self.newEquo = True 1272 mytxt = "%s: %s. %s." % ( 1273 bold("Equo/Entropy"), 1274 blue(_("a new release is available")), 1275 darkred(_("Mind to install it before any other package")), 1276 ) 1277 self.Entropy.updateProgress( 1278 mytxt, 1279 importance = 1, 1280 type = "info", 1281 header = bold(" !!! ") 1282 )

1283

1284 - def handle_downloaded_database_unpack(self, repo, cmethod):

1285 1286 file_to_unpack = etpConst['etpdatabasedump'] 1287 if self.dbformat_eapi == 1: 1288 file_to_unpack = etpConst['etpdatabasefile'] 1289 mytxt = "%s %s %s" % (red(_("Unpacking database to")),darkgreen(file_to_unpack),red("..."),) 1290 self.Entropy.updateProgress( 1291 mytxt, 1292 importance = 0, 1293 type = "info", 1294 header = "\t" 1295 ) 1296 1297 myrc = self.__unpack_downloaded_database(repo, cmethod) 1298 if myrc != 0: 1299 mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),red(_("Skipping repository")),) 1300 self.Entropy.updateProgress( 1301 mytxt, 1302 importance = 1, 1303 type = "warning", 1304 header = "\t" 1305 ) 1306 return False 1307 return True

1308 1309

1310 - def handle_database_checksum_download(self, repo, cmethod):

1311 1312 hashfile = etpConst['etpdatabasehashfile'] 1313 downitem = 'ck' 1314 if self.dbformat_eapi == 2: # EAPI = 2 1315 hashfile = etpConst[cmethod[6]] 1316 downitem = 'dbdumplightck' 1317 1318 mytxt = "%s %s %s" % (red(_("Downloading checksum")),darkgreen(hashfile),red("..."),) 1319 # download checksum 1320 self.Entropy.updateProgress( 1321 mytxt, 1322 importance = 0, 1323 type = "info", 1324 header = "\t" 1325 ) 1326 1327 db_down_status = self.download_item(downitem, repo, cmethod, disallow_redirect = True) 1328 if not db_down_status: 1329 mytxt = "%s %s !" % (red(_("Cannot fetch checksum")),red(_("Cannot verify database integrity")),) 1330 self.Entropy.updateProgress( 1331 mytxt, 1332 importance = 1, 1333 type = "warning", 1334 header = "\t" 1335 ) 1336 return db_down_status

1337

1338 - def load_background_repository_lock_check(self, repo):

1339 # kill previous 1340 self.current_repository_got_locked = False 1341 self.kill_previous_repository_lock_scanner() 1342 self.LockScanner = TimeScheduled(5, self.repository_lock_scanner, repo) 1343 self.LockScanner.start()

1344

1345 - def kill_previous_repository_lock_scanner(self):

1346 if self.LockScanner != None: 1347 self.LockScanner.kill()

1348

1349 - def repository_lock_scanner(self, repo):

1350 locked = self.handle_repository_lock(repo) 1351 if locked: 1352 self.current_repository_got_locked = True

1353

1354 - def repository_lock_scanner_status(self):

1355 # raise an exception if repo got suddenly locked 1356 if self.current_repository_got_locked: 1357 mytxt = _("Current repository got suddenly locked. Download aborted.") 1358 raise RepositoryError('RepositoryError %s' % (mytxt,))

1359

1360 - def handle_database_download(self, repo, cmethod):

1361 1362 def show_repo_locked_message(): 1363 mytxt = "%s: %s." % ( 1364 bold(_("Attention")), 1365 red(_("remote database got suddenly locked")), 1366 ) 1367 self.Entropy.updateProgress( 1368 mytxt, 1369 importance = 1, 1370 type = "warning", 1371 header = "\t" 1372 )

1373 1374 # starting to download 1375 mytxt = "%s ..." % (red(_("Downloading repository database")),) 1376 self.Entropy.updateProgress( 1377 mytxt, 1378 importance = 1, 1379 type = "info", 1380 header = "\t" 1381 ) 1382 1383 down_status = False 1384 if self.dbformat_eapi == 2: 1385 # start a check in background 1386 self.load_background_repository_lock_check(repo) 1387 down_status = self.download_item("dbdumplight", repo, cmethod, 1388 lock_status_func = self.repository_lock_scanner_status, 1389 disallow_redirect = True) 1390 if self.current_repository_got_locked: 1391 self.kill_previous_repository_lock_scanner() 1392 show_repo_locked_message() 1393 return False 1394 if not down_status: # fallback to old db 1395 # start a check in background 1396 self.load_background_repository_lock_check(repo) 1397 self.dbformat_eapi = 1 1398 down_status = self.download_item("db", repo, cmethod, 1399 lock_status_func = self.repository_lock_scanner_status, 1400 disallow_redirect = True) 1401 if self.current_repository_got_locked: 1402 self.kill_previous_repository_lock_scanner() 1403 show_repo_locked_message() 1404 return False 1405 1406 if not down_status: 1407 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database does not exist online")),) 1408 self.Entropy.updateProgress( 1409 mytxt, 1410 importance = 1, 1411 type = "warning", 1412 header = "\t" 1413 ) 1414 1415 self.kill_previous_repository_lock_scanner() 1416 return down_status 1417

1418 - def handle_repository_update(self, repo):

1419 # check if database is already updated to the latest revision 1420 update = self.is_repository_updatable(repo) 1421 if not update: 1422 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database is already up to date")),) 1423 self.Entropy.updateProgress( 1424 mytxt, 1425 importance = 1, 1426 type = "info", 1427 header = "\t" 1428 ) 1429 return True 1430 # also check for eapi3 lock 1431 if self.dbformat_eapi == 3: 1432 locked = self.is_repository_eapi3_locked(repo) 1433 if locked: 1434 mytxt = "%s: %s." % (bold(_("Attention")),red(_("database will be ready soon")),) 1435 self.Entropy.updateProgress( 1436 mytxt, 1437 importance = 1, 1438 type = "info", 1439 header = "\t" 1440 ) 1441 return True 1442 return False

1443

1444 - def handle_repository_lock(self, repo):

1445 # get database lock 1446 unlocked = self.is_repository_unlocked(repo) 1447 if not unlocked: 1448 mytxt = "%s: %s. %s." % ( 1449 bold(_("Attention")), 1450 red(_("Repository is being updated")), 1451 red(_("Try again in a few minutes")), 1452 ) 1453 self.Entropy.updateProgress( 1454 mytxt, 1455 importance = 1, 1456 type = "warning", 1457 header = "\t" 1458 ) 1459 return True 1460 return False

1461

1462 - def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

1463 1464 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 1465 old_dbconn = self.Entropy.open_generic_database(dbfile_old, xcache = False, indexing_override = False) 1466 upd_rc = 0 1467 try: 1468 upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t") 1469 except (self.dbapi2.OperationalError, self.dbapi2.IntegrityError, 1470 self.dbapi2.DatabaseError,): 1471 pass 1472 old_dbconn.closeDB() 1473 dbconn.closeDB() 1474 if upd_rc > 0: 1475 # -1 means no changes, == force used 1476 # 0 means too much hassle 1477 shutil.move(dbfile_old,dbfile) 1478 return upd_rc

1479

1480 - def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

1481 1482 # load the dump into database 1483 mytxt = "%s %s, %s %s" % ( 1484 red(_("Injecting downloaded dump")), 1485 darkgreen(etpConst[cmethod[5]]), 1486 red(_("please wait")), 1487 red("..."), 1488 ) 1489 self.Entropy.updateProgress( 1490 mytxt, 1491 importance = 0, 1492 type = "info", 1493 header = "\t" 1494 ) 1495 dbconn = self.Entropy.open_generic_database(dbfile, xcache = False, indexing_override = False) 1496 rc = dbconn.doDatabaseImport(dumpfile, dbfile) 1497 dbconn.closeDB() 1498 return rc

1499 1500

1501 - def do_update_security_advisories(self):

1502 # update Security Advisories 1503 try: 1504 securityConn = self.Entropy.Security() 1505 securityConn.fetch_advisories() 1506 except Exception, e: 1507 self.entropyTools.print_traceback(f = self.Entropy.clientLog) 1508 mytxt = "%s: %s" % (red(_("Advisories fetch error")),e,) 1509 self.Entropy.updateProgress( 1510 mytxt, 1511 importance = 1, 1512 type = "warning", 1513 header = darkred(" @@ ") 1514 )

1515

1516 - def do_standard_items_download(self, repo):

1517 1518 g_make_conf = os.path.basename(etpConst['spm']['global_make_conf']) 1519 pkg_unmask = os.path.basename(etpConst['spm']['global_package_unmask']) 1520 pkg_keywords = os.path.basename(etpConst['spm']['global_package_keywords']) 1521 pkg_use = os.path.basename(etpConst['spm']['global_package_use']) 1522 profile_link = etpConst['spm']['global_make_profile_link_name'] 1523 notice_board = os.path.basename(self.Entropy.SystemSettings['repositories']['available'][repo]['local_notice_board']) 1524 1525 objects_to_unpack = ("meta_file",) 1526 1527 download_items = [ 1528 ( 1529 "meta_file", 1530 etpConst['etpdatabasemetafilesfile'], 1531 True, 1532 "%s %s %s" % ( 1533 red(_("Downloading repository metafile")), 1534 darkgreen(etpConst['etpdatabasemetafilesfile']), 1535 red("..."), 1536 ) 1537 ), 1538 ( 1539 "ca.cert", 1540 etpConst['etpdatabasecacertfile'], 1541 True, 1542 "%s %s %s" % ( 1543 red(_("Downloading SSL CA certificate")), 1544 darkgreen(etpConst['etpdatabasecacertfile']), 1545 red("..."), 1546 ) 1547 ), 1548 ( 1549 "server.cert", 1550 etpConst['etpdatabaseservercertfile'], 1551 True, 1552 "%s %s %s" % ( 1553 red(_("Downloading SSL Server certificate")), 1554 darkgreen(etpConst['etpdatabaseservercertfile']), 1555 red("..."), 1556 ) 1557 ), 1558 ( 1559 "mask", 1560 etpConst['etpdatabasemaskfile'], 1561 True, 1562 "%s %s %s" % ( 1563 red(_("Downloading package mask")), 1564 darkgreen(etpConst['etpdatabasemaskfile']), 1565 red("..."), 1566 ) 1567 ), 1568 ( 1569 "system_mask", 1570 etpConst['etpdatabasesytemmaskfile'], 1571 True, 1572 "%s %s %s" % ( 1573 red(_("Downloading packages system mask")), 1574 darkgreen(etpConst['etpdatabasesytemmaskfile']), 1575 red("..."), 1576 ) 1577 ), 1578 ( 1579 "conflicting_tagged", 1580 etpConst['etpdatabaseconflictingtaggedfile'], 1581 True, 1582 "%s %s %s" % ( 1583 red(_("Downloading conflicting tagged packages file")), 1584 darkgreen(etpConst['etpdatabaseconflictingtaggedfile']), 1585 red("..."), 1586 ) 1587 ), 1588 ( 1589 "lic_whitelist", 1590 etpConst['etpdatabaselicwhitelistfile'], 1591 True, 1592 "%s %s %s" % ( 1593 red(_("Downloading license whitelist")), 1594 darkgreen(etpConst['etpdatabaselicwhitelistfile']), 1595 red("..."), 1596 ) 1597 ), 1598 ( 1599 "rev", 1600 etpConst['etpdatabaserevisionfile'], 1601 False, 1602 "%s %s %s" % ( 1603 red(_("Downloading revision")), 1604 darkgreen(etpConst['etpdatabaserevisionfile']), 1605 red("..."), 1606 ) 1607 ), 1608 ( 1609 "make.conf", 1610 g_make_conf, 1611 True, 1612 "%s %s %s" % ( 1613 red(_("Downloading SPM global configuration")), 1614 darkgreen(g_make_conf), 1615 red("..."), 1616 ) 1617 ), 1618 ( 1619 "package.unmask", 1620 pkg_unmask, 1621 True, 1622 "%s %s %s" % ( 1623 red(_("Downloading SPM package unmasking configuration")), 1624 darkgreen(pkg_unmask), 1625 red("..."), 1626 ) 1627 ), 1628 ( 1629 "package.keywords", 1630 pkg_keywords, 1631 True, 1632 "%s %s %s" % ( 1633 red(_("Downloading SPM package keywording configuration")), 1634 darkgreen(pkg_keywords), 1635 red("..."), 1636 ) 1637 ), 1638 ( 1639 "package.use", 1640 pkg_use, 1641 True, 1642 "%s %s %s" % ( 1643 red(_("Downloading SPM package USE flags configuration")), 1644 darkgreen(pkg_use), 1645 red("..."), 1646 ) 1647 ), 1648 ( 1649 "profile.link", 1650 profile_link, 1651 True, 1652 "%s %s %s" % ( 1653 red(_("Downloading SPM Profile configuration")), 1654 darkgreen(profile_link), 1655 red("..."), 1656 ) 1657 ), 1658 ( 1659 "notice_board", 1660 notice_board, 1661 True, 1662 "%s %s %s" % ( 1663 red(_("Downloading Notice Board")), 1664 darkgreen(notice_board), 1665 red("..."), 1666 ) 1667 ) 1668 ] 1669 1670 def my_show_info(txt): 1671 self.Entropy.updateProgress( 1672 txt, 1673 importance = 0, 1674 type = "info", 1675 header = "\t", 1676 back = True 1677 )

1678 1679 def my_show_down_status(message, mytype): 1680 self.Entropy.updateProgress( 1681 message, 1682 importance = 0, 1683 type = mytype, 1684 header = "\t" 1685 ) 1686 1687 def my_show_file_unpack(fp): 1688 self.Entropy.updateProgress( 1689 "%s: %s" % (darkgreen(_("unpacked meta file")),brown(fp),), 1690 header = blue(u"\t << ") 1691 ) 1692 1693 downloaded_by_unpack = set() 1694 for item, myfile, ignorable, mytxt in download_items: 1695 1696 # if it's been already downloaded, skip 1697 if myfile in downloaded_by_unpack: continue 1698 1699 my_show_info(mytxt) 1700 mystatus = self.download_item(item, repo, disallow_redirect = True) 1701 mytype = 'info' 1702 1703 # download failed, is it critical? 1704 if not mystatus: 1705 if ignorable: 1706 message = "%s: %s." % (blue(myfile),red(_("not available, it's ok"))) 1707 else: 1708 mytype = 'warning' 1709 message = "%s: %s." % (blue(myfile),darkred(_("not available, not much ok!"))) 1710 my_show_down_status(message, mytype) 1711 continue 1712 1713 myurl, mypath = self._construct_paths(item, repo, None) 1714 message = "%s: %s." % (blue(myfile),darkgreen(_("available, w00t!"))) 1715 my_show_down_status(message, mytype) 1716 if item not in objects_to_unpack: continue 1717 if not (os.path.isfile(mypath) and os.access(mypath,os.R_OK)): continue 1718 1719 while 1: 1720 tmpdir = os.path.join(os.path.dirname(mypath),"meta_unpack_%s" % (random.randint(1,10000),)) 1721 if not os.path.lexists(tmpdir): break 1722 os.makedirs(tmpdir,0775) 1723 1724 repo_dir = self.Entropy.SystemSettings['repositories']['available'][repo]['dbpath'] 1725 try: 1726 done = self.entropyTools.universal_uncompress(mypath, tmpdir, catch_empty = True) 1727 if not done: continue 1728 myfiles_to_move = set(os.listdir(tmpdir)) 1729 1730 # exclude files not available by default 1731 files_not_found_file = etpConst['etpdatabasemetafilesnotfound'] 1732 if files_not_found_file in myfiles_to_move: 1733 myfiles_to_move.remove(files_not_found_file) 1734 try: 1735 with open(os.path.join(tmpdir,files_not_found_file),"r") as f: 1736 f_nf = [x.strip() for x in f.readlines()] 1737 downloaded_by_unpack |= set(f_nf) 1738 except IOError: 1739 pass 1740 1741 for myfile in sorted(myfiles_to_move): 1742 from_mypath = os.path.join(tmpdir,myfile) 1743 to_mypath = os.path.join(repo_dir,myfile) 1744 try: 1745 os.rename(from_mypath,to_mypath) 1746 downloaded_by_unpack.add(myfile) 1747 my_show_file_unpack(myfile) 1748 except OSError: 1749 continue 1750 1751 finally: 1752 1753 shutil.rmtree(tmpdir,True) 1754 try: os.rmdir(tmpdir) 1755 except OSError: pass 1756 1757 1758 mytxt = "%s: %s" % ( 1759 red(_("Repository revision")), 1760 bold(str(self.Entropy.get_repository_revision(repo))), 1761 ) 1762 self.Entropy.updateProgress( 1763 mytxt, 1764 importance = 1, 1765 type = "info", 1766 header = "\t" 1767 ) 1768 1769 1770

1771 - def do_database_indexing(self, repo):

1772 1773 # renice a bit, to avoid eating resources 1774 old_prio = self.Entropy.set_priority(15) 1775 mytxt = red("%s ...") % (_("Indexing Repository metadata"),) 1776 self.Entropy.updateProgress( 1777 mytxt, 1778 importance = 1, 1779 type = "info", 1780 header = "\t", 1781 back = True 1782 ) 1783 dbconn = self.Entropy.open_repository(repo) 1784 dbconn.createAllIndexes() 1785 # get list of indexes 1786 repo_indexes = dbconn.listAllIndexes() 1787 if self.Entropy.clientDbconn != None: 1788 try: # client db can be absent 1789 client_indexes = self.Entropy.clientDbconn.listAllIndexes() 1790 if repo_indexes != client_indexes: 1791 self.Entropy.clientDbconn.createAllIndexes() 1792 except: 1793 pass 1794 self.Entropy.set_priority(old_prio)

1795 1796

1797 - def sync(self):

1798 1799 # close them 1800 self.Entropy.close_all_repositories() 1801 1802 # let's dance! 1803 mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),) 1804 self.Entropy.updateProgress( 1805 mytxt, 1806 importance = 2, 1807 type = "info", 1808 header = darkred(" @@ ") 1809 ) 1810 1811 gave_up = self.Entropy.lock_check(self.Entropy.resources_check_lock) 1812 if gave_up: 1813 return 3 1814 1815 locked = self.Entropy.application_lock_check() 1816 if locked: 1817 self.Entropy.resources_remove_lock() 1818 return 4 1819 1820 # lock 1821 self.Entropy.resources_create_lock() 1822 try: 1823 rc = self.run_sync() 1824 except: 1825 self.Entropy.resources_remove_lock() 1826 raise 1827 if rc: return rc 1828 1829 # remove lock 1830 self.Entropy.resources_remove_lock() 1831 1832 if (self.notAvailable >= len(self.reponames)): 1833 return 2 1834 elif (self.notAvailable > 0): 1835 return 1 1836 1837 return 0

1838

Source Code for Module entropy.client.interfaces.repository