entropy.client.interfaces.methods

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Package Manager Client Miscellaneous functions Interface}. 10 11 """ 12 import os 13 import stat 14 import fcntl 15 import errno 16 import sys 17 import shutil 18 import time 19 import subprocess 20 import tempfile 21 from entropy.i18n import _ 22 from entropy.const import * 23 from entropy.exceptions import * 24 from entropy.db import dbapi2, EntropyRepository 25 from entropy.client.interfaces.db import ClientEntropyRepositoryPlugin 26 from entropy.output import purple, bold, red, blue, darkgreen, darkred, brown 27

28 -class RepositoryMixin:

29 30 __repo_error_messages_cache = set() 31 __repodb_cache = {} 32 _memory_db_instances = {} 33

34 - def validate_repositories(self, quiet = False):

35 self.MirrorStatus.clear() 36 self.__repo_error_messages_cache.clear() 37 38 # clear live masking validation cache, if exists 39 cl_id = self.sys_settings_client_plugin_id 40 client_metadata = self.SystemSettings.get(cl_id, {}) 41 if "masking_validation" in client_metadata: 42 client_metadata['masking_validation']['cache'].clear() 43 44 # valid repositories 45 del self.validRepositories[:] 46 for repoid in self.SystemSettings['repositories']['order']: 47 # open database 48 try: 49 50 dbc = self.open_repository(repoid) 51 dbc.listConfigProtectEntries() 52 dbc.validateDatabase() 53 self.validRepositories.append(repoid) 54 55 except RepositoryError: 56 57 if quiet: 58 continue 59 60 t = _("Repository") + " " + repoid + " " + \ 61 _("is not available") + ". " + _("Cannot validate") 62 t2 = _("Please update your repositories now in order to remove this message!") 63 self.updateProgress( 64 darkred(t), 65 importance = 1, 66 type = "warning" 67 ) 68 self.updateProgress( 69 purple(t2), 70 header = bold("!!! "), 71 importance = 1, 72 type = "warning" 73 ) 74 continue # repo not available 75 except (self.dbapi2.OperationalError, 76 self.dbapi2.DatabaseError, SystemDatabaseError,): 77 78 if quiet: 79 continue 80 81 t = _("Repository") + " " + repoid + " " + \ 82 _("is corrupted") + ". " + _("Cannot validate") 83 self.updateProgress( 84 darkred(t), 85 importance = 1, 86 type = "warning" 87 ) 88 continue 89 90 # to avoid having zillions of open files when loading a lot of EquoInterfaces 91 self.close_all_repositories(mask_clear = False)

92

93 - def __get_repository_cache_key(self, repoid):

94 return (repoid, etpConst['systemroot'],)

95

96 - def init_generic_memory_repository(self, repoid, description, package_mirrors = []):

97 dbc = self.open_memory_database(dbname = repoid) 98 repo_key = self.__get_repository_cache_key(repoid) 99 RepositoryMixin._memory_db_instances[repo_key] = dbc 100 101 # add to self.SystemSettings['repositories']['available'] 102 repodata = { 103 'repoid': repoid, 104 'in_memory': True, 105 'description': description, 106 'packages': package_mirrors, 107 'dbpath': ':memory:', 108 } 109 self.add_repository(repodata) 110 return dbc

111

112 - def close_all_repositories(self, mask_clear = True):

113 for item in self.__repodb_cache: 114 # in-memory repositories cannot be closed 115 # otherwise everything will be lost, to 116 # effectively close these repos you 117 # must call remove_repository method 118 if item in RepositoryMixin._memory_db_instances: 119 continue 120 self.__repodb_cache[item].closeDB() 121 self.__repodb_cache.clear() 122 123 # disable hooks during SystemSettings cleanup 124 # otherwise it makes entropy.client.interfaces.repository crazy 125 old_value = self._can_run_sys_set_hooks 126 self._can_run_sys_set_hooks = False 127 if mask_clear: 128 self.SystemSettings.clear() 129 self._can_run_sys_set_hooks = old_value

130 131

132 - def is_repository_connection_cached(self, repoid):

133 if (repoid, etpConst['systemroot'],) in self.__repodb_cache: 134 return True 135 return False

136

137 - def open_repository(self, repoid):

138 139 key = self.__get_repository_cache_key(repoid) 140 if key not in self.__repodb_cache: 141 dbconn = self.load_repository_database(repoid, xcache = self.xcache, 142 indexing = self.indexing) 143 try: 144 dbconn.checkDatabaseApi() 145 except (self.dbapi2.OperationalError, TypeError,): 146 pass 147 self.__repodb_cache[key] = dbconn 148 return dbconn 149 return self.__repodb_cache.get(key)

150

151 - def load_repository_database(self, repoid, xcache = True, indexing = True):

152 153 if const_isstring(repoid): 154 if repoid.endswith(etpConst['packagesext']): 155 xcache = False 156 157 repo_data = self.SystemSettings['repositories']['available'] 158 if repoid not in repo_data: 159 t = _("bad repository id specified") 160 if repoid not in self.__repo_error_messages_cache: 161 self.updateProgress( 162 darkred(t), 163 importance = 2, 164 type = "warning" 165 ) 166 self.__repo_error_messages_cache.add(repoid) 167 raise RepositoryError("RepositoryError: %s" % (t,)) 168 169 if repo_data[repoid].get('in_memory'): 170 repo_key = self.__get_repository_cache_key(repoid) 171 conn = RepositoryMixin._memory_db_instances.get(repo_key) 172 else: 173 dbfile = repo_data[repoid]['dbpath']+"/"+etpConst['etpdatabasefile'] 174 if not os.path.isfile(dbfile): 175 t = _("Repository %s hasn't been downloaded yet.") % (repoid,) 176 if repoid not in self.__repo_error_messages_cache: 177 self.updateProgress( 178 darkred(t), 179 importance = 2, 180 type = "warning" 181 ) 182 self.__repo_error_messages_cache.add(repoid) 183 raise RepositoryError("RepositoryError: %s" % (t,)) 184 185 conn = EntropyRepository( 186 readOnly = True, 187 dbFile = dbfile, 188 dbname = etpConst['dbnamerepoprefix']+repoid, 189 xcache = xcache, 190 indexing = indexing 191 ) 192 self._add_plugin_to_client_repository(conn) 193 194 if (repoid not in self._treeupdates_repos) and \ 195 (self.entropyTools.is_root()) and \ 196 (not repoid.endswith(etpConst['packagesext'])): 197 # only as root due to Portage 198 try: 199 updated = self.repository_packages_spm_sync(repoid, conn) 200 except (self.dbapi2.OperationalError, self.dbapi2.DatabaseError): 201 updated = False 202 if updated: 203 self.clear_dump_cache(etpCache['world_update']) 204 self.clear_dump_cache(etpCache['critical_update']) 205 self.clear_dump_cache(etpCache['world']) 206 self.clear_dump_cache(etpCache['install']) 207 self.clear_dump_cache(etpCache['remove']) 208 return conn

209

210 - def get_repository_revision(self, reponame):

211 fname = self.SystemSettings['repositories']['available'][reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'] 212 revision = -1 213 if os.path.isfile(fname) and os.access(fname, os.R_OK): 214 with open(fname, "r") as f: 215 try: 216 revision = int(f.readline().strip()) 217 except (OSError, IOError, ValueError,): 218 pass 219 return revision

220

221 - def update_repository_revision(self, reponame):

222 r = self.get_repository_revision(reponame) 223 self.SystemSettings['repositories']['available'][reponame]['dbrevision'] = "0" 224 if r != -1: 225 self.SystemSettings['repositories']['available'][reponame]['dbrevision'] = str(r)

226

227 - def add_repository(self, repodata):

228 product = self.SystemSettings['repositories']['product'] 229 branch = self.SystemSettings['repositories']['branch'] 230 # update self.SystemSettings['repositories']['available'] 231 try: 232 self.SystemSettings['repositories']['available'][repodata['repoid']] = {} 233 self.SystemSettings['repositories']['available'][repodata['repoid']]['description'] = repodata['description'] 234 except KeyError: 235 t = _("repodata dictionary is corrupted") 236 raise InvalidData("InvalidData: %s" % (t,)) 237 238 if repodata['repoid'].endswith(etpConst['packagesext']) or repodata.get('in_memory'): # dynamic repository 239 try: 240 # no need # self.SystemSettings['repositories']['available'][repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] 241 self.SystemSettings['repositories']['available'][repodata['repoid']]['packages'] = repodata['packages'][:] 242 smart_package = repodata.get('smartpackage') 243 if smart_package != None: 244 self.SystemSettings['repositories']['available'][repodata['repoid']]['smartpackage'] = smart_package 245 except KeyError: 246 raise InvalidData("InvalidData: repodata dictionary is corrupted") 247 self.SystemSettings['repositories']['available'][repodata['repoid']]['dbpath'] = repodata.get('dbpath') 248 self.SystemSettings['repositories']['available'][repodata['repoid']]['pkgpath'] = repodata.get('pkgpath') 249 self.SystemSettings['repositories']['available'][repodata['repoid']]['in_memory'] = repodata.get('in_memory') 250 # put at top priority, shift others 251 self.SystemSettings['repositories']['order'].insert(0, repodata['repoid']) 252 else: 253 # XXX it's boring to keep this in sync with entropyConstants stuff, solutions? 254 self.SystemSettings['repositories']['available'][repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] 255 self.SystemSettings['repositories']['available'][repodata['repoid']]['packages'] = [x+"/"+product for x in repodata['plain_packages']] 256 self.SystemSettings['repositories']['available'][repodata['repoid']]['plain_database'] = repodata['plain_database'] 257 self.SystemSettings['repositories']['available'][repodata['repoid']]['database'] = repodata['plain_database'] + \ 258 "/" + product + "/database/" + etpConst['currentarch'] + "/" + branch 259 if not repodata['dbcformat'] in etpConst['etpdatabasesupportedcformats']: 260 repodata['dbcformat'] = etpConst['etpdatabasesupportedcformats'][0] 261 self.SystemSettings['repositories']['available'][repodata['repoid']]['dbcformat'] = repodata['dbcformat'] 262 self.SystemSettings['repositories']['available'][repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + \ 263 "/" + repodata['repoid'] + "/" + product + "/" + etpConst['currentarch'] + "/" + branch 264 # set dbrevision 265 myrev = self.get_repository_revision(repodata['repoid']) 266 if myrev == -1: 267 myrev = 0 268 self.SystemSettings['repositories']['available'][repodata['repoid']]['dbrevision'] = str(myrev) 269 if "position" in repodata: 270 self.SystemSettings['repositories']['order'].insert( 271 repodata['position'], repodata['repoid']) 272 else: 273 self.SystemSettings['repositories']['order'].append( 274 repodata['repoid']) 275 if "service_port" not in repodata: 276 repodata['service_port'] = int(etpConst['socket_service']['port']) 277 if "ssl_service_port" not in repodata: 278 repodata['ssl_service_port'] = int(etpConst['socket_service']['ssl_port']) 279 self.SystemSettings['repositories']['available'][repodata['repoid']]['service_port'] = repodata['service_port'] 280 self.SystemSettings['repositories']['available'][repodata['repoid']]['ssl_service_port'] = repodata['ssl_service_port'] 281 self.repository_move_clear_cache(repodata['repoid']) 282 # save new self.SystemSettings['repositories']['available'] to file 283 self.entropyTools.save_repository_settings(repodata) 284 self.SystemSettings.clear() 285 self.close_all_repositories() 286 self.validate_repositories()

287

288 - def remove_repository(self, repoid, disable = False):

289 290 done = False 291 if repoid in self.SystemSettings['repositories']['available']: 292 del self.SystemSettings['repositories']['available'][repoid] 293 done = True 294 295 if repoid in self.SystemSettings['repositories']['excluded']: 296 del self.SystemSettings['repositories']['excluded'][repoid] 297 done = True 298 299 # also early remove from validRepositories to avoid 300 # issues when reloading SystemSettings which is bound to Entropy Client 301 # SystemSettings plugin, which triggers calculate_world_updates, which 302 # triggers all_repositories_checksum, which triggers open_repository, 303 # which triggers load_repository_database, which triggers an unwanted 304 # output message => "bad repository id specified" 305 if repoid in self.validRepositories: 306 self.validRepositories.remove(repoid) 307 308 # ensure that all dbs are closed 309 self.close_all_repositories() 310 311 if done: 312 313 if repoid in self.SystemSettings['repositories']['order']: 314 self.SystemSettings['repositories']['order'].remove(repoid) 315 316 self.repository_move_clear_cache(repoid) 317 # save new self.SystemSettings['repositories']['available'] to file 318 repodata = {} 319 repodata['repoid'] = repoid 320 if disable: 321 self.entropyTools.save_repository_settings(repodata, disable = True) 322 else: 323 self.entropyTools.save_repository_settings(repodata, remove = True) 324 self.SystemSettings.clear() 325 326 repo_mem_key = self.__get_repository_cache_key(repoid) 327 mem_inst = RepositoryMixin._memory_db_instances.pop(repo_mem_key, None) 328 if isinstance(mem_inst, EntropyRepository): 329 mem_inst.closeDB() 330 331 # reset db cache 332 self.close_all_repositories() 333 self.validate_repositories()

334

335 - def shift_repository(self, repoid, toidx):

336 # update self.SystemSettings['repositories']['order'] 337 self.SystemSettings['repositories']['order'].remove(repoid) 338 self.SystemSettings['repositories']['order'].insert(toidx, repoid) 339 self.entropyTools.write_ordered_repositories_entries( 340 self.SystemSettings['repositories']['order']) 341 self.SystemSettings.clear() 342 self.close_all_repositories() 343 self.repository_move_clear_cache(repoid) 344 self.validate_repositories()

345

346 - def enable_repository(self, repoid):

347 self.repository_move_clear_cache(repoid) 348 # save new self.SystemSettings['repositories']['available'] to file 349 repodata = {} 350 repodata['repoid'] = repoid 351 self.entropyTools.save_repository_settings(repodata, enable = True) 352 self.SystemSettings.clear() 353 self.close_all_repositories() 354 self.validate_repositories()

355

356 - def disable_repository(self, repoid):

357 # update self.SystemSettings['repositories']['available'] 358 done = False 359 try: 360 del self.SystemSettings['repositories']['available'][repoid] 361 done = True 362 except: 363 pass 364 365 if done: 366 try: 367 self.SystemSettings['repositories']['order'].remove(repoid) 368 except (IndexError,): 369 pass 370 # it's not vital to reset 371 # self.SystemSettings['repositories']['order'] counters 372 373 self.repository_move_clear_cache(repoid) 374 # save new self.SystemSettings['repositories']['available'] to file 375 repodata = {} 376 repodata['repoid'] = repoid 377 self.entropyTools.save_repository_settings(repodata, disable = True) 378 self.SystemSettings.clear() 379 380 self.close_all_repositories() 381 self.validate_repositories()

382

383 - def get_repository_settings(self, repoid):

384 try: 385 repodata = self.SystemSettings['repositories']['available'][repoid].copy() 386 except KeyError: 387 if repoid not in self.SystemSettings['repositories']['excluded']: 388 raise 389 repodata = self.SystemSettings['repositories']['excluded'][repoid].copy() 390 return repodata

391 392 # every tbz2 file that would be installed must pass from here

393 - def add_package_to_repos(self, pkg_file):

394 atoms_contained = [] 395 basefile = os.path.basename(pkg_file) 396 db_dir = tempfile.mkdtemp() 397 dbfile = self.entropyTools.extract_edb(pkg_file, 398 dbpath = db_dir+"/packages.db") 399 if dbfile == None: 400 return -1, atoms_contained 401 # add dbfile 402 repodata = {} 403 repodata['repoid'] = basefile 404 repodata['description'] = "Dynamic database from " + basefile 405 repodata['packages'] = [] 406 repodata['dbpath'] = os.path.dirname(dbfile) 407 repodata['pkgpath'] = os.path.realpath(pkg_file) # extra info added 408 repodata['smartpackage'] = False # extra info added 409 410 mydbconn = self.open_generic_database(dbfile) 411 # read all idpackages 412 try: 413 # all branches admitted from external files 414 myidpackages = mydbconn.listAllIdpackages() 415 except (AttributeError, self.dbapi2.DatabaseError, \ 416 self.dbapi2.IntegrityError, self.dbapi2.OperationalError,): 417 return -2, atoms_contained 418 if len(myidpackages) > 1: 419 repodata[basefile]['smartpackage'] = True 420 for myidpackage in myidpackages: 421 compiled_arch = mydbconn.retrieveDownloadURL(myidpackage) 422 if compiled_arch.find("/"+etpSys['arch']+"/") == -1: 423 return -3, atoms_contained 424 atoms_contained.append((int(myidpackage), basefile)) 425 426 self.add_repository(repodata) 427 self.validate_repositories() 428 if basefile not in self.validRepositories: 429 self.remove_repository(basefile) 430 return -4, atoms_contained 431 mydbconn.closeDB() 432 del mydbconn 433 return 0, atoms_contained

434

435 - def reopen_client_repository(self):

436 self.clientDbconn.closeDB() 437 self.open_client_repository() 438 # make sure settings are in sync 439 self.SystemSettings.clear()

440

441 - def _add_plugin_to_client_repository(self, entropy_client_repository):

442 etp_db_meta = { 443 'output_interface': self, 444 } 445 repo_plugin = ClientEntropyRepositoryPlugin(self, 446 metadata = etp_db_meta) 447 entropy_client_repository.add_plugin(repo_plugin)

448

449 - def open_client_repository(self):

450 451 def load_db_from_ram(): 452 self.safe_mode = etpConst['safemodeerrors']['clientdb'] 453 mytxt = "%s, %s" % (_("System database not found or corrupted"), 454 _("running in safe mode using empty database from RAM"),) 455 self.updateProgress( 456 darkred(mytxt), 457 importance = 1, 458 type = "warning", 459 header = bold("!!!"), 460 ) 461 return self.open_memory_database(dbname = etpConst['clientdbid'])

462 463 # if we are in unit testing mode (triggered by unit testing 464 # code), always use db from ram 465 if etpSys['unittest']: 466 self.clientDbconn = load_db_from_ram() 467 return self.clientDbconn 468 469 db_dir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) 470 if not os.path.isdir(db_dir): 471 os.makedirs(db_dir) 472 473 db_path = etpConst['etpdatabaseclientfilepath'] 474 if (not self.noclientdb) and (not os.path.isfile(db_path)): 475 conn = load_db_from_ram() 476 self.entropyTools.print_traceback(f = self.clientLog) 477 else: 478 try: 479 conn = EntropyRepository(readOnly = False, dbFile = db_path, 480 dbname = etpConst['clientdbid'], 481 xcache = self.xcache, indexing = self.indexing 482 ) 483 self._add_plugin_to_client_repository(conn) 484 except (self.dbapi2.DatabaseError,): 485 self.entropyTools.print_traceback(f = self.clientLog) 486 conn = load_db_from_ram() 487 else: 488 # validate database 489 if not self.noclientdb: 490 try: 491 conn.validateDatabase() 492 except SystemDatabaseError: 493 try: 494 conn.closeDB() 495 except: 496 pass 497 self.entropyTools.print_traceback(f = self.clientLog) 498 conn = load_db_from_ram() 499 500 self.clientDbconn = conn 501 return self.clientDbconn

502

503 - def client_repository_sanity_check(self):

504 self.updateProgress( 505 darkred(_("Sanity Check") + ": " + _("system database")), 506 importance = 2, 507 type = "warning" 508 ) 509 idpkgs = self.clientDbconn.listAllIdpackages() 510 length = len(idpkgs) 511 count = 0 512 errors = False 513 scanning_txt = _("Scanning...") 514 for x in idpkgs: 515 count += 1 516 self.updateProgress( 517 darkgreen(scanning_txt), 518 importance = 0, 519 type = "info", 520 back = True, 521 count = (count, length), 522 percent = True 523 ) 524 try: 525 self.clientDbconn.getPackageData(x) 526 except Exception as e: 527 self.entropyTools.print_traceback() 528 errors = True 529 self.updateProgress( 530 darkred(_("Errors on idpackage %s, error: %s")) % (x, str(e)), 531 importance = 0, 532 type = "warning" 533 ) 534 535 if not errors: 536 t = _("Sanity Check") + ": %s" % (bold(_("PASSED")),) 537 self.updateProgress( 538 darkred(t), 539 importance = 2, 540 type = "warning" 541 ) 542 return 0 543 else: 544 t = _("Sanity Check") + ": %s" % (bold(_("CORRUPTED")),) 545 self.updateProgress( 546 darkred(t), 547 importance = 2, 548 type = "warning" 549 ) 550 return -1

551

552 - def open_generic_database(self, dbfile, dbname = None, xcache = None, 553 readOnly = False, indexing_override = None, skipChecks = False):

554 if xcache == None: 555 xcache = self.xcache 556 if indexing_override != None: 557 indexing = indexing_override 558 else: 559 indexing = self.indexing 560 if dbname == None: 561 dbname = etpConst['genericdbid'] 562 conn = EntropyRepository( 563 readOnly = readOnly, 564 dbFile = dbfile, 565 dbname = dbname, 566 xcache = xcache, 567 indexing = indexing, 568 skipChecks = skipChecks 569 ) 570 self._add_plugin_to_client_repository(conn) 571 return conn

572

573 - def open_memory_database(self, dbname = None):

574 if dbname == None: 575 dbname = etpConst['genericdbid'] 576 dbc = EntropyRepository( 577 readOnly = False, 578 dbFile = ':memory:', 579 dbname = dbname, 580 xcache = False, 581 indexing = False, 582 skipChecks = True 583 ) 584 self._add_plugin_to_client_repository(dbc) 585 dbc.initializeDatabase() 586 return dbc

587

588 - def backup_database(self, dbpath, backup_dir = None, silent = False, compress_level = 9):

589 590 if compress_level not in list(range(1, 10)): 591 compress_level = 9 592 593 backup_dir = os.path.dirname(dbpath) 594 if not backup_dir: backup_dir = os.path.dirname(dbpath) 595 dbname = os.path.basename(dbpath) 596 bytes_required = 1024000*300 597 if not (os.access(backup_dir, os.W_OK) and \ 598 os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \ 599 os.access(dbpath, os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)): 600 if not silent: 601 mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")), blue(dbpath), darkred(_("permission denied")),) 602 self.updateProgress( 603 mytxt, 604 importance = 1, 605 type = "error", 606 header = red(" @@ ") 607 ) 608 return False, mytxt 609 610 def get_ts(): 611 from datetime import datetime 612 ts = datetime.fromtimestamp(time.time()) 613 return "%s%s%s_%sh%sm%ss" % (ts.year, ts.month, ts.day, ts.hour, ts.minute, ts.second)

614 615 comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'], dbname, get_ts(),) 616 comp_dbpath = os.path.join(backup_dir, comp_dbname) 617 if not silent: 618 mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")), blue(comp_dbpath),) 619 self.updateProgress( 620 mytxt, 621 importance = 1, 622 type = "info", 623 header = blue(" @@ "), 624 back = True 625 ) 626 import bz2 627 try: 628 self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) 629 except: 630 if not silent: 631 self.entropyTools.print_traceback() 632 return False, _("Unable to compress") 633 634 if not silent: 635 mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")), blue(comp_dbpath),) 636 self.updateProgress( 637 mytxt, 638 importance = 1, 639 type = "info", 640 header = blue(" @@ "), 641 back = True 642 ) 643 return True, _("All fine") 644

645 - def restore_database(self, backup_path, db_destination, silent = False):

646 647 bytes_required = 1024000*300 648 db_dir = os.path.dirname(db_destination) 649 if not (os.access(db_dir, os.W_OK) and os.path.isdir(db_dir) and \ 650 os.path.isfile(backup_path) and os.access(backup_path, os.R_OK) and \ 651 self.entropyTools.check_required_space(db_dir, bytes_required)): 652 653 if not silent: 654 mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")), 655 blue(backup_path), darkred(_("permission denied")),) 656 self.updateProgress( 657 mytxt, 658 importance = 1, 659 type = "error", 660 header = red(" @@ ") 661 ) 662 return False, mytxt 663 664 if not silent: 665 mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")), 666 blue(os.path.basename(backup_path)), blue(db_destination),) 667 self.updateProgress( 668 mytxt, 669 importance = 1, 670 type = "info", 671 header = blue(" @@ "), 672 back = True 673 ) 674 675 import bz2 676 try: 677 self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File) 678 except: 679 if not silent: 680 self.entropyTools.print_traceback() 681 return False, _("Unable to unpack") 682 683 if not silent: 684 mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")), 685 blue(db_destination),) 686 self.updateProgress( 687 mytxt, 688 importance = 1, 689 type = "info", 690 header = blue(" @@ "), 691 back = True 692 ) 693 self.purge_cache() 694 return True, _("All fine")

695

696 - def list_backedup_client_databases(self, client_dbdir = None):

697 if not client_dbdir: 698 client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) 699 return [os.path.join(client_dbdir, x) for x in os.listdir(client_dbdir) \ 700 if x.startswith(etpConst['dbbackupprefix']) and \ 701 os.access(os.path.join(client_dbdir, x), os.R_OK) 702 ]

703

704 - def run_repositories_post_branch_switch_hooks(self, old_branch, new_branch):

705 """ 706 This method is called whenever branch is successfully switched by user. 707 Branch is switched when user wants to upgrade the OS to a new 708 major release. 709 Any repository can be shipped with a sh script which if available, 710 handles system configuration to ease the migration. 711 712 @param old_branch: previously set branch 713 @type old_branch: string 714 @param new_branch: newly set branch 715 @type new_branch: string 716 @return: tuple composed by (1) list of repositories whose script has 717 been run and (2) bool describing if scripts exited with error 718 @rtype: tuple(set, bool) 719 """ 720 721 const_debug_write(__name__, 722 "run_repositories_post_branch_switch_hooks: called") 723 724 client_dbconn = self.clientDbconn 725 hooks_ran = set() 726 if client_dbconn is None: 727 const_debug_write(__name__, 728 "run_repositories_post_branch_switch_hooks: clientdb not avail") 729 return hooks_ran, True 730 731 errors = False 732 repo_data = self.SystemSettings['repositories']['available'] 733 repo_data_excl = self.SystemSettings['repositories']['available'] 734 all_repos = sorted(set(list(repo_data.keys()) + list(repo_data_excl.keys()))) 735 736 for repoid in all_repos: 737 738 const_debug_write(__name__, 739 "run_repositories_post_branch_switch_hooks: %s" % ( 740 repoid,) 741 ) 742 743 mydata = repo_data.get(repoid) 744 if mydata is None: 745 mydata = repo_data_excl.get(repoid) 746 747 if mydata is None: 748 const_debug_write(__name__, 749 "run_repositories_post_branch_switch_hooks: skipping %s" % ( 750 repoid,) 751 ) 752 continue 753 754 branch_mig_script = mydata['post_branch_hop_script'] 755 branch_mig_md5sum = '0' 756 if os.access(branch_mig_script, os.R_OK) and \ 757 os.path.isfile(branch_mig_script): 758 branch_mig_md5sum = self.entropyTools.md5sum(branch_mig_script) 759 760 const_debug_write(__name__, 761 "run_repositories_post_branch_switch_hooks: script md5: %s" % ( 762 branch_mig_md5sum,) 763 ) 764 765 # check if it is needed to run post branch migration script 766 status_md5sums = client_dbconn.isBranchMigrationAvailable( 767 repoid, old_branch, new_branch) 768 if status_md5sums: 769 if branch_mig_md5sum == status_md5sums[0]: # its stored md5 770 const_debug_write(__name__, 771 "run_repositories_post_branch_switch_hooks: skip %s" % ( 772 branch_mig_script,) 773 ) 774 continue # skipping, already ran the same script 775 776 const_debug_write(__name__, 777 "run_repositories_post_branch_switch_hooks: preparing run: %s" % ( 778 branch_mig_script,) 779 ) 780 781 if branch_mig_md5sum != '0': 782 args = ["/bin/sh", branch_mig_script, repoid, 783 etpConst['systemroot'] + "/", old_branch, new_branch] 784 const_debug_write(__name__, 785 "run_repositories_post_branch_switch_hooks: run: %s" % ( 786 args,) 787 ) 788 proc = subprocess.Popen(args, stdin = sys.stdin, 789 stdout = sys.stdout, stderr = sys.stderr) 790 # it is possible to ignore errors because 791 # if it's a critical thing, upstream dev just have to fix 792 # the script and will be automagically re-run 793 br_rc = proc.wait() 794 const_debug_write(__name__, 795 "run_repositories_post_branch_switch_hooks: rc: %s" % ( 796 br_rc,) 797 ) 798 if br_rc != 0: 799 errors = True 800 801 const_debug_write(__name__, 802 "run_repositories_post_branch_switch_hooks: done") 803 804 # update metadata inside database 805 # overriding post branch upgrade md5sum is INTENDED 806 # here but NOT on the other function 807 # this will cause the post-branch upgrade migration 808 # script to be re-run also. 809 client_dbconn.insertBranchMigration(repoid, old_branch, new_branch, 810 branch_mig_md5sum, '0') 811 812 const_debug_write(__name__, 813 "run_repositories_post_branch_switch_hooks: db data: %s" % ( 814 (repoid, old_branch, new_branch, branch_mig_md5sum, '0',),) 815 ) 816 817 hooks_ran.add(repoid) 818 819 return hooks_ran, errors

820

821 - def run_repository_post_branch_upgrade_hooks(self, pretend = False):

822 """ 823 This method is called whenever branch is successfully switched by user 824 and all the updates have been installed (also look at: 825 run_repositories_post_branch_switch_hooks()). 826 Any repository can be shipped with a sh script which if available, 827 handles system configuration to ease the migration. 828 829 @param pretend: do not run hooks but just return list of repos whose 830 scripts should be run 831 @type pretend: bool 832 @return: tuple of length 2 composed by list of repositories whose 833 scripts have been run and errors boolean) 834 @rtype: tuple 835 """ 836 837 const_debug_write(__name__, 838 "run_repository_post_branch_upgrade_hooks: called" 839 ) 840 841 client_dbconn = self.clientDbconn 842 hooks_ran = set() 843 if client_dbconn is None: 844 return hooks_ran, True 845 846 repo_data = self.SystemSettings['repositories']['available'] 847 branch = self.SystemSettings['repositories']['branch'] 848 errors = False 849 850 for repoid in self.validRepositories: 851 852 const_debug_write(__name__, 853 "run_repository_post_branch_upgrade_hooks: repoid: %s" % ( 854 (repoid,), 855 ) 856 ) 857 858 mydata = repo_data.get(repoid) 859 if mydata is None: 860 const_debug_write(__name__, 861 "run_repository_post_branch_upgrade_hooks: repo data N/A") 862 continue 863 864 # check if branch upgrade script exists 865 branch_upg_script = mydata['post_branch_upgrade_script'] 866 branch_upg_md5sum = '0' 867 if os.access(branch_upg_script, os.R_OK) and \ 868 os.path.isfile(branch_upg_script): 869 branch_upg_md5sum = self.entropyTools.md5sum(branch_upg_script) 870 871 if branch_upg_md5sum == '0': 872 # script not found, skip completely 873 const_debug_write(__name__, 874 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 875 repoid, "branch upgrade script not avail",) 876 ) 877 continue 878 879 const_debug_write(__name__, 880 "run_repository_post_branch_upgrade_hooks: script md5: %s" % ( 881 branch_upg_md5sum,) 882 ) 883 884 upgrade_data = client_dbconn.retrieveBranchMigration(branch) 885 if upgrade_data.get(repoid) is None: 886 # no data stored for this repository, skipping 887 const_debug_write(__name__, 888 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 889 repoid, "branch upgrade data not avail",) 890 ) 891 continue 892 repo_upgrade_data = upgrade_data[repoid] 893 894 const_debug_write(__name__, 895 "run_repository_post_branch_upgrade_hooks: upgrade data: %s" % ( 896 repo_upgrade_data,) 897 ) 898 899 for from_branch in sorted(repo_upgrade_data): 900 901 const_debug_write(__name__, 902 "run_repository_post_branch_upgrade_hooks: upgrade: %s" % ( 903 from_branch,) 904 ) 905 906 # yeah, this is run for every branch even if script 907 # which md5 is checked against is the same 908 # this makes the code very flexible 909 post_mig_md5, post_upg_md5 = repo_upgrade_data[from_branch] 910 if branch_upg_md5sum == post_upg_md5: 911 # md5 is equal, this means that it's been already run 912 const_debug_write(__name__, 913 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 914 "already run for from_branch", from_branch,) 915 ) 916 continue 917 918 hooks_ran.add(repoid) 919 920 if pretend: 921 const_debug_write(__name__, 922 "run_repository_post_branch_upgrade_hooks: %s: %s => %s" % ( 923 "pretend enabled, not actually running", 924 repoid, from_branch, 925 ) 926 ) 927 continue 928 929 const_debug_write(__name__, 930 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 931 "running upgrade script from_branch:", from_branch,) 932 ) 933 934 args = ["/bin/sh", branch_upg_script, repoid, 935 etpConst['systemroot'] + "/", from_branch, branch] 936 proc = subprocess.Popen(args, stdin = sys.stdin, 937 stdout = sys.stdout, stderr = sys.stderr) 938 mig_rc = proc.wait() 939 940 const_debug_write(__name__, 941 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 942 "upgrade script exit status", mig_rc,) 943 ) 944 945 if mig_rc != 0: 946 errors = True 947 948 # save branch_upg_md5sum in db 949 client_dbconn.setBranchMigrationPostUpgradeMd5sum(repoid, 950 from_branch, branch, branch_upg_md5sum) 951 952 const_debug_write(__name__, 953 "run_repository_post_branch_upgrade_hooks: %s: %s" % ( 954 "saved upgrade data", 955 (repoid, from_branch, branch, branch_upg_md5sum,), 956 ) 957 ) 958 959 return hooks_ran, errors

960 961

962 -class MiscMixin:

963 964 # resources lock file object container 965 RESOURCES_LOCK_F_REF = None 966 RESOURCES_LOCK_F_COUNT = 0 967

968 - def reload_constants(self):

969 initconfig_entropy_constants(etpSys['rootdir']) 970 self.SystemSettings.clear()

971

972 - def setup_default_file_perms(self, filepath):

973 # setup file permissions 974 os.chmod(filepath, 0o664) 975 if etpConst['entropygid'] != None: 976 os.chown(filepath, -1, etpConst['entropygid'])

977

978 - def resources_create_lock(self):

979 acquired = self.create_pid_file_lock( 980 etpConst['locks']['using_resources']) 981 if acquired: 982 MiscMixin.RESOURCES_LOCK_F_COUNT += 1 983 return acquired

984

985 - def resources_remove_lock(self):

986 987 # decrement lock counter 988 if MiscMixin.RESOURCES_LOCK_F_COUNT > 0: 989 MiscMixin.RESOURCES_LOCK_F_COUNT -= 1 990 991 # if lock counter > 0, still locked 992 # waiting for other upper-level calls 993 if MiscMixin.RESOURCES_LOCK_F_COUNT > 0: 994 return 995 996 f_obj = MiscMixin.RESOURCES_LOCK_F_REF 997 if f_obj is not None: 998 fcntl.flock(f_obj.fileno(), fcntl.LOCK_UN) 999 1000 if f_obj is not None: 1001 f_obj.close() 1002 MiscMixin.RESOURCES_LOCK_F_REF = None 1003 1004 if os.path.isfile(etpConst['locks']['using_resources']): 1005 os.remove(etpConst['locks']['using_resources'])

1006

1007 - def resources_check_lock(self):

1008 return self.check_pid_file_lock(etpConst['locks']['using_resources'])

1009

1010 - def check_pid_file_lock(self, pidfile):

1011 if not os.path.isfile(pidfile): 1012 return False # not locked 1013 f = open(pidfile) 1014 s_pid = f.readline().strip() 1015 f.close() 1016 try: 1017 s_pid = int(s_pid) 1018 except ValueError: 1019 return False # not locked 1020 # is it our pid? 1021 1022 mypid = os.getpid() 1023 if (s_pid != mypid) and const_pid_exists(s_pid): 1024 # is it running 1025 return True # locked 1026 return False

1027

1028 - def create_pid_file_lock(self, pidfile, mypid = None):

1029 1030 if MiscMixin.RESOURCES_LOCK_F_REF is not None: 1031 # already locked, reentrant lock 1032 return True 1033 1034 lockdir = os.path.dirname(pidfile) 1035 if not os.path.isdir(lockdir): 1036 os.makedirs(lockdir, 0o775) 1037 const_setup_perms(lockdir, etpConst['entropygid']) 1038 if mypid == None: 1039 mypid = os.getpid() 1040 1041 pid_f = open(pidfile, "w") 1042 try: 1043 fcntl.flock(pid_f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) 1044 except IOError as err: 1045 if err.errno not in (errno.EACCES, errno.EAGAIN,): 1046 # ouch, wtf? 1047 raise 1048 pid_f.close() 1049 return False # lock already acquired 1050 1051 pid_f.write(str(mypid)) 1052 pid_f.flush() 1053 MiscMixin.RESOURCES_LOCK_F_REF = pid_f 1054 return True

1055

1056 - def application_lock_check(self, silent = False):

1057 # check if another instance is running 1058 etpConst['applicationlock'] = False 1059 const_setup_entropy_pid(just_read = True) 1060 locked = self.entropyTools.application_lock_check(gentle = True) 1061 if locked: 1062 if not silent: 1063 self.updateProgress( 1064 red(_("Another Entropy instance is currently active, cannot satisfy your request.")), 1065 importance = 1, 1066 type = "error", 1067 header = darkred(" @@ ") 1068 ) 1069 return True 1070 return False

1071

1072 - def lock_check(self, check_function):

1073 1074 lock_count = 0 1075 max_lock_count = 600 1076 sleep_seconds = 0.5 1077 1078 # check lock file 1079 while True: 1080 locked = check_function() 1081 if not locked: 1082 if lock_count > 0: 1083 self.updateProgress( 1084 blue(_("Resources unlocked, let's go!")), 1085 importance = 1, 1086 type = "info", 1087 header = darkred(" @@ ") 1088 ) 1089 # wait for other process to exit 1090 # 5 seconds should be enough 1091 time.sleep(5) 1092 break 1093 if lock_count >= max_lock_count: 1094 mycalc = max_lock_count*sleep_seconds/60 1095 self.updateProgress( 1096 blue(_("Resources still locked after %s minutes, giving up!")) % (mycalc,), 1097 importance = 1, 1098 type = "warning", 1099 header = darkred(" @@ ") 1100 ) 1101 return True # gave up 1102 lock_count += 1 1103 self.updateProgress( 1104 blue(_("Resources locked, sleeping %s seconds, check #%s/%s")) % ( 1105 sleep_seconds, 1106 lock_count, 1107 max_lock_count, 1108 ), 1109 importance = 1, 1110 type = "warning", 1111 header = darkred(" @@ "), 1112 back = True 1113 ) 1114 time.sleep(sleep_seconds) 1115 return False # yay!

1116

1117 - def backup_constant(self, constant_name):

1118 if constant_name in etpConst: 1119 myinst = etpConst[constant_name] 1120 if type(etpConst[constant_name]) in (list, tuple): 1121 myinst = etpConst[constant_name][:] 1122 elif type(etpConst[constant_name]) in (dict, set): 1123 myinst = etpConst[constant_name].copy() 1124 else: 1125 myinst = etpConst[constant_name] 1126 etpConst['backed_up'].update({constant_name: myinst}) 1127 else: 1128 t = _("Nothing to backup in etpConst with %s key") % (constant_name,) 1129 raise InvalidData("InvalidData: %s" % (t,))

1130

1131 - def set_priority(self, low = 0):

1132 return const_set_nice_level(low)

1133

1134 - def reload_repositories_config(self, repositories = None):

1135 if repositories is None: 1136 repositories = self.validRepositories 1137 for repoid in repositories: 1138 self.open_repository(repoid)

1139

1140 - def switch_chroot(self, chroot = ""):

1141 # clean caches 1142 self.purge_cache() 1143 self.close_all_repositories() 1144 if chroot.endswith("/"): 1145 chroot = chroot[:-1] 1146 etpSys['rootdir'] = chroot 1147 self.reload_constants() 1148 self.validate_repositories() 1149 self.reopen_client_repository() 1150 # keep them closed, since SystemSettings.clear() is called 1151 # above on reopen_client_repository() 1152 self.close_all_repositories() 1153 if chroot: 1154 try: 1155 self.clientDbconn.resetTreeupdatesDigests() 1156 except: 1157 pass 1158 # I don't think it's safe to keep them open 1159 # isn't it? 1160 self.closeAllSecurity() 1161 self.closeAllQA()

1162

1163 - def get_file_viewer(self):

1164 viewer = None 1165 if os.access("/usr/bin/less", os.X_OK): 1166 viewer = "/usr/bin/less" 1167 elif os.access("/bin/more", os.X_OK): 1168 viewer = "/bin/more" 1169 if not viewer: 1170 viewer = self.get_file_editor() 1171 return viewer

1172

1173 - def get_file_editor(self):

1174 editor = None 1175 if os.getenv("EDITOR"): 1176 editor = "$EDITOR" 1177 elif os.access("/bin/nano", os.X_OK): 1178 editor = "/bin/nano" 1179 elif os.access("/bin/vi", os.X_OK): 1180 editor = "/bin/vi" 1181 elif os.access("/usr/bin/vi", os.X_OK): 1182 editor = "/usr/bin/vi" 1183 elif os.access("/usr/bin/emacs", os.X_OK): 1184 editor = "/usr/bin/emacs" 1185 elif os.access("/bin/emacs", os.X_OK): 1186 editor = "/bin/emacs" 1187 return editor

1188

1189 - def add_user_package_set(self, set_name, set_atoms):

1190 1191 def _ensure_package_sets_dir(): 1192 sets_dir = etpConst['confsetsdir'] 1193 if not os.path.isdir(sets_dir): 1194 if os.path.lexists(sets_dir): 1195 os.remove(sets_dir) 1196 os.makedirs(sets_dir, 0o775) 1197 const_setup_perms(sets_dir, etpConst['entropygid'])

1198 1199 try: 1200 set_name = str(set_name) 1201 except (UnicodeEncodeError, UnicodeDecodeError,): 1202 raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name, _("must be an ASCII string"),)) 1203 1204 if set_name.startswith(etpConst['packagesetprefix']): 1205 raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name, _("cannot start with"), etpConst['packagesetprefix'],)) 1206 set_match, rc = self.package_set_match(set_name) 1207 if rc: return -1, _("Name already taken") 1208 1209 _ensure_package_sets_dir() 1210 set_file = os.path.join(etpConst['confsetsdir'], set_name) 1211 if os.path.isfile(set_file) and os.access(set_file, os.W_OK): 1212 try: 1213 os.remove(set_file) 1214 except OSError: 1215 return -2, _("Cannot remove the old element") 1216 if not os.access(os.path.dirname(set_file), os.W_OK): 1217 return -3, _("Cannot create the element") 1218 1219 f = open(set_file, "w") 1220 for x in set_atoms: f.write("%s\n" % (x,)) 1221 f.flush() 1222 f.close() 1223 self.SystemSettings['system_package_sets'][set_name] = set(set_atoms) 1224 return 0, _("All fine")

1225

1226 - def remove_user_package_set(self, set_name):

1227 1228 try: 1229 set_name = str(set_name) 1230 except (UnicodeEncodeError, UnicodeDecodeError,): 1231 raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name, _("must be an ASCII string"),)) 1232 1233 if set_name.startswith(etpConst['packagesetprefix']): 1234 raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name, _("cannot start with"), etpConst['packagesetprefix'],)) 1235 1236 set_match, rc = self.package_set_match(set_name) 1237 if not rc: return -1, _("Already removed") 1238 set_id, set_x, set_y = set_match 1239 1240 if set_id != etpConst['userpackagesetsid']: 1241 return -2, _("Not defined by user") 1242 set_file = os.path.join(etpConst['confsetsdir'], set_name) 1243 if os.path.isfile(set_file) and os.access(set_file, os.W_OK): 1244 os.remove(set_file) 1245 if set_name in self.SystemSettings['system_package_sets']: 1246 del self.SystemSettings['system_package_sets'][set_name] 1247 return 0, _("All fine") 1248 return -3, _("Set not found or unable to remove")

1249

1250 - def is_installed_idpackage_in_system_mask(self, idpackage):

1251 client_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin'] 1252 mask_installed = self.SystemSettings[client_plugin_id]['system_mask']['repos_installed'] 1253 if idpackage in mask_installed: 1254 return True 1255 return False

1256

1257 - def get_branch_from_download_relative_uri(self, db_download_uri):

1258 return db_download_uri.split("/")[2]

1259

1260 - def swap_branch_in_download_relative_uri(self, new_branch, db_download_uri):

1261 cur_branch = self.get_branch_from_download_relative_uri(db_download_uri) 1262 return db_download_uri.replace("/%s/" % (cur_branch,), 1263 "/%s/" % (new_branch,))

1264

1265 - def unused_packages_test(self, dbconn = None):

1266 if dbconn == None: dbconn = self.clientDbconn 1267 return [x for x in dbconn.retrieveUnusedIdpackages() if self.validate_package_removal(x)]

1268

1269 - def get_licenses_to_accept(self, install_queue):

1270 if not install_queue: 1271 return {} 1272 licenses = {} 1273 cl_id = self.sys_settings_client_plugin_id 1274 repo_sys_data = self.SystemSettings[cl_id]['repositories'] 1275 1276 for match in install_queue: 1277 repoid = match[1] 1278 dbconn = self.open_repository(repoid) 1279 wl = repo_sys_data['license_whitelist'].get(repoid) 1280 if not wl: 1281 continue 1282 keys = dbconn.retrieveLicensedataKeys(match[0]) 1283 for key in keys: 1284 if key not in wl: 1285 found = self.clientDbconn.isLicenseAccepted(key) 1286 if found: 1287 continue 1288 if key not in licenses: 1289 licenses[key] = set() 1290 licenses[key].add(match) 1291 return licenses

1292

1293 - def get_text_license(self, license_name, repoid):

1294 dbconn = self.open_repository(repoid) 1295 text = dbconn.retrieveLicenseText(license_name) 1296 tempfile = self.entropyTools.get_random_temp_file() 1297 f = open(tempfile, "w") 1298 f.write(text) 1299 f.flush() 1300 f.close() 1301 return tempfile

1302

1303 - def set_branch(self, branch):

1304 """ 1305 Set new Entropy branch. This is NOT thread-safe. 1306 Please note that if you call this method all your 1307 repository instance references will become invalid. 1308 This is caused by close_all_repositories and SystemSettings 1309 clear methods. 1310 Once you changed branch, the repository databases won't be 1311 available until you fetch them (through Repositories class) 1312 1313 @param branch -- new branch 1314 @type branch basestring 1315 @return None 1316 """ 1317 self.Cacher.discard() 1318 self.Cacher.stop() 1319 self.purge_cache(showProgress = False) 1320 self.close_all_repositories() 1321 # etpConst should be readonly but we override the rule here 1322 # this is also useful when no config file or parameter into it exists 1323 etpConst['branch'] = branch 1324 self.entropyTools.write_new_branch(branch) 1325 # there are no valid repos atm 1326 del self.validRepositories[:] 1327 self.SystemSettings.clear() 1328 1329 # reset treeupdatesactions 1330 self.reopen_client_repository() 1331 self.clientDbconn.resetTreeupdatesDigests() 1332 self.validate_repositories(quiet = True) 1333 self.close_all_repositories() 1334 if self.xcache: 1335 self.Cacher.start()

1336

1337 - def get_meant_packages(self, search_term, from_installed = False, 1338 valid_repos = None):

1339 1340 if valid_repos is None: 1341 valid_repos = [] 1342 1343 pkg_data = [] 1344 atom_srch = False 1345 if "/" in search_term: 1346 atom_srch = True 1347 1348 if from_installed: 1349 if hasattr(self, 'clientDbconn'): 1350 if self.clientDbconn is not None: 1351 valid_repos.append(self.clientDbconn) 1352 1353 elif not valid_repos: 1354 valid_repos.extend(self.validRepositories[:]) 1355 1356 for repo in valid_repos: 1357 if const_isstring(repo): 1358 dbconn = self.open_repository(repo) 1359 elif isinstance(repo, EntropyRepository): 1360 dbconn = repo 1361 else: 1362 continue 1363 pkg_data.extend([(x, repo,) for x in \ 1364 dbconn.searchSimilarPackages(search_term, atom = atom_srch)]) 1365 1366 return pkg_data

1367

1368 - def get_package_groups(self):

1369 """ 1370 Return Entropy Package Groups metadata. The returned dictionary 1371 contains information to make Entropy Client users to group packages 1372 into "macro" categories. 1373 1374 @return: Entropy Package Groups metadata 1375 @rtype: dict 1376 """ 1377 from entropy.spm.plugins.factory import get_default_class 1378 spm = get_default_class() 1379 groups = spm.get_package_groups().copy() 1380 1381 # expand metadata 1382 categories = self.list_repo_categories() 1383 for data in list(groups.values()): 1384 1385 exp_cats = set() 1386 for g_cat in data['categories']: 1387 exp_cats.update([x for x in categories if x.startswith(g_cat)]) 1388 data['categories'] = sorted(exp_cats) 1389 1390 return groups

1391

1392 - def list_repo_categories(self):

1393 categories = set() 1394 for repo in self.validRepositories: 1395 dbconn = self.open_repository(repo) 1396 catsdata = dbconn.listAllCategories() 1397 categories.update(set([x[1] for x in catsdata])) 1398 return categories

1399

1400 - def list_repo_packages_in_category(self, category):

1401 pkg_matches = [] 1402 for repo in self.validRepositories: 1403 dbconn = self.open_repository(repo) 1404 branch = self.SystemSettings['repositories']['branch'] 1405 catsdata = dbconn.searchPackagesByCategory(category, branch = branch) 1406 pkg_matches.extend([(x[1], repo,) for x in catsdata if (x[1], repo,) not in pkg_matches]) 1407 return pkg_matches

1408

1409 - def get_category_description_data(self, category):

1410 1411 data = {} 1412 for repo in self.validRepositories: 1413 try: 1414 dbconn = self.open_repository(repo) 1415 except RepositoryError: 1416 continue 1417 try: 1418 data = dbconn.retrieveCategoryDescription(category) 1419 except (self.dbapi2.OperationalError, self.dbapi2.IntegrityError,): 1420 continue 1421 if data: break 1422 1423 return data

1424

1425 - def list_installed_packages_in_category(self, category):

1426 pkg_matches = set([x[1] for x in self.clientDbconn.searchPackagesByCategory(category)]) 1427 return pkg_matches

1428

1429 - def get_package_match_config_protect(self, match, mask = False):

1430 1431 idpackage, repoid = match 1432 dbconn = self.open_repository(repoid) 1433 cl_id = self.sys_settings_client_plugin_id 1434 misc_data = self.SystemSettings[cl_id]['misc'] 1435 if mask: 1436 config_protect = set(dbconn.retrieveProtectMask(idpackage).split()) 1437 config_protect |= set(misc_data['configprotectmask']) 1438 else: 1439 config_protect = set(dbconn.retrieveProtect(idpackage).split()) 1440 config_protect |= set(misc_data['configprotect']) 1441 config_protect = [etpConst['systemroot']+x for x in config_protect] 1442 1443 return sorted(config_protect)

1444

1445 - def get_installed_package_config_protect(self, idpackage, mask = False):

1446 1447 if self.clientDbconn == None: 1448 return [] 1449 cl_id = self.sys_settings_client_plugin_id 1450 misc_data = self.SystemSettings[cl_id]['misc'] 1451 if mask: 1452 _pmask = self.clientDbconn.retrieveProtectMask(idpackage).split() 1453 config_protect = set(_pmask) 1454 config_protect |= set(misc_data['configprotectmask']) 1455 else: 1456 _protect = self.clientDbconn.retrieveProtect(idpackage).split() 1457 config_protect = set(_protect) 1458 config_protect |= set(misc_data['configprotect']) 1459 config_protect = [etpConst['systemroot']+x for x in config_protect] 1460 1461 return sorted(config_protect)

1462

1463 - def get_system_config_protect(self, mask = False):

1464 1465 if self.clientDbconn == None: 1466 return [] 1467 1468 # FIXME: workaround because this method is called 1469 # before misc_parser 1470 cl_id = self.sys_settings_client_plugin_id 1471 misc_data = self.SystemSettings[cl_id]['misc'] 1472 if mask: 1473 _pmask = self.clientDbconn.listConfigProtectEntries(mask = True) 1474 config_protect = set(_pmask) 1475 config_protect |= set(misc_data['configprotectmask']) 1476 else: 1477 _protect = self.clientDbconn.listConfigProtectEntries() 1478 config_protect = set(_protect) 1479 config_protect |= set(misc_data['configprotect']) 1480 config_protect = [etpConst['systemroot']+x for x in config_protect] 1481 1482 return sorted(config_protect)

1483

1484 - def inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None):

1485 dbpath = self.get_tmp_dbpath() 1486 dbconn = self.open_generic_database(dbpath) 1487 dbconn.initializeDatabase() 1488 dbconn.addPackage(data, revision = data['revision']) 1489 if treeupdates_actions != None: 1490 dbconn.bumpTreeUpdatesActions(treeupdates_actions) 1491 dbconn.commitChanges() 1492 dbconn.closeDB() 1493 self.entropyTools.aggregate_edb(tbz2file = package_filename, dbfile = dbpath) 1494 return dbpath

1495

1496 - def get_tmp_dbpath(self):

1497 dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.get_random_number()) 1498 while os.path.isfile(dbpath): 1499 dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.get_random_number()) 1500 return dbpath

1501

1502 - def quickpkg(self, atomstring, savedir = None):

1503 if savedir == None: 1504 savedir = etpConst['packagestmpdir'] 1505 if not os.path.isdir(etpConst['packagestmpdir']): 1506 os.makedirs(etpConst['packagestmpdir']) 1507 # match package 1508 match = self.clientDbconn.atomMatch(atomstring) 1509 if match[0] == -1: 1510 return -1, None, None 1511 atom = self.clientDbconn.atomMatch(match[0]) 1512 pkgdata = self.clientDbconn.getPackageData(match[0]) 1513 resultfile = self.quickpkg_handler(pkgdata = pkgdata, dirpath = savedir) 1514 if resultfile == None: 1515 return -1, atom, None 1516 else: 1517 return 0, atom, resultfile

1518

1519 - def quickpkg_handler(self, pkgdata, dirpath, edb = True, 1520 fake = False, compression = "bz2", shiftpath = ""):

1521 1522 import tarfile 1523 1524 if compression not in ("bz2", "", "gz"): 1525 compression = "bz2" 1526 1527 # getting package info 1528 pkgtag = '' 1529 pkgrev = "~"+str(pkgdata['revision']) 1530 if pkgdata['versiontag']: 1531 pkgtag = "#"+pkgdata['versiontag'] 1532 # + version + tag 1533 pkgname = pkgdata['name']+"-"+pkgdata['version']+pkgrev+pkgtag 1534 pkgcat = pkgdata['category'] 1535 pkg_path = dirpath+os.path.sep+pkgname+etpConst['packagesext'] 1536 if os.path.isfile(pkg_path): 1537 os.remove(pkg_path) 1538 tar = tarfile.open(pkg_path, "w:"+compression) 1539 1540 if not fake: 1541 1542 contents = sorted([x for x in pkgdata['content']]) 1543 1544 # collect files 1545 for path in contents: 1546 # convert back to filesystem str 1547 encoded_path = path 1548 path = path.encode('raw_unicode_escape') 1549 path = shiftpath+path 1550 try: 1551 exist = os.lstat(path) 1552 except OSError: 1553 continue # skip file 1554 arcname = path[len(shiftpath):] # remove shiftpath 1555 if arcname.startswith("/"): 1556 arcname = arcname[1:] # remove trailing / 1557 ftype = pkgdata['content'][encoded_path] 1558 if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype 1559 if 'dir' == ftype and \ 1560 not stat.S_ISDIR(exist.st_mode) and \ 1561 os.path.isdir(path): # workaround for directory symlink issues 1562 path = os.path.realpath(path) 1563 1564 tarinfo = tar.gettarinfo(path, arcname) 1565 1566 if stat.S_ISREG(exist.st_mode): 1567 tarinfo.mode = stat.S_IMODE(exist.st_mode) 1568 tarinfo.type = tarfile.REGTYPE 1569 f = open(path) 1570 try: 1571 tar.addfile(tarinfo, f) 1572 finally: 1573 f.close() 1574 else: 1575 tar.addfile(tarinfo) 1576 1577 tar.close() 1578 1579 # append SPM metadata 1580 Spm = self.Spm() 1581 Spm.append_metadata_to_package(pkgcat + "/" + pkgname, pkg_path) 1582 if edb: 1583 self.inject_entropy_database_into_package(pkg_path, pkgdata) 1584 1585 if os.path.isfile(pkg_path): 1586 return pkg_path 1587 return None

1588 1589

1590 -class MatchMixin:

1591

1592 - def get_package_action(self, match):

1593 """ 1594 @input: matched atom (idpackage,repoid) 1595 @output: 1596 upgrade: int(2) 1597 install: int(1) 1598 reinstall: int(0) 1599 downgrade: int(-1) 1600 """ 1601 dbconn = self.open_repository(match[1]) 1602 pkgkey, pkgslot = dbconn.retrieveKeySlot(match[0]) 1603 results = self.clientDbconn.searchKeySlot(pkgkey, pkgslot) 1604 if not results: 1605 return 1 1606 1607 installed_idpackage = results[0][0] 1608 pkgver, pkgtag, pkgrev = dbconn.getVersioningData(match[0]) 1609 installedVer, installedTag, installedRev = self.clientDbconn.getVersioningData(installed_idpackage) 1610 pkgcmp = self.entropyTools.entropy_compare_versions((pkgver, pkgtag, pkgrev), (installedVer, installedTag, installedRev)) 1611 if pkgcmp == 0: 1612 # check digest, if it differs, we should mark pkg as update 1613 # we don't want users to think that they are "reinstalling" stuff 1614 # because it will just confuse them 1615 inst_digest = self.clientDbconn.retrieveDigest(installed_idpackage) 1616 repo_digest = dbconn.retrieveDigest(match[0]) 1617 if inst_digest != repo_digest: 1618 return 2 1619 return 0 1620 elif pkgcmp > 0: 1621 return 2 1622 return -1

1623

1624 - def get_masked_package_reason(self, match):

1625 idpackage, repoid = match 1626 dbconn = self.open_repository(repoid) 1627 idpackage, idreason = dbconn.idpackageValidator(idpackage) 1628 masked = False 1629 if idpackage == -1: masked = True 1630 return masked, idreason, self.SystemSettings['pkg_masking_reasons'].get(idreason)

1631

1632 - def get_match_conflicts(self, match):

1633 m_id, m_repo = match 1634 dbconn = self.open_repository(m_repo) 1635 conflicts = dbconn.retrieveConflicts(m_id) 1636 found_conflicts = set() 1637 for conflict in conflicts: 1638 my_m_id, my_m_rc = self.clientDbconn.atomMatch(conflict) 1639 if my_m_id != -1: 1640 # check if the package shares the same slot 1641 match_data = dbconn.retrieveKeySlot(m_id) 1642 installed_match_data = self.clientDbconn.retrieveKeySlot(my_m_id) 1643 if match_data != installed_match_data: 1644 found_conflicts.add(my_m_id) 1645 return found_conflicts

1646

1647 - def is_match_masked(self, match, live_check = True):

1648 m_id, m_repo = match 1649 dbconn = self.open_repository(m_repo) 1650 idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) 1651 if idpackage != -1: 1652 return False 1653 return True

1654

1655 - def is_match_masked_by_user(self, match, live_check = True):

1656 # (query_status,masked?,) 1657 m_id, m_repo = match 1658 if m_repo not in self.validRepositories: return False 1659 dbconn = self.open_repository(m_repo) 1660 idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) 1661 if idpackage != -1: return False #,False 1662 myr = self.SystemSettings['pkg_masking_reference'] 1663 user_masks = [myr['user_package_mask'], myr['user_license_mask'], myr['user_live_mask']] 1664 if idreason in user_masks: 1665 return True #,True 1666 return False #,True

1667

1668 - def is_match_unmasked_by_user(self, match, live_check = True):

1669 # (query_status,unmasked?,) 1670 m_id, m_repo = match 1671 if m_repo not in self.validRepositories: return False 1672 dbconn = self.open_repository(m_repo) 1673 idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) 1674 if idpackage == -1: return False #,False 1675 myr = self.SystemSettings['pkg_masking_reference'] 1676 user_masks = [ 1677 myr['user_package_unmask'], myr['user_live_unmask'], myr['user_package_keywords'], 1678 myr['user_repo_package_keywords_all'], myr['user_repo_package_keywords'] 1679 ] 1680 if idreason in user_masks: 1681 return True #,True 1682 return False #,True

1683

1684 - def mask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False):

1685 if self.is_match_masked(match, live_check = False): return True 1686 methods = { 1687 'atom': self.mask_match_by_atom, 1688 'keyslot': self.mask_match_by_keyslot, 1689 } 1690 rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) 1691 if dry_run: # inject if done "live" 1692 self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) 1693 self.SystemSettings['live_packagemasking']['mask_matches'].add(match) 1694 return rc

1695

1696 - def unmask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False):

1697 if not self.is_match_masked(match, live_check = False): return True 1698 methods = { 1699 'atom': self.unmask_match_by_atom, 1700 'keyslot': self.unmask_match_by_keyslot, 1701 } 1702 rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) 1703 if dry_run: # inject if done "live" 1704 self.SystemSettings['live_packagemasking']['unmask_matches'].add(match) 1705 self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) 1706 return rc

1707

1708 - def _mask_unmask_match(self, match, method, methods_reference, dry_run = False, clean_all_cache = False):

1709 1710 f = methods_reference.get(method) 1711 if not hasattr(f, '__call__'): 1712 raise IncorrectParameter('IncorrectParameter: %s: %s' % (_("not a valid method"), method,) ) 1713 1714 self.Cacher.discard() 1715 done = f(match, dry_run) 1716 if done and not dry_run: 1717 self.SystemSettings.clear() 1718 1719 # clear atomMatch cache anyway 1720 if clean_all_cache and not dry_run: 1721 self.clear_dump_cache(etpCache['world_available']) 1722 self.clear_dump_cache(etpCache['world_update']) 1723 self.clear_dump_cache(etpCache['critical_update']) 1724 1725 self.clear_dump_cache(etpCache['check_package_update']) 1726 self.clear_dump_cache(etpCache['filter_satisfied_deps']) 1727 self.clear_dump_cache(self.atomMatchCacheKey) 1728 self.clear_dump_cache(etpCache['dep_tree']) 1729 self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'], etpConst['dbnamerepoprefix'], match[1],)) 1730 self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'], etpConst['dbnamerepoprefix'], match[1],)) 1731 1732 cl_id = self.sys_settings_client_plugin_id 1733 self.SystemSettings[cl_id]['masking_validation']['cache'].clear() 1734 return done

1735

1736 - def unmask_match_by_atom(self, match, dry_run = False):

1737 m_id, m_repo = match 1738 dbconn = self.open_repository(m_repo) 1739 atom = dbconn.retrieveAtom(m_id) 1740 return self.unmask_match_generic(match, atom, dry_run = dry_run)

1741

1742 - def unmask_match_by_keyslot(self, match, dry_run = False):

1743 m_id, m_repo = match 1744 dbconn = self.open_repository(m_repo) 1745 keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) 1746 return self.unmask_match_generic(match, keyslot, dry_run = dry_run)

1747

1748 - def mask_match_by_atom(self, match, dry_run = False):

1749 m_id, m_repo = match 1750 dbconn = self.open_repository(m_repo) 1751 atom = dbconn.retrieveAtom(m_id) 1752 return self.mask_match_generic(match, atom, dry_run = dry_run)

1753

1754 - def mask_match_by_keyslot(self, match, dry_run = False):

1755 m_id, m_repo = match 1756 dbconn = self.open_repository(m_repo) 1757 keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) 1758 return self.mask_match_generic(match, keyslot, dry_run = dry_run)

1759

1760 - def unmask_match_generic(self, match, keyword, dry_run = False):

1761 self.clear_match_mask(match, dry_run) 1762 m_file = self.SystemSettings.get_setting_files_data()['unmask'] 1763 return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run)

1764

1765 - def mask_match_generic(self, match, keyword, dry_run = False):

1766 self.clear_match_mask(match, dry_run) 1767 m_file = self.SystemSettings.get_setting_files_data()['mask'] 1768 return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run)

1769

1770 - def _mask_unmask_match_generic(self, keyword, m_file, dry_run = False):

1771 exist = False 1772 if not os.path.isfile(m_file): 1773 if not os.access(os.path.dirname(m_file), os.W_OK): 1774 return False # cannot write 1775 elif not os.access(m_file, os.W_OK): 1776 return False 1777 elif not dry_run: 1778 exist = True 1779 1780 if dry_run: 1781 return True 1782 1783 content = [] 1784 if exist: 1785 f = open(m_file, "r") 1786 content = [x.strip() for x in f.readlines()] 1787 f.close() 1788 content.append(keyword) 1789 m_file_tmp = m_file+".tmp" 1790 f = open(m_file_tmp, "w") 1791 for line in content: 1792 f.write(line+"\n") 1793 f.flush() 1794 f.close() 1795 shutil.move(m_file_tmp, m_file) 1796 return True

1797

1798 - def clear_match_mask(self, match, dry_run = False):

1799 setting_data = self.SystemSettings.get_setting_files_data() 1800 masking_list = [setting_data['mask'], setting_data['unmask']] 1801 return self._clear_match_generic(match, masking_list = masking_list, dry_run = dry_run)

1802

1803 - def _clear_match_generic(self, match, masking_list = [], dry_run = False):

1804 1805 self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) 1806 self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) 1807 1808 if dry_run: return 1809 1810 for mask_file in masking_list: 1811 if not (os.path.isfile(mask_file) and os.access(mask_file, os.W_OK)): continue 1812 f = open(mask_file, "r") 1813 newf = self.entropyTools.open_buffer() 1814 line = f.readline() 1815 while line: 1816 line = line.strip() 1817 if line.startswith("#"): 1818 newf.write(line+"\n") 1819 line = f.readline() 1820 continue 1821 elif not line: 1822 newf.write("\n") 1823 line = f.readline() 1824 continue 1825 mymatch = self.atom_match(line, packagesFilter = False) 1826 if mymatch == match: 1827 line = f.readline() 1828 continue 1829 newf.write(line+"\n") 1830 line = f.readline() 1831 f.close() 1832 tmpfile = mask_file+".w_tmp" 1833 f = open(tmpfile, "w") 1834 f.write(newf.getvalue()) 1835 f.flush() 1836 f.close() 1837 newf.close() 1838 shutil.move(tmpfile, mask_file)

1839

Source Code for Module entropy.client.interfaces.methods