entropy.db

Source Code for Module entropy.db

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework repository database module}. 10 Entropy repositories (server and client) are implemented as relational 11 databases. Currently, LocalRepository class is the object that wraps 12 sqlite3 database queries and repository logic: there are no more 13 abstractions between the two because there is only one implementation 14 available at this time. In future, entropy.db will feature more backends 15 such as MySQL embedded, SparQL, remote repositories support via TCP socket, 16 etc. This will require a new layer between the repository interface now 17 offered by LocalRepository and the underlying data retrieval logic. 18 Every repository interface available inherits from EntropyRepository 19 class and has to reimplement its own Schema subclass and its get_init 20 method (see LocalRepository documentation for more information). 21 22 I{LocalRepository} is the sqlite3 implementation of the repository 23 interface, as written above. 24 25 I{ServerRepositoryStatus} is a singleton containing the status of 26 server-side repositories. It is used to determine if repository has 27 been modified (tainted) or has been revision bumped already. 28 Revision bumps are automatic and happen on the very first data "commit". 29 Every repository features a revision number which is stored into the 30 "packages.db.revision" file. Only server-side (or community) repositories 31 are subject to this automation (revision file update on commit). 32 33 """ 34 35 from __future__ import with_statement 36 import os 37 import shutil 38 from entropy.const import etpConst, etpCache 39 from entropy.exceptions import IncorrectParameter, InvalidAtom, \ 40 SystemDatabaseError, OperationNotPermitted 41 from entropy.i18n import _ 42 from entropy.output import brown, bold, red, blue, purple, darkred, darkgreen, \ 43 TextInterface 44 from entropy.cache import EntropyCacher 45 from entropy.core import Singleton, SystemSettings 46 47 try: # try with sqlite3 from >=python 2.5 48 from sqlite3 import dbapi2 49 except ImportError: # fallback to pysqlite 50 try: 51 from pysqlite2 import dbapi2 52 except ImportError, e: 53 raise SystemError( 54 "%s. %s: %s" % ( 55 _("Entropy needs Python compiled with sqlite3 support"), 56 _("Error"), 57 e, 58 ) 59 ) 60

61 -class ServerRepositoryStatus(Singleton):

62

63 - def init_singleton(self):

64 self.__data = {}

65

66 - def __create_if_necessary(self, db):

67 if db not in self.__data: 68 self.__data[db] = {} 69 self.__data[db]['tainted'] = False 70 self.__data[db]['bumped'] = False 71 self.__data[db]['unlock_msg'] = False

72

73 - def set_unlock_msg(self, db):

74 self.__create_if_necessary(db) 75 self.__data[db]['unlock_msg'] = True

76

77 - def unset_unlock_msg(self, db):

78 self.__create_if_necessary(db) 79 self.__data[db]['unlock_msg'] = False

80

81 - def set_tainted(self, db):

82 self.__create_if_necessary(db) 83 self.__data[db]['tainted'] = True

84

85 - def unset_tainted(self, db):

86 self.__create_if_necessary(db) 87 self.__data[db]['tainted'] = False

88

89 - def set_bumped(self, db):

90 self.__create_if_necessary(db) 91 self.__data[db]['bumped'] = True

92

93 - def unset_bumped(self, db):

94 self.__create_if_necessary(db) 95 self.__data[db]['bumped'] = False

96

97 - def is_tainted(self, db):

98 self.__create_if_necessary(db) 99 return self.__data[db]['tainted']

100

101 - def is_bumped(self, db):

102 self.__create_if_necessary(db) 103 return self.__data[db]['bumped']

104

105 - def is_unlock_msg(self, db):

106 self.__create_if_necessary(db) 107 return self.__data[db]['unlock_msg']

108 109

110 -class EntropyRepository:

111 112 """ 113 Entropy repository interface base class. 114 This class contains database implementation independent logic, and 115 is inherited by all the Entropy repository interfaces implementations 116 (actually only LocalRepository). 117 """ 118

119 - class Schema:

120

121 - def get_init(self):

122 raise NotImplementedError

123

124 - def __init__(self):

125 """ 126 EntropyRepository base repository interface class constructor. 127 128 @return: None 129 @rtype: None 130 """ 131 self.SystemSettings = SystemSettings() 132 self.srv_sys_settings_plugin = \ 133 etpConst['system_settings_plugins_ids']['server_plugin'] 134 self.dbMatchCacheKey = etpCache['dbMatch'] 135 self.client_settings_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin'] 136 self.db_branch = self.SystemSettings['repositories']['branch'] 137 self.Cacher = EntropyCacher()

138 139

140 -class LocalRepository(EntropyRepository):

141

142 - class Schema:

143

144 - def get_init(self):

145 return """ 146 CREATE TABLE baseinfo ( 147 idpackage INTEGER PRIMARY KEY AUTOINCREMENT, 148 atom VARCHAR, 149 idcategory INTEGER, 150 name VARCHAR, 151 version VARCHAR, 152 versiontag VARCHAR, 153 revision INTEGER, 154 branch VARCHAR, 155 slot VARCHAR, 156 idlicense INTEGER, 157 etpapi INTEGER, 158 trigger INTEGER 159 ); 160 161 CREATE TABLE extrainfo ( 162 idpackage INTEGER PRIMARY KEY, 163 description VARCHAR, 164 homepage VARCHAR, 165 download VARCHAR, 166 size VARCHAR, 167 idflags INTEGER, 168 digest VARCHAR, 169 datecreation VARCHAR 170 ); 171 172 CREATE TABLE content ( 173 idpackage INTEGER, 174 file VARCHAR, 175 type VARCHAR 176 ); 177 178 CREATE TABLE provide ( 179 idpackage INTEGER, 180 atom VARCHAR 181 ); 182 183 CREATE TABLE dependencies ( 184 idpackage INTEGER, 185 iddependency INTEGER, 186 type INTEGER 187 ); 188 189 CREATE TABLE dependenciesreference ( 190 iddependency INTEGER PRIMARY KEY AUTOINCREMENT, 191 dependency VARCHAR 192 ); 193 194 CREATE TABLE dependstable ( 195 iddependency INTEGER PRIMARY KEY, 196 idpackage INTEGER 197 ); 198 199 CREATE TABLE conflicts ( 200 idpackage INTEGER, 201 conflict VARCHAR 202 ); 203 204 CREATE TABLE mirrorlinks ( 205 mirrorname VARCHAR, 206 mirrorlink VARCHAR 207 ); 208 209 CREATE TABLE sources ( 210 idpackage INTEGER, 211 idsource INTEGER 212 ); 213 214 CREATE TABLE sourcesreference ( 215 idsource INTEGER PRIMARY KEY AUTOINCREMENT, 216 source VARCHAR 217 ); 218 219 CREATE TABLE useflags ( 220 idpackage INTEGER, 221 idflag INTEGER 222 ); 223 224 CREATE TABLE useflagsreference ( 225 idflag INTEGER PRIMARY KEY AUTOINCREMENT, 226 flagname VARCHAR 227 ); 228 229 CREATE TABLE keywords ( 230 idpackage INTEGER, 231 idkeyword INTEGER 232 ); 233 234 CREATE TABLE keywordsreference ( 235 idkeyword INTEGER PRIMARY KEY AUTOINCREMENT, 236 keywordname VARCHAR 237 ); 238 239 CREATE TABLE categories ( 240 idcategory INTEGER PRIMARY KEY AUTOINCREMENT, 241 category VARCHAR 242 ); 243 244 CREATE TABLE licenses ( 245 idlicense INTEGER PRIMARY KEY AUTOINCREMENT, 246 license VARCHAR 247 ); 248 249 CREATE TABLE flags ( 250 idflags INTEGER PRIMARY KEY AUTOINCREMENT, 251 chost VARCHAR, 252 cflags VARCHAR, 253 cxxflags VARCHAR 254 ); 255 256 CREATE TABLE configprotect ( 257 idpackage INTEGER PRIMARY KEY, 258 idprotect INTEGER 259 ); 260 261 CREATE TABLE configprotectmask ( 262 idpackage INTEGER PRIMARY KEY, 263 idprotect INTEGER 264 ); 265 266 CREATE TABLE configprotectreference ( 267 idprotect INTEGER PRIMARY KEY AUTOINCREMENT, 268 protect VARCHAR 269 ); 270 271 CREATE TABLE systempackages ( 272 idpackage INTEGER PRIMARY KEY 273 ); 274 275 CREATE TABLE injected ( 276 idpackage INTEGER PRIMARY KEY 277 ); 278 279 CREATE TABLE installedtable ( 280 idpackage INTEGER PRIMARY KEY, 281 repositoryname VARCHAR, 282 source INTEGER 283 ); 284 285 CREATE TABLE sizes ( 286 idpackage INTEGER PRIMARY KEY, 287 size INTEGER 288 ); 289 290 CREATE TABLE messages ( 291 idpackage INTEGER, 292 message VARCHAR 293 ); 294 295 CREATE TABLE counters ( 296 counter INTEGER, 297 idpackage INTEGER, 298 branch VARCHAR, 299 PRIMARY KEY(idpackage,branch) 300 ); 301 302 CREATE TABLE trashedcounters ( 303 counter INTEGER 304 ); 305 306 CREATE TABLE eclasses ( 307 idpackage INTEGER, 308 idclass INTEGER 309 ); 310 311 CREATE TABLE eclassesreference ( 312 idclass INTEGER PRIMARY KEY AUTOINCREMENT, 313 classname VARCHAR 314 ); 315 316 CREATE TABLE needed ( 317 idpackage INTEGER, 318 idneeded INTEGER, 319 elfclass INTEGER 320 ); 321 322 CREATE TABLE neededreference ( 323 idneeded INTEGER PRIMARY KEY AUTOINCREMENT, 324 library VARCHAR 325 ); 326 327 CREATE TABLE neededlibrarypaths ( 328 library VARCHAR, 329 path VARCHAR, 330 elfclass INTEGER, 331 PRIMARY KEY(library, path, elfclass) 332 ); 333 334 CREATE TABLE neededlibraryidpackages ( 335 idpackage INTEGER, 336 library VARCHAR, 337 elfclass INTEGER 338 ); 339 340 CREATE TABLE treeupdates ( 341 repository VARCHAR PRIMARY KEY, 342 digest VARCHAR 343 ); 344 345 CREATE TABLE treeupdatesactions ( 346 idupdate INTEGER PRIMARY KEY AUTOINCREMENT, 347 repository VARCHAR, 348 command VARCHAR, 349 branch VARCHAR, 350 date VARCHAR 351 ); 352 353 CREATE TABLE licensedata ( 354 licensename VARCHAR UNIQUE, 355 text BLOB, 356 compressed INTEGER 357 ); 358 359 CREATE TABLE licenses_accepted ( 360 licensename VARCHAR UNIQUE 361 ); 362 363 CREATE TABLE triggers ( 364 idpackage INTEGER PRIMARY KEY, 365 data BLOB 366 ); 367 368 CREATE TABLE entropy_misc_counters ( 369 idtype INTEGER PRIMARY KEY, 370 counter INTEGER 371 ); 372 373 CREATE TABLE categoriesdescription ( 374 category VARCHAR, 375 locale VARCHAR, 376 description VARCHAR 377 ); 378 379 CREATE TABLE packagesets ( 380 setname VARCHAR, 381 dependency VARCHAR 382 ); 383 384 CREATE TABLE packagechangelogs ( 385 category VARCHAR, 386 name VARCHAR, 387 changelog BLOB, 388 PRIMARY KEY (category, name) 389 ); 390 391 CREATE TABLE automergefiles ( 392 idpackage INTEGER, 393 configfile VARCHAR, 394 md5 VARCHAR 395 ); 396 397 CREATE TABLE packagesignatures ( 398 idpackage INTEGER PRIMARY KEY, 399 sha1 VARCHAR, 400 sha256 VARCHAR, 401 sha512 VARCHAR 402 ); 403 404 CREATE TABLE packagespmphases ( 405 idpackage INTEGER PRIMARY KEY, 406 phases VARCHAR 407 ); 408 409 CREATE TABLE entropy_branch_migration ( 410 repository VARCHAR, 411 from_branch VARCHAR, 412 to_branch VARCHAR, 413 post_migration_md5sum VARCHAR, 414 post_upgrade_md5sum VARCHAR, 415 PRIMARY KEY (repository, from_branch, to_branch) 416 ); 417 418 """

419 420 import entropy.tools as entropyTools 421 import entropy.dump as dumpTools 422 import threading

423 - def __init__(self, readOnly = False, noUpload = False, dbFile = None, 424 clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], 425 indexing = True, OutputInterface = None, ServiceInterface = None, 426 skipChecks = False, useBranch = None, lockRemote = True):

427 428 EntropyRepository.__init__(self) 429 430 self.dbname = dbname 431 self.lockRemote = lockRemote 432 if self.dbname == etpConst['clientdbid']: 433 self.db_branch = None 434 if useBranch != None: 435 self.db_branch = useBranch 436 437 if OutputInterface == None: 438 OutputInterface = TextInterface() 439 440 if dbFile == None: 441 raise IncorrectParameter("IncorrectParameter: %s" % ( 442 _("valid database path needed"),) ) 443 444 self.__write_mutex = self.threading.Lock() 445 self.dbapi2 = dbapi2 446 # setup output interface 447 self.OutputInterface = OutputInterface 448 self.updateProgress = self.OutputInterface.updateProgress 449 self.askQuestion = self.OutputInterface.askQuestion 450 # setup service interface 451 self.ServiceInterface = ServiceInterface 452 self.readOnly = readOnly 453 self.noUpload = noUpload 454 self.clientDatabase = clientDatabase 455 self.xcache = xcache 456 self.indexing = indexing 457 self.skipChecks = skipChecks 458 if not self.skipChecks: 459 if not self.entropyTools.is_user_in_entropy_group(): 460 # forcing since we won't have write access to db 461 self.indexing = False 462 # live systems don't like wasting RAM 463 if self.entropyTools.islive(): 464 self.indexing = False 465 self.dbFile = dbFile 466 self.dbclosed = True 467 self.server_repo = None 468 469 if not self.clientDatabase: 470 self.server_repo = self.dbname[len(etpConst['serverdbid']):] 471 self.create_dbstatus_data() 472 473 if not self.skipChecks: 474 # no caching for non root and server connections 475 if (self.dbname.startswith(etpConst['serverdbid'])) or \ 476 (not self.entropyTools.is_user_in_entropy_group()): 477 self.xcache = False 478 self.live_cache = {} 479 480 # create connection 481 self.connection = self.dbapi2.connect(dbFile, timeout=300.0, 482 check_same_thread = False) 483 self.cursor = self.connection.cursor() 484 485 if not self.skipChecks: 486 try: 487 if os.access(self.dbFile, os.W_OK) and \ 488 self.doesTableExist('baseinfo') and \ 489 self.doesTableExist('extrainfo'): 490 491 if self.entropyTools.islive() and etpConst['systemroot']: 492 self.databaseStructureUpdates() 493 else: 494 self.databaseStructureUpdates() 495 except self.dbapi2.Error: 496 self.cursor.close() 497 self.connection.close() 498 raise 499 500 # now we can set this to False 501 self.dbclosed = False

502

503 - def setCacheSize(self, size):

504 self.cursor.execute('PRAGMA cache_size = '+str(size))

505

506 - def setDefaultCacheSize(self, size):

507 self.cursor.execute('PRAGMA default_cache_size = '+str(size))

508 509

510 - def __del__(self):

511 if not self.dbclosed: 512 self.closeDB()

513

514 - def create_dbstatus_data(self):

515 taint_file = self.ServiceInterface.get_local_database_taint_file( 516 self.server_repo) 517 if os.path.isfile(taint_file): 518 dbs = ServerRepositoryStatus() 519 dbs.set_tainted(self.dbFile) 520 dbs.set_bumped(self.dbFile)

521

522 - def closeDB(self):

523 524 self.dbclosed = True 525 526 # if the class is opened readOnly, close and forget 527 if self.readOnly: 528 self.cursor.close() 529 self.connection.close() 530 return 531 532 if self.clientDatabase: 533 self.commitChanges() 534 self.cursor.close() 535 self.connection.close() 536 return 537 538 sts = ServerRepositoryStatus() 539 if not sts.is_tainted(self.dbFile): 540 # we can unlock it, no changes were made 541 self.ServiceInterface.MirrorsService.lock_mirrors(False, 542 repo = self.server_repo) 543 elif not sts.is_unlock_msg(self.dbFile): 544 u_msg = _("Mirrors have not been unlocked. Remember to sync them.") 545 self.updateProgress( 546 darkgreen(u_msg), 547 importance = 1, 548 type = "info", 549 header = brown(" * ") 550 ) 551 sts.set_unlock_msg(self.dbFile) # avoid spamming 552 553 self.commitChanges() 554 self.cursor.close() 555 self.connection.close()

556

557 - def vacuum(self):

558 self.cursor.execute("vacuum")

559

560 - def commitChanges(self):

561 562 if self.readOnly: 563 return 564 565 try: 566 self.connection.commit() 567 except self.dbapi2.Error: 568 pass 569 570 if not self.clientDatabase: 571 self.taintDatabase() 572 dbs = ServerRepositoryStatus() 573 if (dbs.is_tainted(self.dbFile)) and \ 574 (not dbs.is_bumped(self.dbFile)): 575 # bump revision, setting DatabaseBump causes 576 # the session to just bump once 577 dbs.set_bumped(self.dbFile) 578 self.revisionBump()

579

580 - def taintDatabase(self):

581 # if it's equo to open it, this should be avoided 582 if self.clientDatabase: 583 return 584 # taint the database status 585 taint_file = self.ServiceInterface.get_local_database_taint_file( 586 repo = self.server_repo) 587 f = open(taint_file, "w") 588 f.write(etpConst['currentarch']+" database tainted\n") 589 f.flush() 590 f.close() 591 ServerRepositoryStatus().set_tainted(self.dbFile)

592

593 - def untaintDatabase(self):

594 # if it's equo to open it, this should be avoided 595 if self.clientDatabase: 596 return 597 ServerRepositoryStatus().unset_tainted(self.dbFile) 598 # untaint the database status 599 taint_file = self.ServiceInterface.get_local_database_taint_file( 600 repo = self.server_repo) 601 if os.path.isfile(taint_file): 602 os.remove(taint_file)

603

604 - def revisionBump(self):

605 revision_file = self.ServiceInterface.get_local_database_revision_file( 606 repo = self.server_repo) 607 if not os.path.isfile(revision_file): 608 revision = 1 609 else: 610 f = open(revision_file, "r") 611 revision = int(f.readline().strip()) 612 revision += 1 613 f.close() 614 f = open(revision_file, "w") 615 f.write(str(revision)+"\n") 616 f.flush() 617 f.close()

618

619 - def isDatabaseTainted(self):

620 taint_file = self.ServiceInterface.get_local_database_taint_file( 621 repo = self.server_repo) 622 if os.path.isfile(taint_file): 623 return True 624 return False

625 626 # never use this unless you know what you're doing

627 - def initializeDatabase(self):

628 self.checkReadOnly() 629 my = self.Schema() 630 for table in self.listAllTables(): 631 try: 632 self.cursor.execute("DROP TABLE %s" % (table,)) 633 except self.dbapi2.OperationalError: 634 # skip tables that can't be dropped 635 continue 636 self.cursor.executescript(my.get_init()) 637 self.databaseStructureUpdates() 638 # set cache size 639 self.setCacheSize(8192) 640 self.setDefaultCacheSize(8192) 641 self.commitChanges()

642

643 - def checkReadOnly(self):

644 if self.readOnly: 645 raise OperationNotPermitted("OperationNotPermitted: %s." % ( 646 _("can't do that on a readonly database"), 647 ) 648 )

649 650 # check for /usr/portage/profiles/updates changes

651 - def serverUpdatePackagesData(self):

652 653 etpConst['server_treeupdatescalled'].add(self.server_repo) 654 655 repo_updates_file = self.ServiceInterface.get_local_database_treeupdates_file(self.server_repo) 656 doRescan = False 657 658 stored_digest = self.retrieveRepositoryUpdatesDigest(self.server_repo) 659 if stored_digest == -1: 660 doRescan = True 661 662 # check portage files for changes if doRescan is still false 663 portage_dirs_digest = "0" 664 if not doRescan: 665 666 treeupdates_dict = self.ServiceInterface.repository_treeupdate_digests 667 668 if treeupdates_dict.has_key(self.server_repo): 669 portage_dirs_digest = treeupdates_dict.get(self.server_repo) 670 else: 671 672 spm = self.ServiceInterface.SpmService 673 # grab portdir 674 updates_dir = etpConst['systemroot'] + \ 675 spm.get_spm_setting("PORTDIR") + "/profiles/updates" 676 if os.path.isdir(updates_dir): 677 # get checksum 678 mdigest = self.entropyTools.md5sum_directory(updates_dir, 679 get_obj = True) 680 # also checksum etpConst['etpdatabaseupdatefile'] 681 if os.path.isfile(repo_updates_file): 682 f = open(repo_updates_file) 683 block = f.read(1024) 684 while block: 685 mdigest.update(block) 686 block = f.read(1024) 687 f.close() 688 portage_dirs_digest = mdigest.hexdigest() 689 treeupdates_dict[self.server_repo] = portage_dirs_digest 690 del updates_dir 691 692 if doRescan or (str(stored_digest) != str(portage_dirs_digest)): 693 694 # force parameters 695 self.readOnly = False 696 self.noUpload = True 697 698 # reset database tables 699 self.clearTreeupdatesEntries(self.server_repo) 700 701 spm = self.ServiceInterface.SpmService 702 updates_dir = etpConst['systemroot'] + \ 703 spm.get_spm_setting("PORTDIR") + "/profiles/updates" 704 update_files = self.entropyTools.sort_update_files( 705 os.listdir(updates_dir)) 706 update_files = [os.path.join(updates_dir, x) for x in update_files] 707 # now load actions from files 708 update_actions = [] 709 for update_file in update_files: 710 f = open(update_file, "r") 711 mycontent = f.readlines() 712 f.close() 713 lines = [x.strip() for x in mycontent if x.strip()] 714 update_actions.extend(lines) 715 716 # add entropy packages.db.repo_updates content 717 if os.path.isfile(repo_updates_file): 718 f = open(repo_updates_file, "r") 719 mycontent = f.readlines() 720 f.close() 721 lines = [x.strip() for x in mycontent if x.strip() and \ 722 not x.strip().startswith("#")] 723 update_actions.extend(lines) 724 # now filter the required actions 725 update_actions = self.filterTreeUpdatesActions(update_actions) 726 if update_actions: 727 728 mytxt = "%s: %s. %s %s" % ( 729 bold(_("ATTENTION")), 730 red(_("forcing package updates")), 731 red(_("Syncing with")), 732 blue(updates_dir), 733 ) 734 self.updateProgress( 735 mytxt, 736 importance = 1, 737 type = "info", 738 header = brown(" * ") 739 ) 740 # lock database 741 if self.lockRemote: 742 self.ServiceInterface.do_server_repository_sync_lock( 743 self.server_repo, self.noUpload) 744 # now run queue 745 try: 746 self.runTreeUpdatesActions(update_actions) 747 except: 748 # destroy digest 749 self.setRepositoryUpdatesDigest(self.server_repo, "-1") 750 raise 751 752 # store new actions 753 self.addRepositoryUpdatesActions( 754 self.server_repo, update_actions, self.db_branch) 755 756 # store new digest into database 757 self.setRepositoryUpdatesDigest( 758 self.server_repo, portage_dirs_digest) 759 self.commitChanges()

760

761 - def clientUpdatePackagesData(self, clientDbconn, force = False):

762 763 if clientDbconn == None: 764 return 765 766 repository = self.dbname[len(etpConst['dbnamerepoprefix']):] 767 etpConst['client_treeupdatescalled'].add(repository) 768 769 doRescan = False 770 shell_rescan = os.getenv("ETP_TREEUPDATES_RESCAN") 771 if shell_rescan: doRescan = True 772 773 # check database digest 774 stored_digest = self.retrieveRepositoryUpdatesDigest(repository) 775 if stored_digest == -1: 776 doRescan = True 777 778 # check stored value in client database 779 client_digest = "0" 780 if not doRescan: 781 client_digest = clientDbconn.retrieveRepositoryUpdatesDigest( 782 repository) 783 784 if doRescan or (str(stored_digest) != str(client_digest)) or force: 785 786 # reset database tables 787 clientDbconn.clearTreeupdatesEntries(repository) 788 789 # load updates 790 update_actions = self.retrieveTreeUpdatesActions(repository) 791 # now filter the required actions 792 update_actions = clientDbconn.filterTreeUpdatesActions( 793 update_actions) 794 795 if update_actions: 796 797 mytxt = "%s: %s." % ( 798 bold(_("ATTENTION")), 799 red(_("forcing packages metadata update")), 800 ) 801 self.updateProgress( 802 mytxt, 803 importance = 1, 804 type = "info", 805 header = darkred(" * ") 806 ) 807 mytxt = "%s %s." % ( 808 red(_("Updating system database using repository")), 809 blue(repository), 810 ) 811 self.updateProgress( 812 mytxt, 813 importance = 1, 814 type = "info", 815 header = darkred(" * ") 816 ) 817 # run stuff 818 clientDbconn.runTreeUpdatesActions(update_actions) 819 820 # store new digest into database 821 clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest) 822 # store new actions 823 clientDbconn.addRepositoryUpdatesActions(etpConst['clientdbid'], 824 update_actions, self.SystemSettings['repositories']['branch']) 825 clientDbconn.commitChanges() 826 # clear client cache 827 clientDbconn.clearCache() 828 return True

829 830 # this functions will filter either data from /usr/portage/profiles/updates/* 831 # or repository database returning only the needed actions

832 - def filterTreeUpdatesActions(self, actions):

833 new_actions = [] 834 for action in actions: 835 836 if action in new_actions: # skip dupies 837 continue 838 839 doaction = action.split() 840 if doaction[0] == "slotmove": 841 842 # slot move 843 atom = doaction[1] 844 from_slot = doaction[2] 845 to_slot = doaction[3] 846 atom_key = self.entropyTools.dep_getkey(atom) 847 category = atom_key.split("/")[0] 848 matches = self.atomMatch(atom, matchSlot = from_slot, 849 multiMatch = True) 850 found = False 851 # found atoms, check category 852 for idpackage in matches[0]: 853 myslot = self.retrieveSlot(idpackage) 854 mycategory = self.retrieveCategory(idpackage) 855 if mycategory == category: 856 if (myslot != to_slot) and \ 857 (action not in new_actions): 858 new_actions.append(action) 859 found = True 860 break 861 if found: 862 continue 863 # if we get here it means found == False 864 # search into dependencies 865 dep_atoms = self.searchDependency(atom_key, like = True, 866 multi = True, strings = True) 867 dep_atoms = [x for x in dep_atoms if x.endswith(":"+from_slot) \ 868 and self.entropyTools.dep_getkey(x) == atom_key] 869 if dep_atoms: 870 new_actions.append(action) 871 872 elif doaction[0] == "move": 873 atom = doaction[1] # usually a key 874 atom_key = self.entropyTools.dep_getkey(atom) 875 category = atom_key.split("/")[0] 876 matches = self.atomMatch(atom, multiMatch = True) 877 found = False 878 for idpackage in matches[0]: 879 mycategory = self.retrieveCategory(idpackage) 880 if (mycategory == category) and (action \ 881 not in new_actions): 882 new_actions.append(action) 883 found = True 884 break 885 if found: 886 continue 887 # if we get here it means found == False 888 # search into dependencies 889 dep_atoms = self.searchDependency(atom_key, like = True, 890 multi = True, strings = True) 891 dep_atoms = [x for x in dep_atoms if \ 892 self.entropyTools.dep_getkey(x) == atom_key] 893 if dep_atoms: 894 new_actions.append(action) 895 return new_actions

896 897 # this is the place to add extra actions support

898 - def runTreeUpdatesActions(self, actions):

899 900 mytxt = "%s: %s, %s." % ( 901 bold(_("SPM")), 902 blue(_("Running fixpackages")), 903 red(_("it could take a while")), 904 ) 905 self.updateProgress( 906 mytxt, 907 importance = 1, 908 type = "warning", 909 header = darkred(" * ") 910 ) 911 if self.clientDatabase: 912 try: 913 spm = self.ServiceInterface.Spm() 914 spm.run_fixpackages() 915 except: 916 pass 917 else: 918 self.ServiceInterface.SpmService.run_fixpackages() 919 920 spm_moves = set() 921 quickpkg_atoms = set() 922 for action in actions: 923 command = action.split() 924 mytxt = "%s: %s: %s." % ( 925 bold(_("ENTROPY")), 926 red(_("action")), 927 blue(action), 928 ) 929 self.updateProgress( 930 mytxt, 931 importance = 1, 932 type = "warning", 933 header = darkred(" * ") 934 ) 935 if command[0] == "move": 936 spm_moves.add(action) 937 quickpkg_atoms |= self.runTreeUpdatesMoveAction(command[1:], 938 quickpkg_atoms) 939 elif command[0] == "slotmove": 940 quickpkg_atoms |= self.runTreeUpdatesSlotmoveAction(command[1:], 941 quickpkg_atoms) 942 943 mytxt = "%s: %s." % ( 944 bold(_("ENTROPY")), 945 blue(_("package move actions complete")), 946 ) 947 self.updateProgress( 948 mytxt, 949 importance = 1, 950 type = "info", 951 header = purple(" @@ ") 952 ) 953 954 if quickpkg_atoms and not self.clientDatabase: 955 # quickpkg package and packages owning it as a dependency 956 try: 957 self.runTreeUpdatesQuickpkgAction(quickpkg_atoms) 958 except: 959 self.entropyTools.print_traceback() 960 mytxt = "%s: %s: %s, %s." % ( 961 bold(_("WARNING")), 962 red(_("Cannot complete quickpkg for atoms")), 963 blue(str(list(quickpkg_atoms))), 964 _("do it manually"), 965 ) 966 self.updateProgress( 967 mytxt, 968 importance = 1, 969 type = "warning", 970 header = darkred(" * ") 971 ) 972 self.commitChanges() 973 974 if spm_moves: 975 try: 976 self.doTreeupdatesSpmCleanup(spm_moves) 977 except Exception, e: 978 mytxt = "%s: %s: %s, %s." % ( 979 bold(_("WARNING")), 980 red(_("Cannot run SPM cleanup, error")), 981 Exception, 982 e, 983 ) 984 self.entropyTools.print_traceback() 985 986 mytxt = "%s: %s." % ( 987 bold(_("ENTROPY")), 988 blue(_("package moves completed successfully")), 989 ) 990 self.updateProgress( 991 mytxt, 992 importance = 1, 993 type = "info", 994 header = brown(" @@ ") 995 ) 996 997 # discard cache 998 self.clearCache()

999 1000 1001 # -- move action: 1002 # 1) move package key to the new name: category + name + atom 1003 # 2) update all the dependencies in dependenciesreference to the new key 1004 # 3) run fixpackages which will update /var/db/pkg files 1005 # 4) automatically run quickpkg() to build the new binary and 1006 # tainted binaries owning tainted iddependency and taint database

1007 - def runTreeUpdatesMoveAction(self, move_command, quickpkg_queue):

1008 1009 dep_from = move_command[0] 1010 key_from = self.entropyTools.dep_getkey(dep_from) 1011 key_to = move_command[1] 1012 cat_to = key_to.split("/")[0] 1013 name_to = key_to.split("/")[1] 1014 matches = self.atomMatch(dep_from, multiMatch = True) 1015 iddependencies = set() 1016 1017 for idpackage in matches[0]: 1018 1019 slot = self.retrieveSlot(idpackage) 1020 old_atom = self.retrieveAtom(idpackage) 1021 new_atom = old_atom.replace(key_from, key_to) 1022 1023 ### UPDATE DATABASE 1024 # update category 1025 self.setCategory(idpackage, cat_to) 1026 # update name 1027 self.setName(idpackage, name_to) 1028 # update atom 1029 self.setAtom(idpackage, new_atom) 1030 1031 # look for packages we need to quickpkg again 1032 # note: quickpkg_queue is simply ignored if self.clientDatabase 1033 quickpkg_queue.add(key_to+":"+str(slot)) 1034 1035 if not self.clientDatabase: 1036 1037 # check for injection and warn the developer 1038 injected = self.isInjected(idpackage) 1039 if injected: 1040 mytxt = "%s: %s %s. %s !!! %s." % ( 1041 bold(_("INJECT")), 1042 blue(str(new_atom)), 1043 red(_("has been injected")), 1044 red(_("quickpkg manually to update embedded db")), 1045 red(_("Repository database updated anyway")), 1046 ) 1047 self.updateProgress( 1048 mytxt, 1049 importance = 1, 1050 type = "warning", 1051 header = darkred(" * ") 1052 ) 1053 1054 iddeps = self.searchDependency(key_from, like = True, multi = True) 1055 for iddep in iddeps: 1056 # update string 1057 mydep = self.retrieveDependencyFromIddependency(iddep) 1058 mydep_key = self.entropyTools.dep_getkey(mydep) 1059 # avoid changing wrong atoms -> dev-python/qscintilla-python would 1060 # become x11-libs/qscintilla if we don't do this check 1061 if mydep_key != key_from: 1062 continue 1063 mydep = mydep.replace(key_from, key_to) 1064 # now update 1065 # dependstable on server is always re-generated 1066 self.setDependency(iddep, mydep) 1067 # we have to repackage also package owning this iddep 1068 iddependencies |= self.searchIdpackageFromIddependency(iddep) 1069 1070 self.commitChanges() 1071 quickpkg_queue = list(quickpkg_queue) 1072 for x in range(len(quickpkg_queue)): 1073 myatom = quickpkg_queue[x] 1074 myatom = myatom.replace(key_from, key_to) 1075 quickpkg_queue[x] = myatom 1076 quickpkg_queue = set(quickpkg_queue) 1077 for idpackage_owner in iddependencies: 1078 myatom = self.retrieveAtom(idpackage_owner) 1079 myatom = myatom.replace(key_from, key_to) 1080 quickpkg_queue.add(myatom) 1081 return quickpkg_queue

1082 1083 1084 # -- slotmove action: 1085 # 1) move package slot 1086 # 2) update all the dependencies in dependenciesreference owning 1087 # same matched atom + slot 1088 # 3) run fixpackages which will update /var/db/pkg files 1089 # 4) automatically run quickpkg() to build the new 1090 # binary and tainted binaries owning tainted iddependency 1091 # and taint database

1092 - def runTreeUpdatesSlotmoveAction(self, slotmove_command, quickpkg_queue):

1093 1094 atom = slotmove_command[0] 1095 atomkey = self.entropyTools.dep_getkey(atom) 1096 slot_from = slotmove_command[1] 1097 slot_to = slotmove_command[2] 1098 matches = self.atomMatch(atom, multiMatch = True) 1099 iddependencies = set() 1100 1101 matched_idpackages = matches[0] 1102 for idpackage in matched_idpackages: 1103 1104 ### UPDATE DATABASE 1105 # update slot 1106 self.setSlot(idpackage, slot_to) 1107 1108 # look for packages we need to quickpkg again 1109 # note: quickpkg_queue is simply ignored if self.clientDatabase 1110 quickpkg_queue.add(atom+":"+str(slot_to)) 1111 1112 if not self.clientDatabase: 1113 1114 # check for injection and warn the developer 1115 injected = self.isInjected(idpackage) 1116 if injected: 1117 mytxt = "%s: %s %s. %s !!! %s." % ( 1118 bold(_("INJECT")), 1119 blue(str(atom)), 1120 red(_("has been injected")), 1121 red(_("quickpkg manually to update embedded db")), 1122 red(_("Repository database updated anyway")), 1123 ) 1124 self.updateProgress( 1125 mytxt, 1126 importance = 1, 1127 type = "warning", 1128 header = darkred(" * ") 1129 ) 1130 1131 # only if we've found VALID matches ! 1132 iddeps = self.searchDependency(atomkey, like = True, multi = True) 1133 for iddep in iddeps: 1134 # update string 1135 mydep = self.retrieveDependencyFromIddependency(iddep) 1136 mydep_key = self.entropyTools.dep_getkey(mydep) 1137 if mydep_key != atomkey: 1138 continue 1139 if not mydep.endswith(":"+slot_from): # probably slotted dep 1140 continue 1141 mydep_match = self.atomMatch(mydep) 1142 if mydep_match not in matched_idpackages: 1143 continue 1144 mydep = mydep.replace(":"+slot_from, ":"+slot_to) 1145 # now update 1146 # dependstable on server is always re-generated 1147 self.setDependency(iddep, mydep) 1148 # we have to repackage also package owning this iddep 1149 iddependencies |= self.searchIdpackageFromIddependency(iddep) 1150 1151 self.commitChanges() 1152 for idpackage_owner in iddependencies: 1153 myatom = self.retrieveAtom(idpackage_owner) 1154 quickpkg_queue.add(myatom) 1155 return quickpkg_queue

1156

1157 - def runTreeUpdatesQuickpkgAction(self, atoms):

1158 1159 self.commitChanges() 1160 1161 package_paths = set() 1162 runatoms = set() 1163 for myatom in atoms: 1164 mymatch = self.atomMatch(myatom) 1165 if mymatch[0] == -1: 1166 continue 1167 myatom = self.retrieveAtom(mymatch[0]) 1168 runatoms.add(myatom) 1169 1170 for myatom in runatoms: 1171 self.updateProgress( 1172 red("%s: " % (_("repackaging"),) )+blue(myatom), 1173 importance = 1, 1174 type = "warning", 1175 header = blue(" # ") 1176 ) 1177 mydest = self.ServiceInterface.get_local_store_directory( 1178 repo = self.server_repo) 1179 try: 1180 mypath = self.ServiceInterface.quickpkg(myatom, mydest) 1181 except: 1182 # remove broken bin before raising 1183 mypath = os.path.join(mydest, 1184 os.path.basename(myatom)+etpConst['packagesext']) 1185 if os.path.isfile(mypath): 1186 os.remove(mypath) 1187 self.entropyTools.print_traceback() 1188 mytxt = "%s: %s: %s, %s." % ( 1189 bold(_("WARNING")), 1190 red(_("Cannot complete quickpkg for atom")), 1191 blue(myatom), 1192 _("do it manually"), 1193 ) 1194 self.updateProgress( 1195 mytxt, 1196 importance = 1, 1197 type = "warning", 1198 header = darkred(" * ") 1199 ) 1200 continue 1201 package_paths.add(mypath) 1202 packages_data = [(x, False,) for x in package_paths] 1203 idpackages = self.ServiceInterface.add_packages_to_repository( 1204 packages_data, repo = self.server_repo) 1205 1206 if not idpackages: 1207 1208 mytxt = "%s: %s. %s." % ( 1209 bold(_("ATTENTION")), 1210 red(_("runTreeUpdatesQuickpkgAction did not run properly")), 1211 red(_("Please update packages manually")), 1212 ) 1213 self.updateProgress( 1214 mytxt, 1215 importance = 1, 1216 type = "warning", 1217 header = darkred(" * ") 1218 )

1219

1220 - def doTreeupdatesSpmCleanup(self, spm_moves):

1221 1222 # now erase Spm entries if necessary 1223 for action in spm_moves: 1224 command = action.split() 1225 if len(command) < 2: 1226 continue 1227 key = command[1] 1228 name = key.split("/")[1] 1229 if self.clientDatabase: 1230 try: 1231 spm = self.ServiceInterface.Spm() 1232 except: 1233 continue 1234 else: 1235 spm = self.ServiceInterface.SpmService 1236 vdb_path = spm.get_vdb_path() 1237 pkg_path = os.path.join(vdb_path, key.split("/")[0]) 1238 try: 1239 mydirs = [os.path.join(pkg_path, x) for x in \ 1240 os.listdir(pkg_path) if x.startswith(name)] 1241 except OSError: # no dir, no party! 1242 continue 1243 mydirs = [x for x in mydirs if os.path.isdir(x)] 1244 # now move these dirs 1245 for mydir in mydirs: 1246 to_path = os.path.join(etpConst['packagestmpdir'], 1247 os.path.basename(mydir)) 1248 mytxt = "%s: %s '%s' %s '%s'" % ( 1249 bold(_("SPM")), 1250 red(_("Moving old entry")), 1251 blue(mydir), 1252 red(_("to")), 1253 blue(to_path), 1254 ) 1255 self.updateProgress( 1256 mytxt, 1257 importance = 1, 1258 type = "warning", 1259 header = darkred(" * ") 1260 ) 1261 if os.path.isdir(to_path): 1262 shutil.rmtree(to_path, True) 1263 try: 1264 os.rmdir(to_path) 1265 except OSError: 1266 pass 1267 shutil.move(mydir, to_path)

1268 1269

1270 - def handlePackage(self, etpData, forcedRevision = -1, 1271 formattedContent = False):

1272 1273 self.checkReadOnly() 1274 1275 if self.clientDatabase: 1276 return self.addPackage(etpData, revision = forcedRevision, 1277 formatted_content = formattedContent) 1278 1279 # build atom string, server side 1280 pkgatom = self.entropyTools.create_package_atom_string( 1281 etpData['category'], etpData['name'], etpData['version'], 1282 etpData['versiontag']) 1283 foundid = self.isPackageAvailable(pkgatom) 1284 if foundid < 0: # same atom doesn't exist in any branch 1285 return self.addPackage(etpData, revision = forcedRevision, 1286 formatted_content = formattedContent) 1287 1288 idpackage = self.getIDPackage(pkgatom) 1289 curRevision = forcedRevision 1290 if forcedRevision == -1: 1291 curRevision = 0 1292 if idpackage != -1: 1293 curRevision = self.retrieveRevision(idpackage) 1294 1295 # remove old package atom, we do it here because othersie 1296 if idpackage != -1: 1297 # injected packages wouldn't be removed by addPackages 1298 self.removePackage(idpackage) 1299 if forcedRevision == -1: curRevision += 1 1300 1301 # add the new one 1302 return self.addPackage(etpData, revision = curRevision, 1303 formatted_content = formattedContent)

1304

1305 - def retrieve_packages_to_remove(self, name, category, slot, injected):

1306 1307 removelist = set() 1308 1309 # support for expiration-based packages handling, also internally 1310 # called Fat Scope. 1311 filter_similar = False 1312 srv_ss_plg = etpConst['system_settings_plugins_ids']['server_plugin'] 1313 srv_ss_fs_plg = \ 1314 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 1315 1316 if not self.clientDatabase: # server-side db 1317 srv_plug_settings = self.SystemSettings.get(srv_ss_plg) 1318 if srv_plug_settings != None: 1319 if srv_plug_settings['server']['exp_based_scope']: 1320 # in case support is enabled, return an empty set 1321 filter_similar = True 1322 1323 searchsimilar = self.searchPackagesByNameAndCategory( 1324 name = name, 1325 category = category, 1326 sensitive = True 1327 ) 1328 if filter_similar: 1329 # filter out packages in the same scope that are allowed to stay 1330 idpkgs = self.SystemSettings[srv_ss_fs_plg]['repos'].get( 1331 self.server_repo) 1332 if idpkgs: 1333 if -1 in idpkgs: 1334 del searchsimilar[:] 1335 else: 1336 searchsimilar = [x for x in searchsimilar if x[1] \ 1337 not in idpkgs] 1338 1339 if not injected: 1340 # read: if package has been injected, we'll skip 1341 # the removal of packages in the same slot, 1342 # usually used server side btw 1343 for atom, idpackage in searchsimilar: 1344 # get the package slot 1345 myslot = self.retrieveSlot(idpackage) 1346 # we merely ignore packages with 1347 # negative counters, since they're the injected ones 1348 if self.isInjected(idpackage): continue 1349 if slot == myslot: 1350 # remove! 1351 removelist.add(idpackage) 1352 1353 return removelist

1354

1355 - def addPackage(self, etpData, revision = -1, idpackage = None, 1356 do_remove = True, do_commit = True, formatted_content = False):

1357 1358 self.checkReadOnly() 1359 1360 if revision == -1: 1361 try: 1362 revision = int(etpData['revision']) 1363 except (KeyError, ValueError): 1364 etpData['revision'] = 0 # revision not specified 1365 revision = 0 1366 elif not etpData.has_key('revision'): 1367 etpData['revision'] = revision 1368 1369 manual_deps = set() 1370 if do_remove: 1371 removelist = self.retrieve_packages_to_remove( 1372 etpData['name'], 1373 etpData['category'], 1374 etpData['slot'], 1375 etpData['injected'] 1376 ) 1377 for r_idpackage in removelist: 1378 manual_deps |= self.retrieveManualDependencies(r_idpackage) 1379 self.removePackage(r_idpackage, do_cleanup = False, 1380 do_commit = False) 1381 1382 # create new category if it doesn't exist 1383 catid = self.isCategoryAvailable(etpData['category']) 1384 if catid == -1: catid = self.addCategory(etpData['category']) 1385 1386 # create new license if it doesn't exist 1387 licid = self.isLicenseAvailable(etpData['license']) 1388 if licid == -1: licid = self.addLicense(etpData['license']) 1389 1390 idprotect = self.isProtectAvailable(etpData['config_protect']) 1391 if idprotect == -1: idprotect = self.addProtect( 1392 etpData['config_protect']) 1393 1394 idprotect_mask = self.isProtectAvailable( 1395 etpData['config_protect_mask']) 1396 if idprotect_mask == -1: idprotect_mask = self.addProtect( 1397 etpData['config_protect_mask']) 1398 1399 idflags = self.areCompileFlagsAvailable( 1400 etpData['chost'],etpData['cflags'],etpData['cxxflags']) 1401 if idflags == -1: idflags = self.addCompileFlags( 1402 etpData['chost'],etpData['cflags'],etpData['cxxflags']) 1403 1404 1405 trigger = 0 1406 if etpData['trigger']: 1407 trigger = 1 1408 1409 # baseinfo 1410 pkgatom = self.entropyTools.create_package_atom_string( 1411 etpData['category'], etpData['name'], etpData['version'], 1412 etpData['versiontag']) 1413 # add atom metadatum 1414 etpData['atom'] = pkgatom 1415 1416 mybaseinfo_data = (pkgatom, catid, etpData['name'], etpData['version'], 1417 etpData['versiontag'], revision, etpData['branch'], etpData['slot'], 1418 licid, etpData['etpapi'], trigger,) 1419 1420 myidpackage_string = 'NULL' 1421 if isinstance(idpackage, int): 1422 manual_deps |= self.retrieveManualDependencies(idpackage) 1423 # does it exist? 1424 self.removePackage(idpackage, do_cleanup = False, 1425 do_commit = False, do_rss = False) 1426 myidpackage_string = '?' 1427 mybaseinfo_data = (idpackage,)+mybaseinfo_data 1428 else: 1429 idpackage = None 1430 1431 # merge old manual dependencies 1432 for manual_dep in manual_deps: 1433 if manual_dep in etpData['dependencies']: continue 1434 etpData['dependencies'][manual_dep] = etpConst['spm']['mdepend_id'] 1435 1436 with self.__write_mutex: 1437 1438 self.cursor.execute(""" 1439 INSERT INTO baseinfo VALUES (%s,?,?,?,?,?,?,?,?,?,?,?)""" % ( 1440 myidpackage_string,), mybaseinfo_data) 1441 if idpackage == None: 1442 idpackage = self.cursor.lastrowid 1443 1444 # extrainfo 1445 self.cursor.execute( 1446 'INSERT INTO extrainfo VALUES (?,?,?,?,?,?,?,?)', 1447 ( idpackage, 1448 etpData['description'], 1449 etpData['homepage'], 1450 etpData['download'], 1451 etpData['size'], 1452 idflags, 1453 etpData['digest'], 1454 etpData['datecreation'], 1455 ) 1456 ) 1457 ### other information iserted below are not as critical as these above 1458 1459 # tables using a select 1460 self.insertEclasses(idpackage, etpData['eclasses']) 1461 self.insertNeeded(idpackage, etpData['needed']) 1462 self.insertDependencies(idpackage, etpData['dependencies']) 1463 self.insertSources(idpackage, etpData['sources']) 1464 self.insertUseflags(idpackage, etpData['useflags']) 1465 self.insertKeywords(idpackage, etpData['keywords']) 1466 self.insertLicenses(etpData['licensedata']) 1467 self.insertMirrors(etpData['mirrorlinks']) 1468 # package ChangeLog 1469 if etpData.get('changelog'): 1470 self.insertChangelog(etpData['category'], etpData['name'], 1471 etpData['changelog']) 1472 # package signatures 1473 if etpData.get('signatures'): 1474 self.insertSignatures(idpackage, etpData['signatures']) 1475 # needed libraries paths 1476 if etpData.get('needed_paths'): 1477 for lib in sorted(etpData['needed_paths']): 1478 self.insertNeededPaths(lib, etpData['needed_paths'][lib]) 1479 1480 # spm phases 1481 if etpData.get('spm_phases') != None: 1482 self.insertSpmPhases(idpackage, etpData['spm_phases']) 1483 1484 # not depending on other tables == no select done 1485 self.insertContent(idpackage, etpData['content'], 1486 already_formatted = formatted_content) 1487 etpData['counter'] = int(etpData['counter']) # cast to integer 1488 etpData['counter'] = self.insertPortageCounter( 1489 idpackage, 1490 etpData['counter'], 1491 etpData['branch'], 1492 etpData['injected'] 1493 ) 1494 self.insertOnDiskSize(idpackage, etpData['disksize']) 1495 if etpData['trigger']: 1496 self.insertTrigger(idpackage, etpData['trigger']) 1497 self.insertConflicts(idpackage, etpData['conflicts']) 1498 self.insertProvide(idpackage, etpData['provide']) 1499 self.insertMessages(idpackage, etpData['messages']) 1500 self.insertConfigProtect(idpackage, idprotect) 1501 self.insertConfigProtect(idpackage, idprotect_mask, mask = True) 1502 # injected? 1503 if etpData.get('injected'): 1504 self.setInjected(idpackage, do_commit = False) 1505 # is it a system package? 1506 if etpData.get('systempackage'): 1507 self.setSystemPackage(idpackage, do_commit = False) 1508 1509 self.clearCache() # we do live_cache.clear() here too 1510 if do_commit: 1511 self.commitChanges() 1512 1513 ### RSS Atom support 1514 ### dictionary will be elaborated by activator 1515 if self.SystemSettings.has_key(self.srv_sys_settings_plugin): 1516 if self.SystemSettings[self.srv_sys_settings_plugin]['server']['rss']['enabled'] and \ 1517 not self.clientDatabase: 1518 1519 self._write_rss_for_added_package(pkgatom, revision, 1520 etpData['description'], etpData['homepage']) 1521 1522 # Update category description 1523 if not self.clientDatabase: 1524 mycategory = etpData['category'] 1525 descdata = {} 1526 try: 1527 descdata = self.get_category_description_from_disk(mycategory) 1528 except (IOError, OSError, EOFError,): 1529 pass 1530 if descdata: 1531 self.setCategoryDescription(mycategory, descdata) 1532 1533 return idpackage, revision, etpData

1534

1535 - def _write_rss_for_added_package(self, pkgatom, revision, description, 1536 homepage):

1537 1538 rssAtom = pkgatom+"~"+str(revision) 1539 rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name']) 1540 if rssObj: 1541 self.ServiceInterface.rssMessages = rssObj.copy() 1542 if not isinstance(self.ServiceInterface.rssMessages, dict): 1543 self.ServiceInterface.rssMessages = {} 1544 if not self.ServiceInterface.rssMessages.has_key('added'): 1545 self.ServiceInterface.rssMessages['added'] = {} 1546 if not self.ServiceInterface.rssMessages.has_key('removed'): 1547 self.ServiceInterface.rssMessages['removed'] = {} 1548 if rssAtom in self.ServiceInterface.rssMessages['removed']: 1549 del self.ServiceInterface.rssMessages['removed'][rssAtom] 1550 1551 self.ServiceInterface.rssMessages['added'][rssAtom] = {} 1552 self.ServiceInterface.rssMessages['added'][rssAtom]['description'] = \ 1553 description 1554 self.ServiceInterface.rssMessages['added'][rssAtom]['homepage'] = \ 1555 homepage 1556 self.ServiceInterface.rssMessages['light'][rssAtom] = {} 1557 self.ServiceInterface.rssMessages['light'][rssAtom]['description'] = \ 1558 description 1559 self.dumpTools.dumpobj(etpConst['rss-dump-name'], 1560 self.ServiceInterface.rssMessages)

1561

1562 - def _write_rss_for_removed_package(self, idpackage):

1563 1564 rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name']) 1565 if rssObj: 1566 self.ServiceInterface.rssMessages = rssObj.copy() 1567 rssAtom = self.retrieveAtom(idpackage) 1568 rssRevision = self.retrieveRevision(idpackage) 1569 rssAtom += "~"+str(rssRevision) 1570 if not isinstance(self.ServiceInterface.rssMessages, dict): 1571 self.ServiceInterface.rssMessages = {} 1572 if not self.ServiceInterface.rssMessages.has_key('added'): 1573 self.ServiceInterface.rssMessages['added'] = {} 1574 if not self.ServiceInterface.rssMessages.has_key('removed'): 1575 self.ServiceInterface.rssMessages['removed'] = {} 1576 if rssAtom in self.ServiceInterface.rssMessages['added']: 1577 del self.ServiceInterface.rssMessages['added'][rssAtom] 1578 if rssAtom in self.ServiceInterface.rssMessages.get('light', []): 1579 del self.ServiceInterface.rssMessages['light'][rssAtom] 1580 1581 mydict = {} 1582 try: 1583 mydict['description'] = self.retrieveDescription(idpackage) 1584 except TypeError: 1585 mydict['description'] = "N/A" 1586 try: 1587 mydict['homepage'] = self.retrieveHomepage(idpackage) 1588 except TypeError: 1589 mydict['homepage'] = "" 1590 1591 self.dumpTools.dumpobj(etpConst['rss-dump-name'], 1592 self.ServiceInterface.rssMessages) 1593 1594 self.ServiceInterface.rssMessages['removed'][rssAtom] = mydict

1595

1596 - def removePackage(self, idpackage, do_cleanup = True, do_commit = True, 1597 do_rss = True):

1598 1599 self.checkReadOnly() 1600 # clear caches 1601 self.clearCache() 1602 1603 ### RSS Atom support 1604 ### dictionary will be elaborated by activator 1605 if self.SystemSettings.has_key(self.srv_sys_settings_plugin): 1606 if self.SystemSettings[self.srv_sys_settings_plugin]['server']['rss']['enabled'] and \ 1607 (not self.clientDatabase) and do_rss: 1608 1609 # store addPackage action 1610 self._write_rss_for_removed_package(idpackage) 1611 1612 with self.__write_mutex: 1613 1614 r_tup = (idpackage,)*20 1615 self.cursor.executescript(""" 1616 DELETE FROM baseinfo WHERE idpackage = %d; 1617 DELETE FROM extrainfo WHERE idpackage = %d; 1618 DELETE FROM dependencies WHERE idpackage = %d; 1619 DELETE FROM provide WHERE idpackage = %d; 1620 DELETE FROM conflicts WHERE idpackage = %d; 1621 DELETE FROM configprotect WHERE idpackage = %d; 1622 DELETE FROM configprotectmask WHERE idpackage = %d; 1623 DELETE FROM sources WHERE idpackage = %d; 1624 DELETE FROM useflags WHERE idpackage = %d; 1625 DELETE FROM keywords WHERE idpackage = %d; 1626 DELETE FROM content WHERE idpackage = %d; 1627 DELETE FROM messages WHERE idpackage = %d; 1628 DELETE FROM counters WHERE idpackage = %d; 1629 DELETE FROM sizes WHERE idpackage = %d; 1630 DELETE FROM eclasses WHERE idpackage = %d; 1631 DELETE FROM needed WHERE idpackage = %d; 1632 DELETE FROM triggers WHERE idpackage = %d; 1633 DELETE FROM systempackages WHERE idpackage = %d; 1634 DELETE FROM injected WHERE idpackage = %d; 1635 DELETE FROM installedtable WHERE idpackage = %d; 1636 """ % r_tup) 1637 1638 # not yet possible to add these calls above 1639 try: 1640 self.removeAutomergefiles(idpackage) 1641 except self.dbapi2.OperationalError: 1642 pass 1643 try: 1644 self.removeSignatures(idpackage) 1645 except self.dbapi2.OperationalError: 1646 pass 1647 try: 1648 self.removeSpmPhases(idpackage) 1649 except self.dbapi2.OperationalError: 1650 pass 1651 1652 # Remove from dependstable if exists 1653 self.removePackageFromDependsTable(idpackage) 1654 1655 if do_cleanup: 1656 # Cleanups if at least one package has been removed 1657 self.doCleanups() 1658 1659 if do_commit: 1660 self.commitChanges()

1661

1662 - def removeMirrorEntries(self, mirrorname):

1663 with self.__write_mutex: 1664 self.cursor.execute(""" 1665 DELETE FROM mirrorlinks WHERE mirrorname = (?) 1666 """,(mirrorname,))

1667

1668 - def addMirrors(self, mirrorname, mirrorlist):

1669 with self.__write_mutex: 1670 data = [(mirrorname, x,) for x in mirrorlist] 1671 self.cursor.executemany(""" 1672 INSERT into mirrorlinks VALUES (?,?) 1673 """, data)

1674

1675 - def addCategory(self, category):

1676 with self.__write_mutex: 1677 self.cursor.execute(""" 1678 INSERT into categories VALUES (NULL,?) 1679 """, (category,)) 1680 return self.cursor.lastrowid

1681

1682 - def addProtect(self, protect):

1683 with self.__write_mutex: 1684 self.cursor.execute(""" 1685 INSERT into configprotectreference VALUES (NULL,?) 1686 """, (protect,)) 1687 return self.cursor.lastrowid

1688 1689

1690 - def addSource(self, source):

1691 with self.__write_mutex: 1692 self.cursor.execute(""" 1693 INSERT into sourcesreference VALUES (NULL,?) 1694 """, (source,)) 1695 return self.cursor.lastrowid

1696

1697 - def addDependency(self, dependency):

1698 with self.__write_mutex: 1699 self.cursor.execute('INSERT into dependenciesreference VALUES (NULL,?)', (dependency,)) 1700 return self.cursor.lastrowid

1701

1702 - def addKeyword(self, keyword):

1703 with self.__write_mutex: 1704 self.cursor.execute('INSERT into keywordsreference VALUES (NULL,?)', (keyword,)) 1705 return self.cursor.lastrowid

1706

1707 - def addUseflag(self, useflag):

1708 with self.__write_mutex: 1709 self.cursor.execute('INSERT into useflagsreference VALUES (NULL,?)', (useflag,)) 1710 return self.cursor.lastrowid

1711

1712 - def addEclass(self, eclass):

1713 with self.__write_mutex: 1714 self.cursor.execute('INSERT into eclassesreference VALUES (NULL,?)', (eclass,)) 1715 return self.cursor.lastrowid

1716

1717 - def addNeeded(self, needed):

1718 with self.__write_mutex: 1719 self.cursor.execute('INSERT into neededreference VALUES (NULL,?)', (needed,)) 1720 return self.cursor.lastrowid

1721

1722 - def addLicense(self, pkglicense):

1723 if not self.entropyTools.is_valid_string(pkglicense): 1724 pkglicense = ' ' # workaround for broken license entries 1725 with self.__write_mutex: 1726 self.cursor.execute('INSERT into licenses VALUES (NULL,?)', (pkglicense,)) 1727 return self.cursor.lastrowid

1728

1729 - def addCompileFlags(self, chost, cflags, cxxflags):

1730 with self.__write_mutex: 1731 self.cursor.execute('INSERT into flags VALUES (NULL,?,?,?)', (chost,cflags,cxxflags,)) 1732 return self.cursor.lastrowid

1733

1734 - def setSystemPackage(self, idpackage, do_commit = True):

1735 with self.__write_mutex: 1736 self.cursor.execute('INSERT into systempackages VALUES (?)', (idpackage,)) 1737 if do_commit: 1738 self.commitChanges()

1739

1740 - def setInjected(self, idpackage, do_commit = True):

1741 with self.__write_mutex: 1742 if not self.isInjected(idpackage): 1743 self.cursor.execute('INSERT into injected VALUES (?)', (idpackage,)) 1744 if do_commit: 1745 self.commitChanges()

1746 1747 # date expressed the unix way

1748 - def setDateCreation(self, idpackage, date):

1749 with self.__write_mutex: 1750 self.cursor.execute('UPDATE extrainfo SET datecreation = (?) WHERE idpackage = (?)', (str(date), idpackage,)) 1751 self.commitChanges()

1752

1753 - def setDigest(self, idpackage, digest):

1754 with self.__write_mutex: 1755 self.cursor.execute('UPDATE extrainfo SET digest = (?) WHERE idpackage = (?)', (digest, idpackage,)) 1756 self.commitChanges()

1757

1758 - def setSignatures(self, idpackage, signatures):

1759 with self.__write_mutex: 1760 sha1, sha256, sha512 = signatures['sha1'], signatures['sha256'], \ 1761 signatures['sha512'] 1762 self.cursor.execute(""" 1763 UPDATE packagesignatures SET sha1 = (?), sha256 = (?), sha512 = (?) 1764 WHERE idpackage = (?) 1765 """, (sha1, sha256, sha512, idpackage))

1766

1767 - def setDownloadURL(self, idpackage, url):

1768 with self.__write_mutex: 1769 self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (url, idpackage,)) 1770 self.commitChanges()

1771

1772 - def setCategory(self, idpackage, category):

1773 # create new category if it doesn't exist 1774 catid = self.isCategoryAvailable(category) 1775 if (catid == -1): 1776 # create category 1777 catid = self.addCategory(category) 1778 with self.__write_mutex: 1779 self.cursor.execute('UPDATE baseinfo SET idcategory = (?) WHERE idpackage = (?)', (catid, idpackage,)) 1780 self.commitChanges()

1781

1782 - def setCategoryDescription(self, category, description_data):

1783 with self.__write_mutex: 1784 self.cursor.execute('DELETE FROM categoriesdescription WHERE category = (?)', (category,)) 1785 for locale in description_data: 1786 mydesc = description_data[locale] 1787 #if type(mydesc) is unicode: 1788 # mydesc = mydesc.encode('raw_unicode_escape') 1789 self.cursor.execute('INSERT INTO categoriesdescription VALUES (?,?,?)', (category, locale, mydesc,)) 1790 self.commitChanges()

1791

1792 - def setName(self, idpackage, name):

1793 with self.__write_mutex: 1794 self.cursor.execute('UPDATE baseinfo SET name = (?) WHERE idpackage = (?)', (name, idpackage,)) 1795 self.commitChanges()

1796

1797 - def setDependency(self, iddependency, dependency):

1798 with self.__write_mutex: 1799 self.cursor.execute('UPDATE dependenciesreference SET dependency = (?) WHERE iddependency = (?)', (dependency, iddependency,)) 1800 self.commitChanges()

1801

1802 - def setAtom(self, idpackage, atom):

1803 with self.__write_mutex: 1804 self.cursor.execute('UPDATE baseinfo SET atom = (?) WHERE idpackage = (?)', (atom, idpackage,)) 1805 self.commitChanges()

1806

1807 - def setSlot(self, idpackage, slot):

1808 with self.__write_mutex: 1809 self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot, idpackage,)) 1810 self.commitChanges()

1811

1812 - def removeLicensedata(self, license_name):

1813 with self.__write_mutex: 1814 self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,))

1815

1816 - def removeDependencies(self, idpackage):

1817 with self.__write_mutex: 1818 self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,)) 1819 self.commitChanges()

1820

1821 - def insertDependencies(self, idpackage, depdata):

1822 1823 dcache = set() 1824 add_dep = self.addDependency 1825 is_dep_avail = self.isDependencyAvailable 1826 def mymf(dep): 1827 1828 if dep in dcache: return 0 1829 iddep = is_dep_avail(dep) 1830 if iddep == -1: iddep = add_dep(dep) 1831 1832 deptype = 0 1833 if isinstance(depdata, dict): 1834 deptype = depdata[dep] 1835 1836 dcache.add(dep) 1837 return (idpackage, iddep, deptype,)

1838 1839 # do not place inside the with statement, otherwise there'll be an obvious lockup 1840 deps = [x for x in map(mymf, depdata) if type(x) != int] 1841 with self.__write_mutex: 1842 self.cursor.executemany('INSERT into dependencies VALUES (?,?,?)', deps)

1843

1844 - def insertManualDependencies(self, idpackage, manual_deps):

1845 mydict = {} 1846 for manual_dep in manual_deps: 1847 mydict[manual_dep] = etpConst['spm']['mdepend_id'] 1848 return self.insertDependencies(idpackage, mydict)

1849

1850 - def removeContent(self, idpackage):

1851 with self.__write_mutex: 1852 self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,)) 1853 self.commitChanges()

1854

1855 - def insertContent(self, idpackage, content, already_formatted = False):

1856 1857 with self.__write_mutex: 1858 if already_formatted: 1859 self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',((idpackage, x, y,) for a, x, y in content)) 1860 return 1861 def my_cmap(xfile): 1862 return (idpackage, xfile, content[xfile],)

1863 self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',map(my_cmap, content)) 1864

1865 - def insertNeededPaths(self, library, paths):

1866 with self.__write_mutex: 1867 self.cursor.executemany('INSERT OR IGNORE INTO neededlibrarypaths VALUES (?,?,?)', 1868 ((library, path, elfclass,) for path, elfclass in paths))

1869

1870 - def insertAutomergefiles(self, idpackage, automerge_data):

1871 with self.__write_mutex: 1872 self.cursor.executemany('INSERT INTO automergefiles VALUES (?,?,?)', 1873 ((idpackage, x, y,) for x, y in automerge_data))

1874

1875 - def removeAutomergefiles(self, idpackage):

1876 with self.__write_mutex: 1877 self.cursor.execute('DELETE FROM automergefiles WHERE idpackage = (?)', (idpackage,))

1878

1879 - def removeSignatures(self, idpackage):

1880 with self.__write_mutex: 1881 self.cursor.execute('DELETE FROM packagesignatures WHERE idpackage = (?)', (idpackage,))

1882

1883 - def removeSpmPhases(self, idpackage):

1884 with self.__write_mutex: 1885 self.cursor.execute('DELETE FROM packagespmphases WHERE idpackage = (?)', (idpackage,))

1886

1887 - def insertChangelog(self, category, name, changelog_txt):

1888 with self.__write_mutex: 1889 mytxt = changelog_txt.encode('raw_unicode_escape') 1890 self.cursor.execute('DELETE FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,)) 1891 self.cursor.execute('INSERT INTO packagechangelogs VALUES (?,?,?)', (category, name, buffer(mytxt),))

1892

1893 - def removeChangelog(self, category, name):

1894 with self.__write_mutex: 1895 self.cursor.execute('DELETE FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,))

1896

1897 - def insertLicenses(self, licenses_data):

1898 1899 mylicenses = licenses_data.keys() 1900 is_lic_avail = self.isLicensedataKeyAvailable 1901 def my_mf(mylicense): 1902 return not is_lic_avail(mylicense)

1903 1904 def my_mm(mylicense): 1905 lic_data = licenses_data.get(mylicense,'') 1906 # support both utf8 and str input 1907 if isinstance(lic_data, unicode): # encode to str 1908 try: 1909 lic_data = lic_data.encode('raw_unicode_escape') 1910 except (UnicodeDecodeError,): 1911 lic_data = lic_data.encode('utf-8') 1912 return (mylicense, buffer(lic_data), 0,) 1913 1914 with self.__write_mutex: 1915 self.cursor.executemany('INSERT into licensedata VALUES (?,?,?)',map(my_mm, list(set(filter(my_mf, mylicenses))))) 1916

1917 - def insertConfigProtect(self, idpackage, idprotect, mask = False):

1918 1919 mytable = 'configprotect' 1920 if mask: mytable += 'mask' 1921 with self.__write_mutex: 1922 self.cursor.execute('INSERT into %s VALUES (?,?)' % (mytable,), (idpackage, idprotect,))

1923

1924 - def insertMirrors(self, mirrors):

1925 1926 for mirrorname, mirrorlist in mirrors: 1927 # remove old 1928 self.removeMirrorEntries(mirrorname) 1929 # add new 1930 self.addMirrors(mirrorname, mirrorlist)

1931

1932 - def insertKeywords(self, idpackage, keywords):

1933 1934 mydata = set() 1935 for key in keywords: 1936 idkeyword = self.isKeywordAvailable(key) 1937 if (idkeyword == -1): 1938 # create category 1939 idkeyword = self.addKeyword(key) 1940 mydata.add((idpackage, idkeyword,)) 1941 1942 with self.__write_mutex: 1943 self.cursor.executemany('INSERT into keywords VALUES (?,?)', mydata)

1944

1945 - def insertUseflags(self, idpackage, useflags):

1946 1947 mydata = set() 1948 for flag in useflags: 1949 iduseflag = self.isUseflagAvailable(flag) 1950 if (iduseflag == -1): 1951 # create category 1952 iduseflag = self.addUseflag(flag) 1953 mydata.add((idpackage, iduseflag,)) 1954 1955 with self.__write_mutex: 1956 self.cursor.executemany('INSERT into useflags VALUES (?,?)', mydata)

1957

1958 - def insertSignatures(self, idpackage, signatures):

1959 with self.__write_mutex: 1960 sha1, sha256, sha512 = signatures['sha1'], signatures['sha256'], \ 1961 signatures['sha512'] 1962 self.cursor.execute(""" 1963 INSERT INTO packagesignatures VALUES (?,?,?,?) 1964 """, (idpackage, sha1, sha256, sha512))

1965

1966 - def insertSpmPhases(self, idpackage, phases):

1967 with self.__write_mutex: 1968 self.cursor.execute(""" 1969 INSERT INTO packagespmphases VALUES (?,?) 1970 """, (idpackage,phases,))

1971

1972 - def insertSources(self, idpackage, sources):

1973 1974 mydata = set() 1975 for source in sources: 1976 if (not source) or (source == "") or (not self.entropyTools.is_valid_string(source)): 1977 continue 1978 idsource = self.isSourceAvailable(source) 1979 if (idsource == -1): 1980 # create category 1981 idsource = self.addSource(source) 1982 mydata.add((idpackage, idsource,)) 1983 1984 with self.__write_mutex: 1985 self.cursor.executemany('INSERT into sources VALUES (?,?)', mydata)

1986

1987 - def insertConflicts(self, idpackage, conflicts):

1988 1989 def myiter(): 1990 for conflict in conflicts: 1991 yield (idpackage, conflict,)

1992 1993 with self.__write_mutex: 1994 self.cursor.executemany('INSERT into conflicts VALUES (?,?)', myiter()) 1995

1996 - def insertMessages(self, idpackage, messages):

1997 1998 def myiter(): 1999 for message in messages: 2000 yield (idpackage, message,)

2001 2002 with self.__write_mutex: 2003 self.cursor.executemany('INSERT into messages VALUES (?,?)', myiter()) 2004

2005 - def insertProvide(self, idpackage, provides):

2006 2007 def myiter(): 2008 for atom in provides: 2009 yield (idpackage, atom,)

2010 2011 with self.__write_mutex: 2012 self.cursor.executemany('INSERT into provide VALUES (?,?)', myiter()) 2013

2014 - def insertNeeded(self, idpackage, neededs):

2015 2016 mydata = set() 2017 for needed, elfclass in neededs: 2018 idneeded = self.isNeededAvailable(needed) 2019 if idneeded == -1: 2020 # create eclass 2021 idneeded = self.addNeeded(needed) 2022 mydata.add((idpackage, idneeded, elfclass)) 2023 2024 with self.__write_mutex: 2025 self.cursor.executemany('INSERT into needed VALUES (?,?,?)', mydata)

2026

2027 - def insertEclasses(self, idpackage, eclasses):

2028 2029 mydata = set() 2030 for eclass in eclasses: 2031 idclass = self.isEclassAvailable(eclass) 2032 if (idclass == -1): 2033 # create eclass 2034 idclass = self.addEclass(eclass) 2035 mydata.add((idpackage, idclass,)) 2036 2037 with self.__write_mutex: 2038 self.cursor.executemany('INSERT into eclasses VALUES (?,?)', mydata)

2039

2040 - def insertOnDiskSize(self, idpackage, mysize):

2041 with self.__write_mutex: 2042 self.cursor.execute('INSERT into sizes VALUES (?,?)', (idpackage, mysize,))

2043

2044 - def insertTrigger(self, idpackage, trigger):

2045 with self.__write_mutex: 2046 self.cursor.execute('INSERT into triggers VALUES (?,?)', (idpackage, buffer(trigger),))

2047

2048 - def insertBranchMigration(self, repository, from_branch, to_branch, 2049 post_migration_md5sum, post_upgrade_md5sum):

2050 with self.__write_mutex: 2051 self.cursor.execute(""" 2052 INSERT OR REPLACE INTO entropy_branch_migration VALUES (?,?,?,?,?) 2053 """, ( 2054 repository, from_branch, 2055 to_branch, post_migration_md5sum, 2056 post_upgrade_md5sum, 2057 ) 2058 )

2059

2060 - def setBranchMigrationPostUpgradeMd5sum(self, repository, from_branch, 2061 to_branch, post_upgrade_md5sum):

2062 with self.__write_mutex: 2063 self.cursor.execute(""" 2064 UPDATE entropy_branch_migration SET post_upgrade_md5sum = (?) WHERE 2065 repository = (?) AND from_branch = (?) AND to_branch = (?) 2066 """, (post_upgrade_md5sum, repository, from_branch, to_branch,))

2067 2068

2069 - def insertPortageCounter(self, idpackage, counter, branch, injected):

2070 2071 if (counter != -1) and not injected: 2072 2073 if counter <= -2: 2074 # special cases 2075 counter = self.getNewNegativeCounter() 2076 2077 with self.__write_mutex: 2078 try: 2079 self.cursor.execute( 2080 'INSERT into counters VALUES ' 2081 '(?,?,?)' 2082 , ( counter, 2083 idpackage, 2084 branch, 2085 ) 2086 ) 2087 except self.dbapi2.IntegrityError: 2088 # we have a PRIMARY KEY we need to remove 2089 self._migrateCountersTable() 2090 self.cursor.execute( 2091 'INSERT into counters VALUES ' 2092 '(?,?,?)' 2093 , ( counter, 2094 idpackage, 2095 branch, 2096 ) 2097 ) 2098 except: 2099 if self.dbname == etpConst['clientdbid']: 2100 # force only for client database 2101 if self.doesTableExist("counters"): 2102 raise 2103 self.cursor.execute( 2104 'INSERT into counters VALUES ' 2105 '(?,?,?)' 2106 , ( counter, 2107 idpackage, 2108 branch, 2109 ) 2110 ) 2111 elif self.dbname.startswith(etpConst['serverdbid']): 2112 raise 2113 2114 return counter

2115

2116 - def insertCounter(self, idpackage, counter, branch = None):

2117 if not branch: branch = self.db_branch 2118 if not branch: branch = self.SystemSettings['repositories']['branch'] 2119 with self.__write_mutex: 2120 self.cursor.execute(""" 2121 DELETE FROM counters 2122 WHERE (counter = (?) OR 2123 idpackage = (?)) AND 2124 branch = (?)""", (counter, idpackage, branch,)) 2125 self.cursor.execute('INSERT INTO counters VALUES (?,?,?)', (counter, idpackage, branch,)) 2126 self.commitChanges()

2127

2128 - def setTrashedCounter(self, counter):

2129 with self.__write_mutex: 2130 self.cursor.execute('DELETE FROM trashedcounters WHERE counter = (?)', (counter,)) 2131 self.cursor.execute('INSERT INTO trashedcounters VALUES (?)', (counter,)) 2132 self.commitChanges()

2133

2134 - def setCounter(self, idpackage, counter, branch = None):

2135 2136 branchstring = '' 2137 insertdata = [counter, idpackage] 2138 if branch: 2139 branchstring = ', branch = (?)' 2140 insertdata.insert(1, branch) 2141 2142 with self.__write_mutex: 2143 try: 2144 self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata) 2145 self.commitChanges() 2146 except: 2147 if self.dbname == etpConst['clientdbid']: 2148 raise

2149

2150 - def contentDiff(self, idpackage, dbconn, dbconn_idpackage):

2151 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2152 # create a random table and fill 2153 randomtable = "cdiff%s" % (self.entropyTools.get_random_number(),) 2154 while self.doesTableExist(randomtable): 2155 randomtable = "cdiff%s" % (self.entropyTools.get_random_number(),) 2156 self.cursor.execute('CREATE TEMPORARY TABLE %s ( file VARCHAR )' % (randomtable,)) 2157 2158 try: 2159 dbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2160 dbconn.cursor.execute('select file from content where idpackage = (?)', (dbconn_idpackage,)) 2161 xfile = dbconn.cursor.fetchone() 2162 while xfile: 2163 self.cursor.execute('INSERT INTO %s VALUES (?)' % (randomtable,), (xfile[0],)) 2164 xfile = dbconn.cursor.fetchone() 2165 2166 # now compare 2167 self.cursor.execute(""" 2168 SELECT file FROM content 2169 WHERE content.idpackage = (?) AND 2170 content.file NOT IN (SELECT file from %s)""" % (randomtable,), (idpackage,)) 2171 diff = self.fetchall2set(self.cursor.fetchall()) 2172 return diff 2173 finally: 2174 self.cursor.execute('DROP TABLE IF EXISTS %s' % (randomtable,))

2175

2176 - def doCleanups(self):

2177 self.cleanupUseflags() 2178 self.cleanupSources() 2179 self.cleanupEclasses() 2180 self.cleanupNeeded() 2181 self.cleanupNeededPaths() 2182 self.cleanupDependencies() 2183 self.cleanupChangelogs()

2184

2185 - def cleanupUseflags(self):

2186 with self.__write_mutex: 2187 self.cursor.execute(""" 2188 DELETE FROM useflagsreference 2189 WHERE idflag NOT IN (SELECT idflag FROM useflags)""")

2190

2191 - def cleanupSources(self):

2192 with self.__write_mutex: 2193 self.cursor.execute(""" 2194 DELETE FROM sourcesreference 2195 WHERE idsource NOT IN (SELECT idsource FROM sources)""")

2196

2197 - def cleanupEclasses(self):

2198 with self.__write_mutex: 2199 self.cursor.execute(""" 2200 DELETE FROM eclassesreference 2201 WHERE idclass NOT IN (SELECT idclass FROM eclasses)""")

2202

2203 - def cleanupNeeded(self):

2204 with self.__write_mutex: 2205 self.cursor.execute(""" 2206 DELETE FROM neededreference 2207 WHERE idneeded NOT IN (SELECT idneeded FROM needed)""")

2208

2209 - def cleanupNeededPaths(self):

2210 with self.__write_mutex: 2211 self.cursor.execute(""" 2212 DELETE FROM neededlibrarypaths 2213 WHERE library NOT IN (SELECT library FROM neededreference)""")

2214

2215 - def cleanupDependencies(self):

2216 with self.__write_mutex: 2217 self.cursor.execute(""" 2218 DELETE FROM dependenciesreference 2219 WHERE iddependency NOT IN (SELECT iddependency FROM dependencies)""")

2220

2221 - def cleanupChangelogs(self):

2222 with self.__write_mutex: 2223 self.cursor.execute(""" 2224 DELETE FROM packagechangelogs 2225 WHERE category || "/" || name NOT IN 2226 (SELECT categories.category || "/" || baseinfo.name FROM baseinfo,categories 2227 WHERE baseinfo.idcategory = categories.idcategory 2228 )""")

2229

2230 - def getNewNegativeCounter(self):

2231 counter = -2 2232 try: 2233 self.cursor.execute('SELECT min(counter) FROM counters') 2234 dbcounter = self.cursor.fetchone() 2235 mycounter = 0 2236 if dbcounter: 2237 mycounter = dbcounter[0] 2238 2239 if mycounter >= -1: 2240 counter = -2 2241 else: 2242 counter = mycounter-1 2243 2244 except: 2245 pass 2246 return counter

2247

2248 - def getApi(self):

2249 self.cursor.execute('SELECT max(etpapi) FROM baseinfo') 2250 api = self.cursor.fetchone() 2251 if api: api = api[0] 2252 else: api = -1 2253 return api

2254

2255 - def getCategory(self, idcategory):

2256 self.cursor.execute('SELECT category from categories WHERE idcategory = (?)', (idcategory,)) 2257 cat = self.cursor.fetchone() 2258 if cat: cat = cat[0] 2259 return cat

2260

2261 - def get_category_description_from_disk(self, category):

2262 if not self.ServiceInterface: 2263 return {} 2264 return self.ServiceInterface.SpmService.get_category_description_data(category)

2265

2266 - def getIDPackage(self, atom, branch = None):

2267 branch_string = '' 2268 params = [atom] 2269 if branch: 2270 branch_string = ' AND branch = (?)' 2271 params.append(branch) 2272 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)'+branch_string, params) 2273 idpackage = self.cursor.fetchone() 2274 if idpackage: return idpackage[0] 2275 return -1

2276

2277 - def getIDPackageFromDownload(self, download_relative_path, endswith = False):

2278 if endswith: 2279 self.cursor.execute(""" 2280 SELECT baseinfo.idpackage FROM baseinfo,extrainfo 2281 WHERE extrainfo.download LIKE (?)""", ("%"+download_relative_path,)) 2282 else: 2283 self.cursor.execute(""" 2284 SELECT baseinfo.idpackage FROM baseinfo,extrainfo 2285 WHERE extrainfo.download = (?)""", (download_relative_path,)) 2286 idpackage = self.cursor.fetchone() 2287 if idpackage: return idpackage[0] 2288 return -1

2289

2290 - def getIDPackagesFromFile(self, file):

2291 self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (file,)) 2292 return self.fetchall2list(self.cursor.fetchall())

2293

2294 - def getIDCategory(self, category):

2295 self.cursor.execute('SELECT "idcategory" FROM categories WHERE category = (?)', (category,)) 2296 idcat = self.cursor.fetchone() 2297 if idcat: return idcat[0] 2298 return -1

2299

2300 - def getVersioningData(self, idpackage):

2301 self.cursor.execute('SELECT version,versiontag,revision FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2302 return self.cursor.fetchone()

2303

2304 - def getStrictData(self, idpackage):

2305 self.cursor.execute(""" 2306 SELECT categories.category || "/" || baseinfo.name, 2307 baseinfo.slot,baseinfo.version,baseinfo.versiontag, 2308 baseinfo.revision,baseinfo.atom FROM baseinfo,categories 2309 WHERE baseinfo.idpackage = (?) AND 2310 baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2311 return self.cursor.fetchone()

2312

2313 - def getStrictScopeData(self, idpackage):

2314 self.cursor.execute(""" 2315 SELECT atom,slot,revision FROM baseinfo 2316 WHERE idpackage = (?)""", (idpackage,)) 2317 rslt = self.cursor.fetchone() 2318 return rslt

2319

2320 - def getScopeData(self, idpackage):

2321 self.cursor.execute(""" 2322 SELECT 2323 baseinfo.atom, 2324 categories.category, 2325 baseinfo.name, 2326 baseinfo.version, 2327 baseinfo.slot, 2328 baseinfo.versiontag, 2329 baseinfo.revision, 2330 baseinfo.branch, 2331 baseinfo.etpapi 2332 FROM 2333 baseinfo, 2334 categories 2335 WHERE 2336 baseinfo.idpackage = (?) 2337 and baseinfo.idcategory = categories.idcategory 2338 """, (idpackage,)) 2339 return self.cursor.fetchone()

2340

2341 - def getBaseData(self, idpackage):

2342 2343 sql = """ 2344 SELECT 2345 baseinfo.atom, 2346 baseinfo.name, 2347 baseinfo.version, 2348 baseinfo.versiontag, 2349 extrainfo.description, 2350 categories.category, 2351 flags.chost, 2352 flags.cflags, 2353 flags.cxxflags, 2354 extrainfo.homepage, 2355 licenses.license, 2356 baseinfo.branch, 2357 extrainfo.download, 2358 extrainfo.digest, 2359 baseinfo.slot, 2360 baseinfo.etpapi, 2361 extrainfo.datecreation, 2362 extrainfo.size, 2363 baseinfo.revision 2364 FROM 2365 baseinfo, 2366 extrainfo, 2367 categories, 2368 flags, 2369 licenses 2370 WHERE 2371 baseinfo.idpackage = (?) 2372 and baseinfo.idpackage = extrainfo.idpackage 2373 and baseinfo.idcategory = categories.idcategory 2374 and extrainfo.idflags = flags.idflags 2375 and baseinfo.idlicense = licenses.idlicense 2376 """ 2377 self.cursor.execute(sql, (idpackage,)) 2378 return self.cursor.fetchone()

2379

2380 - def getTriggerInfo(self, idpackage, content = True):

2381 2382 atom, category, name, \ 2383 version, slot, versiontag, \ 2384 revision, branch, etpapi = self.getScopeData(idpackage) 2385 chost, cflags, cxxflags = self.retrieveCompileFlags(idpackage) 2386 2387 pkg_content = set() 2388 if content: 2389 pkg_content = self.retrieveContent(idpackage) 2390 2391 data = { 2392 'atom': atom, 2393 'category': category, 2394 'name': name, 2395 'version': version, 2396 'versiontag': versiontag, 2397 'revision': revision, 2398 'branch': branch, 2399 'chost': chost, 2400 'cflags': cflags, 2401 'cxxflags': cxxflags, 2402 'etpapi': etpapi, 2403 'trigger': self.retrieveTrigger(idpackage), 2404 'eclasses': self.retrieveEclasses(idpackage), 2405 'content': pkg_content, 2406 'spm_phases': self.retrieveSpmPhases(idpackage), 2407 } 2408 return data

2409

2410 - def getPackageData(self, idpackage, get_content = True, 2411 content_insert_formatted = False, trigger_unicode = True):

2412 data = {} 2413 2414 try: 2415 atom, name, version, versiontag, \ 2416 description, category, chost, \ 2417 cflags, cxxflags,homepage, \ 2418 mylicense, branch, download, \ 2419 digest, slot, etpapi, \ 2420 datecreation, size, revision = self.getBaseData(idpackage) 2421 except TypeError: 2422 return None 2423 2424 content = {} 2425 if get_content: 2426 content = self.retrieveContent( 2427 idpackage, extended = True, 2428 formatted = True, insert_formatted = content_insert_formatted 2429 ) 2430 2431 sources = self.retrieveSources(idpackage) 2432 mirrornames = set() 2433 for x in sources: 2434 if x.startswith("mirror://"): 2435 mirrornames.add(x.split("/")[2]) 2436 2437 data = { 2438 'atom': atom, 2439 'name': name, 2440 'version': version, 2441 'versiontag':versiontag, 2442 'description': description, 2443 'category': category, 2444 'chost': chost, 2445 'cflags': cflags, 2446 'cxxflags': cxxflags, 2447 'homepage': homepage, 2448 'license': mylicense, 2449 'branch': branch, 2450 'download': download, 2451 'digest': digest, 2452 'slot': slot, 2453 'etpapi': etpapi, 2454 'datecreation': datecreation, 2455 'size': size, 2456 'revision': revision, 2457 # risky to add to the sql above, still 2458 'counter': self.retrieveCounter(idpackage), 2459 'messages': self.retrieveMessages(idpackage), 2460 'trigger': self.retrieveTrigger(idpackage, get_unicode = trigger_unicode), 2461 'disksize': self.retrieveOnDiskSize(idpackage), 2462 'changelog': self.retrieveChangelog(idpackage), 2463 'injected': self.isInjected(idpackage), 2464 'systempackage': self.isSystemPackage(idpackage), 2465 'config_protect': self.retrieveProtect(idpackage), 2466 'config_protect_mask': self.retrieveProtectMask(idpackage), 2467 'useflags': self.retrieveUseflags(idpackage), 2468 'keywords': self.retrieveKeywords(idpackage), 2469 'sources': sources, 2470 'eclasses': self.retrieveEclasses(idpackage), 2471 'needed': self.retrieveNeeded(idpackage, extended = True), 2472 'needed_paths': self.retrieveNeededPaths(idpackage), 2473 'provide': self.retrieveProvide(idpackage), 2474 'conflicts': self.retrieveConflicts(idpackage), 2475 'licensedata': self.retrieveLicensedata(idpackage), 2476 'content': content, 2477 'dependencies': dict((x, y,) for x, y in \ 2478 self.retrieveDependencies(idpackage, extended = True)), 2479 'mirrorlinks': [[x,self.retrieveMirrorInfo(x)] for x in mirrornames], 2480 'signatures': self.retrieveSignatures(idpackage), 2481 'spm_phases': self.retrieveSpmPhases(idpackage), 2482 } 2483 2484 return data

2485

2486 - def fetchall2set(self, item):

2487 mycontent = set() 2488 for x in item: 2489 mycontent |= set(x) 2490 return mycontent

2491

2492 - def fetchall2list(self, item):

2493 content = [] 2494 for x in item: 2495 content += list(x) 2496 return content

2497

2498 - def fetchone2list(self, item):

2499 return list(item)

2500

2501 - def fetchone2set(self, item):

2502 return set(item)

2503

2504 - def clearCache(self, depends = False):

2505 2506 self.live_cache.clear() 2507 def do_clear(name): 2508 dump_path = os.path.join(etpConst['dumpstoragedir'], name) 2509 dump_dir = os.path.dirname(dump_path) 2510 if os.path.isdir(dump_dir): 2511 for item in os.listdir(dump_dir): 2512 try: os.remove(os.path.join(dump_dir, item)) 2513 except OSError: pass

2514 2515 do_clear("%s/%s/" % (self.dbMatchCacheKey, self.dbname,)) 2516 if depends: 2517 do_clear(etpCache['depends_tree']) 2518 do_clear(etpCache['dep_tree']) 2519 do_clear(etpCache['filter_satisfied_deps']) 2520

2521 - def retrieveRepositoryUpdatesDigest(self, repository):

2522 if not self.doesTableExist("treeupdates"): 2523 return -1 2524 self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,)) 2525 mydigest = self.cursor.fetchone() 2526 if mydigest: 2527 return mydigest[0] 2528 else: 2529 return -1

2530

2531 - def listAllTreeUpdatesActions(self, no_ids_repos = False):

2532 if no_ids_repos: 2533 self.cursor.execute('SELECT command,branch,date FROM treeupdatesactions') 2534 else: 2535 self.cursor.execute('SELECT * FROM treeupdatesactions') 2536 return self.cursor.fetchall()

2537

2538 - def retrieveTreeUpdatesActions(self, repository, forbranch = None):

2539 2540 if not self.doesTableExist("treeupdatesactions"): return [] 2541 if forbranch == None: forbranch = self.db_branch 2542 params = [repository] 2543 branch_string = '' 2544 if forbranch: 2545 branch_string = 'and branch = (?)' 2546 params.append(forbranch) 2547 2548 self.cursor.execute(""" 2549 SELECT command FROM treeupdatesactions WHERE 2550 repository = (?) %s order by date""" % (branch_string,), params) 2551 return self.fetchall2list(self.cursor.fetchall())

2552 2553 # mainly used to restore a previous table, used by reagent in --initialize

2554 - def bumpTreeUpdatesActions(self, updates):

2555 with self.__write_mutex: 2556 self.cursor.execute('DELETE FROM treeupdatesactions') 2557 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (?,?,?,?,?)', updates) 2558 self.commitChanges()

2559

2560 - def removeTreeUpdatesActions(self, repository):

2561 with self.__write_mutex: 2562 self.cursor.execute('DELETE FROM treeupdatesactions WHERE repository = (?)', (repository,)) 2563 self.commitChanges()

2564

2565 - def insertTreeUpdatesActions(self, updates, repository):

2566 with self.__write_mutex: 2567 myupdates = [[repository]+list(x) for x in updates] 2568 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', myupdates) 2569 self.commitChanges()

2570

2571 - def setRepositoryUpdatesDigest(self, repository, digest):

2572 with self.__write_mutex: 2573 self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety 2574 self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository, digest,))

2575

2576 - def addRepositoryUpdatesActions(self, repository, actions, branch):

2577 2578 mytime = str(self.entropyTools.get_current_unix_time()) 2579 with self.__write_mutex: 2580 myupdates = [ 2581 (repository, x, branch, mytime,) for x in actions \ 2582 if not self.doesTreeupdatesActionExist(repository, x, branch) 2583 ] 2584 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', myupdates)

2585

2586 - def doesTreeupdatesActionExist(self, repository, command, branch):

2587 self.cursor.execute(""" 2588 SELECT * FROM treeupdatesactions 2589 WHERE repository = (?) and command = (?) and branch = (?)""", (repository, command, branch,)) 2590 result = self.cursor.fetchone() 2591 if result: 2592 return True 2593 return False

2594

2595 - def clearPackageSets(self):

2596 self.cursor.execute('DELETE FROM packagesets')

2597

2598 - def insertPackageSets(self, sets_data):

2599 2600 mysets = [] 2601 for setname in sorted(sets_data.keys()): 2602 for dependency in sorted(sets_data[setname]): 2603 try: 2604 mysets.append((unicode(setname), unicode(dependency),)) 2605 except (UnicodeDecodeError, UnicodeEncodeError,): 2606 continue 2607 2608 with self.__write_mutex: 2609 self.cursor.executemany('INSERT INTO packagesets VALUES (?,?)', mysets)

2610

2611 - def retrievePackageSets(self):

2612 if not self.doesTableExist("packagesets"): return {} 2613 self.cursor.execute('SELECT setname,dependency FROM packagesets') 2614 data = self.cursor.fetchall() 2615 sets = {} 2616 for setname, dependency in data: 2617 if not sets.has_key(setname): 2618 sets[setname] = set() 2619 sets[setname].add(dependency) 2620 return sets

2621

2622 - def retrievePackageSet(self, setname):

2623 self.cursor.execute('SELECT dependency FROM packagesets WHERE setname = (?)', (setname,)) 2624 return self.fetchall2set(self.cursor.fetchall())

2625

2626 - def retrieveSystemPackages(self):

2627 self.cursor.execute('SELECT idpackage FROM systempackages') 2628 return self.fetchall2set(self.cursor.fetchall())

2629

2630 - def retrieveAtom(self, idpackage):

2631 self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2632 atom = self.cursor.fetchone() 2633 if atom: return atom[0]

2634

2635 - def retrieveBranch(self, idpackage):

2636 self.cursor.execute('SELECT branch FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2637 br = self.cursor.fetchone() 2638 if br: return br[0]

2639

2640 - def retrieveTrigger(self, idpackage, get_unicode = False):

2641 self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,)) 2642 trigger = self.cursor.fetchone() 2643 if not trigger: 2644 return '' # FIXME backward compatibility with <=0.52.x 2645 if not get_unicode: 2646 return trigger[0] 2647 return unicode(trigger[0], 'raw_unicode_escape')

2648

2649 - def retrieveDownloadURL(self, idpackage):

2650 self.cursor.execute('SELECT download FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2651 download = self.cursor.fetchone() 2652 if download: return download[0]

2653

2654 - def retrieveDescription(self, idpackage):

2655 self.cursor.execute('SELECT description FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2656 description = self.cursor.fetchone() 2657 if description: return description[0]

2658

2659 - def retrieveHomepage(self, idpackage):

2660 self.cursor.execute('SELECT homepage FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2661 home = self.cursor.fetchone() 2662 if home: return home[0]

2663

2664 - def retrieveCounter(self, idpackage):

2665 self.cursor.execute(""" 2666 SELECT counters.counter FROM counters,baseinfo 2667 WHERE counters.idpackage = (?) AND 2668 baseinfo.idpackage = counters.idpackage AND 2669 baseinfo.branch = counters.branch""", (idpackage,)) 2670 mycounter = self.cursor.fetchone() 2671 if mycounter: return mycounter[0] 2672 return -1

2673

2674 - def retrieveMessages(self, idpackage):

2675 self.cursor.execute('SELECT message FROM messages WHERE idpackage = (?)', (idpackage,)) 2676 return self.fetchall2list(self.cursor.fetchall())

2677 2678 # in bytes

2679 - def retrieveSize(self, idpackage):

2680 self.cursor.execute('SELECT size FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2681 size = self.cursor.fetchone() 2682 if size: return size[0]

2683 2684 # in bytes

2685 - def retrieveOnDiskSize(self, idpackage):

2686 self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,)) 2687 size = self.cursor.fetchone() # do not use [0]! 2688 if not size: size = 0 2689 else: size = size[0] 2690 return size

2691

2692 - def retrieveDigest(self, idpackage):

2693 self.cursor.execute('SELECT digest FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2694 digest = self.cursor.fetchone() 2695 if digest: return digest[0]

2696

2697 - def retrieveSignatures(self, idpackage):

2698 mydict = { 2699 'sha1': None, 2700 'sha256': None, 2701 'sha512': None, 2702 } 2703 # FIXME: remove this check in future 2704 if self.doesTableExist('packagesignatures'): 2705 self.cursor.execute(""" 2706 SELECT sha1, sha256, sha512 FROM packagesignatures 2707 WHERE idpackage = (?)""", (idpackage,)) 2708 data = self.cursor.fetchone() 2709 if data: 2710 mydict['sha1'], mydict['sha256'], mydict['sha512'] = data 2711 return mydict

2712

2713 - def retrieveName(self, idpackage):

2714 self.cursor.execute('SELECT name FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2715 name = self.cursor.fetchone() 2716 if name: return name[0]

2717

2718 - def retrieveKeySlot(self, idpackage):

2719 self.cursor.execute(""" 2720 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories 2721 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2722 data = self.cursor.fetchone() 2723 return data

2724

2725 - def retrieveKeySlotAggregated(self, idpackage):

2726 self.cursor.execute(""" 2727 SELECT categories.category || "/" || baseinfo.name || ":" || baseinfo.slot FROM baseinfo,categories 2728 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2729 data = self.cursor.fetchone() 2730 if data: return data[0]

2731

2732 - def retrieveKeySlotTag(self, idpackage):

2733 self.cursor.execute(""" 2734 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot,baseinfo.versiontag FROM baseinfo,categories 2735 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2736 data = self.cursor.fetchone() 2737 return data

2738

2739 - def retrieveVersion(self, idpackage):

2740 self.cursor.execute('SELECT version FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2741 ver = self.cursor.fetchone() 2742 if ver: return ver[0]

2743

2744 - def retrieveRevision(self, idpackage):

2745 self.cursor.execute('SELECT revision FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2746 rev = self.cursor.fetchone() 2747 if rev: return rev[0]

2748

2749 - def retrieveDateCreation(self, idpackage):

2750 self.cursor.execute('SELECT datecreation FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2751 date = self.cursor.fetchone() 2752 if date: return date[0]

2753

2754 - def retrieveApi(self, idpackage):

2755 self.cursor.execute('SELECT etpapi FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2756 api = self.cursor.fetchone() 2757 if api: return api[0]

2758

2759 - def retrieveUseflags(self, idpackage):

2760 self.cursor.execute(""" 2761 SELECT flagname FROM useflags,useflagsreference 2762 WHERE useflags.idpackage = (?) AND 2763 useflags.idflag = useflagsreference.idflag""", (idpackage,)) 2764 return self.fetchall2set(self.cursor.fetchall())

2765

2766 - def retrieveEclasses(self, idpackage):

2767 self.cursor.execute(""" 2768 SELECT classname FROM eclasses,eclassesreference 2769 WHERE eclasses.idpackage = (?) AND 2770 eclasses.idclass = eclassesreference.idclass""", (idpackage,)) 2771 return self.fetchall2set(self.cursor.fetchall())

2772

2773 - def retrieveSpmPhases(self, idpackage):

2774 # FIXME: remove this check in future: 2775 if not self.doesTableExist('packagespmphases'): 2776 return None 2777 self.cursor.execute(""" 2778 SELECT phases FROM packagespmphases 2779 WHERE idpackage = (?) 2780 """, (idpackage,)) 2781 rslt = self.cursor.fetchone() 2782 if rslt: return rslt[0]

2783

2784 - def retrieveNeededRaw(self, idpackage):

2785 self.cursor.execute(""" 2786 SELECT library FROM needed,neededreference 2787 WHERE needed.idpackage = (?) AND 2788 needed.idneeded = neededreference.idneeded""", (idpackage,)) 2789 return self.fetchall2set(self.cursor.fetchall())

2790

2791 - def retrieveNeeded(self, idpackage, extended = False, format = False):

2792 2793 if extended: 2794 self.cursor.execute(""" 2795 SELECT library,elfclass FROM needed,neededreference 2796 WHERE needed.idpackage = (?) AND 2797 needed.idneeded = neededreference.idneeded order by library""", (idpackage,)) 2798 needed = self.cursor.fetchall() 2799 else: 2800 self.cursor.execute(""" 2801 SELECT library FROM needed,neededreference 2802 WHERE needed.idpackage = (?) AND 2803 needed.idneeded = neededreference.idneeded ORDER BY library""", (idpackage,)) 2804 needed = self.fetchall2list(self.cursor.fetchall()) 2805 2806 if extended and format: 2807 data = {} 2808 for lib, elfclass in needed: 2809 data[lib] = elfclass 2810 needed = data 2811 2812 return needed

2813

2814 - def retrieveNeededPaths(self, idpackage):

2815 if not self.doesTableExist('neededlibrarypaths'): 2816 return {} 2817 self.cursor.execute(""" 2818 SELECT neededlibrarypaths.library, neededlibrarypaths.path, 2819 neededlibrarypaths.elfclass FROM 2820 neededlibrarypaths, neededreference, needed WHERE 2821 needed.idpackage = (?) AND needed.idneeded = neededreference.idneeded 2822 AND neededreference.library = neededlibrarypaths.library 2823 """, (idpackage,)) 2824 data = {} 2825 for lib, path, elfclass in self.cursor.fetchall(): 2826 obj = data.setdefault(lib, set()) 2827 obj.add((path, elfclass)) 2828 return data

2829

2830 - def retrieveNeededLibraryPaths(self, needed_library_name, elfclass):

2831 if not self.doesTableExist('neededlibrarypaths'): 2832 return set() 2833 self.cursor.execute(""" 2834 SELECT path FROM neededlibrarypaths, neededreference, needed 2835 WHERE neededlibrarypaths.library = (?) AND 2836 neededlibrarypaths.elfclass = (?) AND 2837 neededreference.library = neededlibrarypaths.library AND 2838 needed.elfclass = neededlibrarypaths.elfclass AND 2839 needed.idneeded = neededreference.idneeded 2840 """, (needed_library_name, elfclass,)) 2841 return self.fetchall2set(self.cursor.fetchall())

2842

2843 - def retrieveNeededLibraryIdpackages(self):

2844 if not self.doesTableExist('neededlibraryidpackages'): 2845 return [] 2846 self.cursor.execute('SELECT idpackage, library, elfclass FROM neededlibraryidpackages') 2847 return self.cursor.fetchall()

2848

2849 - def clearNeededLibraryIdpackages(self):

2850 if not self.doesTableExist('neededlibraryidpackages'): 2851 return 2852 self.cursor.execute('DELETE FROM neededlibraryidpackages')

2853

2854 - def setNeededLibraryIdpackages(self, library_map):

2855 if not self.doesTableExist('neededlibraryidpackages'): 2856 return 2857 self.cursor.executemany('INSERT INTO neededlibraryidpackages VALUES (?,?,?)', library_map)

2858

2859 - def retrieveConflicts(self, idpackage):

2860 self.cursor.execute('SELECT conflict FROM conflicts WHERE idpackage = (?)', (idpackage,)) 2861 return self.fetchall2set(self.cursor.fetchall())

2862

2863 - def retrieveProvide(self, idpackage):

2864 self.cursor.execute('SELECT atom FROM provide WHERE idpackage = (?)', (idpackage,)) 2865 return self.fetchall2set(self.cursor.fetchall())

2866

2867 - def retrieveDependenciesList(self, idpackage):

2868 self.cursor.execute(""" 2869 SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference 2870 WHERE dependencies.idpackage = (?) AND 2871 dependencies.iddependency = dependenciesreference.iddependency 2872 UNION SELECT "!" || conflict FROM conflicts 2873 WHERE idpackage = (?)""", (idpackage, idpackage,)) 2874 return self.fetchall2set(self.cursor.fetchall())

2875

2876 - def retrievePostDependencies(self, idpackage, extended = False):

2877 return self.retrieveDependencies(idpackage, extended = extended, deptype = etpConst['spm']['pdepend_id'])

2878

2879 - def retrieveManualDependencies(self, idpackage, extended = False):

2880 return self.retrieveDependencies(idpackage, extended = extended, deptype = etpConst['spm']['mdepend_id'])

2881

2882 - def retrieveDependencies(self, idpackage, extended = False, deptype = None, 2883 exclude_deptypes = None):

2884 2885 searchdata = [idpackage] 2886 2887 depstring = '' 2888 if deptype != None: 2889 depstring = ' and dependencies.type = (?)' 2890 searchdata.append(deptype) 2891 2892 excluded_deptypes_query = "" 2893 if exclude_deptypes != None: 2894 for dep_type in exclude_deptypes: 2895 excluded_deptypes_query += " AND dependencies.type != %s" % ( 2896 dep_type,) 2897 2898 if extended: 2899 self.cursor.execute(""" 2900 SELECT dependenciesreference.dependency,dependencies.type 2901 FROM dependencies,dependenciesreference 2902 WHERE dependencies.idpackage = (?) AND 2903 dependencies.iddependency = dependenciesreference.iddependency %s %s""" % ( 2904 depstring,excluded_deptypes_query,), searchdata) 2905 deps = self.cursor.fetchall() 2906 else: 2907 self.cursor.execute(""" 2908 SELECT dependenciesreference.dependency 2909 FROM dependencies,dependenciesreference 2910 WHERE dependencies.idpackage = (?) AND 2911 dependencies.iddependency = dependenciesreference.iddependency %s %s""" % ( 2912 depstring,excluded_deptypes_query,), searchdata) 2913 deps = self.fetchall2set(self.cursor.fetchall()) 2914 2915 return deps

2916

2917 - def retrieveIdDependencies(self, idpackage):

2918 self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = (?)', (idpackage,)) 2919 return self.fetchall2set(self.cursor.fetchall())

2920

2921 - def retrieveDependencyFromIddependency(self, iddependency):

2922 self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = (?)', (iddependency,)) 2923 dep = self.cursor.fetchone() 2924 if dep: dep = dep[0] 2925 return dep

2926

2927 - def retrieveKeywords(self, idpackage):

2928 self.cursor.execute(""" 2929 SELECT keywordname FROM keywords,keywordsreference 2930 WHERE keywords.idpackage = (?) AND 2931 keywords.idkeyword = keywordsreference.idkeyword""", (idpackage,)) 2932 return self.fetchall2set(self.cursor.fetchall())

2933

2934 - def retrieveProtect(self, idpackage):

2935 self.cursor.execute(""" 2936 SELECT protect FROM configprotect,configprotectreference 2937 WHERE configprotect.idpackage = (?) AND 2938 configprotect.idprotect = configprotectreference.idprotect""", (idpackage,)) 2939 protect = self.cursor.fetchone() 2940 if not protect: protect = '' 2941 else: protect = protect[0] 2942 return protect

2943

2944 - def retrieveProtectMask(self, idpackage):

2945 self.cursor.execute(""" 2946 SELECT protect FROM configprotectmask,configprotectreference 2947 WHERE idpackage = (?) AND 2948 configprotectmask.idprotect = configprotectreference.idprotect""", (idpackage,)) 2949 protect = self.cursor.fetchone() 2950 if not protect: protect = '' 2951 else: protect = protect[0] 2952 return protect

2953

2954 - def retrieveSources(self, idpackage, extended = False):

2955 self.cursor.execute(""" 2956 SELECT sourcesreference.source FROM sources,sourcesreference 2957 WHERE idpackage = (?) AND 2958 sources.idsource = sourcesreference.idsource""", (idpackage,)) 2959 sources = self.fetchall2set(self.cursor.fetchall()) 2960 if not extended: 2961 return sources 2962 2963 source_data = {} 2964 mirror_str = "mirror://" 2965 for source in sources: 2966 source_data[source] = set() 2967 if source.startswith(mirror_str): 2968 mirrorname = source.split("/")[2] 2969 mirror_url = source.split("/", 3)[3:][0] 2970 source_data[source] |= set([os.path.join(url, mirror_url) for url in self.retrieveMirrorInfo(mirrorname)]) 2971 else: 2972 source_data[source].add(source) 2973 2974 return source_data

2975

2976 - def retrieveAutomergefiles(self, idpackage, get_dict = False):

2977 if not self.doesTableExist('automergefiles'): 2978 self.createAutomergefilesTable() 2979 # like portage does 2980 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2981 self.cursor.execute('SELECT configfile, md5 FROM automergefiles WHERE idpackage = (?)', (idpackage,)) 2982 data = self.cursor.fetchall() 2983 if get_dict: 2984 data = dict(((x, y,) for x, y in data)) 2985 return data

2986

2987 - def retrieveContent(self, idpackage, extended = False, contentType = None, 2988 formatted = False, insert_formatted = False, order_by = ''):

2989 2990 extstring = '' 2991 if extended: 2992 extstring = ",type" 2993 extstring_idpackage = '' 2994 if insert_formatted: 2995 extstring_idpackage = 'idpackage,' 2996 2997 searchkeywords = [idpackage] 2998 contentstring = '' 2999 if contentType: 3000 searchkeywords.append(contentType) 3001 contentstring = ' and type = (?)' 3002 3003 order_by_string = '' 3004 if order_by: 3005 order_by_string = ' order by %s' % (order_by,) 3006 3007 did_try = False 3008 while 1: 3009 try: 3010 3011 self.cursor.execute('SELECT %s file%s FROM content WHERE idpackage = (?) %s%s' % ( 3012 extstring_idpackage, extstring, contentstring, order_by_string,), 3013 searchkeywords) 3014 3015 if extended and insert_formatted: 3016 fl = self.cursor.fetchall() 3017 elif extended and formatted: 3018 fl = {} 3019 items = self.cursor.fetchone() 3020 while items: 3021 fl[items[0]] = items[1] 3022 items = self.cursor.fetchone() 3023 elif extended: 3024 fl = self.cursor.fetchall() 3025 else: 3026 if order_by: 3027 fl = self.fetchall2list(self.cursor.fetchall()) 3028 else: 3029 fl = self.fetchall2set(self.cursor.fetchall()) 3030 break 3031 except (self.dbapi2.OperationalError,): 3032 if did_try: 3033 raise 3034 did_try = True 3035 # XXX support for old entropy db entries, which were 3036 # not inserted in utf-8 3037 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3038 continue 3039 return fl

3040

3041 - def retrieveChangelog(self, idpackage):

3042 if not self.doesTableExist('packagechangelogs'): 3043 return None 3044 self.cursor.execute(""" 3045 SELECT packagechangelogs.changelog FROM packagechangelogs,baseinfo,categories 3046 WHERE baseinfo.idpackage = (?) AND 3047 baseinfo.idcategory = categories.idcategory AND 3048 packagechangelogs.name = baseinfo.name AND 3049 packagechangelogs.category = categories.category""", (idpackage,)) 3050 changelog = self.cursor.fetchone() 3051 if changelog: 3052 changelog = changelog[0] 3053 try: 3054 return unicode(changelog, 'raw_unicode_escape') 3055 except UnicodeDecodeError: 3056 return unicode(changelog, 'utf-8')

3057

3058 - def retrieveChangelogByKey(self, category, name):

3059 if not self.doesTableExist('packagechangelogs'): 3060 return None 3061 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3062 self.cursor.execute('SELECT changelog FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,)) 3063 changelog = self.cursor.fetchone() 3064 if changelog: return unicode(changelog[0], 'raw_unicode_escape')

3065

3066 - def retrieveSlot(self, idpackage):

3067 self.cursor.execute('SELECT slot FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3068 slot = self.cursor.fetchone() 3069 if slot: return slot[0]

3070

3071 - def retrieveVersionTag(self, idpackage):

3072 self.cursor.execute('SELECT versiontag FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3073 vtag = self.cursor.fetchone() 3074 if vtag: return vtag[0]

3075

3076 - def retrieveMirrorInfo(self, mirrorname):

3077 self.cursor.execute('SELECT mirrorlink FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) 3078 mirrorlist = self.fetchall2set(self.cursor.fetchall()) 3079 return mirrorlist

3080

3081 - def retrieveCategory(self, idpackage):

3082 self.cursor.execute(""" 3083 SELECT category FROM baseinfo,categories 3084 WHERE baseinfo.idpackage = (?) AND 3085 baseinfo.idcategory = categories.idcategory""", (idpackage,)) 3086 cat = self.cursor.fetchone() 3087 if cat: return cat[0]

3088

3089 - def retrieveCategoryDescription(self, category):

3090 data = {} 3091 self.cursor.execute('SELECT description,locale FROM categoriesdescription WHERE category = (?)', (category,)) 3092 description_data = self.cursor.fetchall() 3093 for description, locale in description_data: 3094 data[locale] = description 3095 return data

3096

3097 - def retrieveLicensedata(self, idpackage):

3098 3099 # insert license information 3100 licenses = self.retrieveLicense(idpackage) 3101 if licenses == None: 3102 return {} 3103 licenses = licenses.split() 3104 licdata = {} 3105 for licname in licenses: 3106 licname = licname.strip() 3107 if not self.entropyTools.is_valid_string(licname): 3108 continue 3109 3110 self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,)) 3111 lictext = self.cursor.fetchone() 3112 if lictext != None: 3113 lictext = lictext[0] 3114 try: 3115 licdata[licname] = unicode(lictext, 'raw_unicode_escape') 3116 except UnicodeDecodeError: 3117 licdata[licname] = unicode(lictext, 'utf-8') 3118 3119 return licdata

3120

3121 - def retrieveLicensedataKeys(self, idpackage):

3122 3123 licenses = self.retrieveLicense(idpackage) 3124 if licenses == None: 3125 return set() 3126 licenses = licenses.split() 3127 licdata = set() 3128 for licname in licenses: 3129 licname = licname.strip() 3130 if not self.entropyTools.is_valid_string(licname): 3131 continue 3132 self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,)) 3133 licidentifier = self.cursor.fetchone() 3134 if licidentifier: 3135 licdata.add(licidentifier[0]) 3136 3137 return licdata

3138

3139 - def retrieveLicenseText(self, license_name):

3140 3141 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3142 3143 self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (license_name,)) 3144 text = self.cursor.fetchone() 3145 if not text: 3146 return None 3147 return str(text[0])

3148

3149 - def retrieveLicense(self, idpackage):

3150 self.cursor.execute(""" 3151 SELECT license FROM baseinfo,licenses 3152 WHERE baseinfo.idpackage = (?) AND 3153 baseinfo.idlicense = licenses.idlicense""", (idpackage,)) 3154 licname = self.cursor.fetchone() 3155 if licname: return licname[0]

3156

3157 - def retrieveCompileFlags(self, idpackage):

3158 self.cursor.execute(""" 3159 SELECT chost,cflags,cxxflags FROM flags,extrainfo 3160 WHERE extrainfo.idpackage = (?) AND 3161 extrainfo.idflags = flags.idflags""", (idpackage,)) 3162 flags = self.cursor.fetchone() 3163 if not flags: 3164 flags = ("N/A", "N/A", "N/A",) 3165 return flags

3166

3167 - def retrieveDepends(self, idpackage, atoms = False, key_slot = False, 3168 exclude_deptypes = None):

3169 3170 # WARNING: never remove this, otherwise equo.db 3171 # (client database) dependstable will be always broken (trust me) 3172 # sanity check on the table 3173 if not self.isDependsTableSane(): # is empty, need generation 3174 self.regenerateDependsTable(output = False) 3175 3176 excluded_deptypes_query = "" 3177 if exclude_deptypes != None: 3178 for dep_type in exclude_deptypes: 3179 excluded_deptypes_query += " AND dependencies.type != %s" % ( 3180 dep_type,) 3181 3182 if atoms: 3183 self.cursor.execute(""" 3184 SELECT baseinfo.atom FROM dependstable,dependencies,baseinfo 3185 WHERE dependstable.idpackage = (?) AND 3186 dependstable.iddependency = dependencies.iddependency AND 3187 baseinfo.idpackage = dependencies.idpackage %s""" % ( 3188 excluded_deptypes_query,), (idpackage,)) 3189 result = self.fetchall2set(self.cursor.fetchall()) 3190 elif key_slot: 3191 self.cursor.execute(""" 3192 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot 3193 FROM baseinfo,categories,dependstable,dependencies 3194 WHERE dependstable.idpackage = (?) AND 3195 dependstable.iddependency = dependencies.iddependency AND 3196 baseinfo.idpackage = dependencies.idpackage AND 3197 categories.idcategory = baseinfo.idcategory %s""" % ( 3198 excluded_deptypes_query,), (idpackage,)) 3199 result = self.cursor.fetchall() 3200 else: 3201 self.cursor.execute(""" 3202 SELECT dependencies.idpackage FROM dependstable,dependencies 3203 WHERE dependstable.idpackage = (?) AND 3204 dependstable.iddependency = dependencies.iddependency %s""" % ( 3205 excluded_deptypes_query,), (idpackage,)) 3206 result = self.fetchall2set(self.cursor.fetchall()) 3207 3208 return result

3209

3210 - def retrieveUnusedIdpackages(self):

3211 3212 # WARNING: never remove this, otherwise equo.db (client database) 3213 # dependstable will be always broken (trust me) 3214 # sanity check on the table 3215 if not self.isDependsTableSane(): # is empty, need generation 3216 self.regenerateDependsTable(output = False) 3217 3218 self.cursor.execute(""" 3219 SELECT idpackage FROM baseinfo 3220 WHERE idpackage NOT IN (SELECT idpackage FROM dependstable) ORDER BY atom 3221 """) 3222 return self.fetchall2list(self.cursor.fetchall())

3223

3224 - def isPackageAvailable(self, pkgatom):

3225 # You must provide the full atom to this function 3226 # WARNING: this function does not support branches 3227 pkgatom = self.entropyTools.remove_package_operators(pkgatom) 3228 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)', (pkgatom,)) 3229 result = self.cursor.fetchone() 3230 if result: return result[0] 3231 return -1

3232

3233 - def isIDPackageAvailable(self, idpackage):

3234 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3235 result = self.cursor.fetchone() 3236 if not result: 3237 return False 3238 return True

3239

3240 - def areIDPackagesAvailable(self, idpackages):

3241 sql = 'SELECT count(idpackage) FROM baseinfo WHERE idpackage IN (%s)' % (','.join([str(x) for x in set(idpackages)]),) 3242 self.cursor.execute(sql) 3243 count = self.cursor.fetchone()[0] 3244 if count != len(idpackages): 3245 return False 3246 return True

3247

3248 - def isCategoryAvailable(self, category):

3249 self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) 3250 result = self.cursor.fetchone() 3251 if result: return result[0] 3252 return -1

3253

3254 - def isProtectAvailable(self, protect):

3255 self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,)) 3256 result = self.cursor.fetchone() 3257 if result: return result[0] 3258 return -1

3259

3260 - def isFileAvailable(self, myfile, get_id = False):

3261 self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (myfile,)) 3262 result = self.cursor.fetchall() 3263 if get_id: 3264 return self.fetchall2set(result) 3265 elif result: 3266 return True 3267 return False

3268

3269 - def resolveNeeded(self, needed, elfclass = -1, extended = False):

3270 3271 args = [needed] 3272 elfclass_txt = '' 3273 3274 if extended: 3275 if elfclass != -1: 3276 elfclass_txt = ' AND neededlibraryidpackages.elfclass = (?)' 3277 args.append(elfclass) 3278 self.cursor.execute(""" 3279 SELECT neededlibraryidpackages.idpackage, 3280 neededlibrarypaths.path 3281 FROM neededlibraryidpackages, neededlibrarypaths 3282 WHERE neededlibraryidpackages.library = (?) AND 3283 neededlibraryidpackages.library = neededlibrarypaths.library AND 3284 neededlibraryidpackages.elfclass = neededlibrarypaths.elfclass 3285 """ + elfclass_txt, args) 3286 return self.cursor.fetchall() 3287 else: 3288 if elfclass != -1: 3289 elfclass_txt = ' AND elfclass = (?)' 3290 args.append(elfclass) 3291 self.cursor.execute(""" 3292 SELECT idpackage FROM neededlibraryidpackages 3293 WHERE library = (?) 3294 """ + elfclass_txt, args) 3295 return self.fetchall2set(self.cursor.fetchall())

3296

3297 - def isSourceAvailable(self, source):

3298 self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = (?)', (source,)) 3299 result = self.cursor.fetchone() 3300 if result: return result[0] 3301 return -1

3302

3303 - def isDependencyAvailable(self, dependency):

3304 self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = (?)', (dependency,)) 3305 result = self.cursor.fetchone() 3306 if result: return result[0] 3307 return -1

3308

3309 - def isKeywordAvailable(self, keyword):

3310 self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = (?)', (keyword,)) 3311 result = self.cursor.fetchone() 3312 if result: return result[0] 3313 return -1

3314

3315 - def isUseflagAvailable(self, useflag):

3316 self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = (?)', (useflag,)) 3317 result = self.cursor.fetchone() 3318 if result: return result[0] 3319 return -1

3320

3321 - def isEclassAvailable(self, eclass):

3322 self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,)) 3323 result = self.cursor.fetchone() 3324 if result: return result[0] 3325 return -1

3326

3327 - def isNeededAvailable(self, needed):

3328 self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,)) 3329 result = self.cursor.fetchone() 3330 if result: return result[0] 3331 return -1

3332

3333 - def isCounterAvailable(self, counter, branch = None, branch_operator = "="):

3334 params = [counter] 3335 branch_string = '' 3336 if branch: 3337 branch_string = ' and branch '+branch_operator+' (?)' 3338 params = [counter, branch] 3339 3340 self.cursor.execute('SELECT counter FROM counters WHERE counter = (?)'+branch_string, params) 3341 result = self.cursor.fetchone() 3342 if result: return True 3343 return False

3344

3345 - def isCounterTrashed(self, counter):

3346 self.cursor.execute('SELECT counter FROM trashedcounters WHERE counter = (?)', (counter,)) 3347 result = self.cursor.fetchone() 3348 if result: return True 3349 return False

3350

3351 - def isLicensedataKeyAvailable(self, license_name):

3352 self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,)) 3353 result = self.cursor.fetchone() 3354 if not result: 3355 return False 3356 return True

3357

3358 - def isLicenseAccepted(self, license_name):

3359 self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,)) 3360 result = self.cursor.fetchone() 3361 if not result: 3362 return False 3363 return True

3364

3365 - def acceptLicense(self, license_name):

3366 if self.readOnly or (not self.entropyTools.is_user_in_entropy_group()): 3367 return 3368 if self.isLicenseAccepted(license_name): 3369 return 3370 with self.__write_mutex: 3371 self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,)) 3372 self.commitChanges()

3373

3374 - def isLicenseAvailable(self, pkglicense):

3375 if not self.entropyTools.is_valid_string(pkglicense): 3376 pkglicense = ' ' 3377 self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,)) 3378 result = self.cursor.fetchone() 3379 if result: return result[0] 3380 return -1

3381

3382 - def isSystemPackage(self, idpackage):

3383 self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) 3384 result = self.cursor.fetchone() 3385 if result: 3386 return True 3387 return False

3388

3389 - def isInjected(self, idpackage):

3390 self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,)) 3391 result = self.cursor.fetchone() 3392 if result: 3393 return True 3394 return False

3395

3396 - def areCompileFlagsAvailable(self, chost, cflags, cxxflags):

3397 3398 self.cursor.execute('SELECT idflags FROM flags WHERE chost = (?) AND cflags = (?) AND cxxflags = (?)', 3399 (chost, cflags, cxxflags,) 3400 ) 3401 result = self.cursor.fetchone() 3402 if result: return result[0] 3403 return -1

3404

3405 - def searchBelongs(self, file, like = False, branch = None, branch_operator = "="):

3406 3407 branchstring = '' 3408 searchkeywords = [file] 3409 if branch: 3410 searchkeywords.append(branch) 3411 branchstring = ' and baseinfo.branch '+branch_operator+' (?)' 3412 3413 if like: 3414 self.cursor.execute(""" 3415 SELECT content.idpackage FROM content,baseinfo 3416 WHERE file LIKE (?) AND 3417 content.idpackage = baseinfo.idpackage %s""" % (branchstring,), searchkeywords) 3418 else: 3419 self.cursor.execute("""SELECT content.idpackage FROM content,baseinfo 3420 WHERE file = (?) AND 3421 content.idpackage = baseinfo.idpackage %s""" % (branchstring,), searchkeywords) 3422 3423 return self.fetchall2set(self.cursor.fetchall())

3424 3425 ''' search packages that uses the eclass provided '''

3426 - def searchEclassedPackages(self, eclass, atoms = False): # atoms = return atoms directly

3427 if atoms: 3428 self.cursor.execute(""" 3429 SELECT baseinfo.atom,eclasses.idpackage FROM baseinfo,eclasses,eclassesreference 3430 WHERE eclassesreference.classname = (?) AND 3431 eclassesreference.idclass = eclasses.idclass AND 3432 eclasses.idpackage = baseinfo.idpackage""", (eclass,)) 3433 return self.cursor.fetchall() 3434 else: 3435 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (eclass,)) 3436 return self.fetchall2set(self.cursor.fetchall()) 3437 3438 ''' search packages whose versiontag matches the one provided '''

3439 - def searchTaggedPackages(self, tag, atoms = False): # atoms = return atoms directly

3440 if atoms: 3441 self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) 3442 return self.cursor.fetchall() 3443 else: 3444 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) 3445 return self.fetchall2set(self.cursor.fetchall()) 3446

3447 - def searchLicenses(self, mylicense, caseSensitive = False, atoms = False):

3448 3449 if not self.entropyTools.is_valid_string(mylicense): 3450 return [] 3451 3452 request = "baseinfo.idpackage" 3453 if atoms: 3454 request = "baseinfo.atom,baseinfo.idpackage" 3455 3456 if caseSensitive: 3457 self.cursor.execute(""" 3458 SELECT %s FROM baseinfo,licenses 3459 WHERE licenses.license LIKE (?) AND 3460 licenses.idlicense = baseinfo.idlicense""" % (request,), ("%"+mylicense+"%",)) 3461 else: 3462 self.cursor.execute(""" 3463 SELECT %s FROM baseinfo,licenses 3464 WHERE LOWER(licenses.license) LIKE (?) AND 3465 licenses.idlicense = baseinfo.idlicense""" % (request,), ("%"+mylicense+"%".lower(),)) 3466 if atoms: 3467 return self.cursor.fetchall() 3468 return self.fetchall2set(self.cursor.fetchall())

3469 3470 ''' search packages whose slot matches the one provided '''

3471 - def searchSlottedPackages(self, slot, atoms = False): # atoms = return atoms directly

3472 if atoms: 3473 self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE slot = (?)', (slot,)) 3474 return self.cursor.fetchall() 3475 else: 3476 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE slot = (?)', (slot,)) 3477 return self.fetchall2set(self.cursor.fetchall()) 3478

3479 - def searchKeySlot(self, key, slot, branch = None):

3480 3481 branchstring = '' 3482 cat, name = key.split("/") 3483 params = [cat, name, slot] 3484 if branch: 3485 params.append(branch) 3486 branchstring = 'and baseinfo.branch = (?)' 3487 3488 self.cursor.execute(""" 3489 SELECT idpackage FROM baseinfo,categories 3490 WHERE baseinfo.idcategory = categories.idcategory AND 3491 categories.category = (?) AND 3492 baseinfo.name = (?) AND 3493 baseinfo.slot = (?) %s""" % (branchstring,), params) 3494 return self.cursor.fetchall()

3495 3496 ''' search packages that need the specified library (in neededreference table) specified by keyword '''

3497 - def searchNeeded(self, keyword, like = False):

3498 if like: 3499 self.cursor.execute(""" 3500 SELECT needed.idpackage FROM needed,neededreference 3501 WHERE library LIKE (?) AND 3502 needed.idneeded = neededreference.idneeded""", (keyword,)) 3503 else: 3504 self.cursor.execute(""" 3505 SELECT needed.idpackage FROM needed,neededreference 3506 WHERE library = (?) AND 3507 needed.idneeded = neededreference.idneeded""", (keyword,)) 3508 return self.fetchall2set(self.cursor.fetchall())

3509 3510 ''' search dependency string inside dependenciesreference table and retrieve iddependency '''

3511 - def searchDependency(self, dep, like = False, multi = False, strings = False):

3512 sign = "=" 3513 if like: 3514 sign = "LIKE" 3515 dep = "%"+dep+"%" 3516 item = 'iddependency' 3517 if strings: 3518 item = 'dependency' 3519 self.cursor.execute('SELECT %s FROM dependenciesreference WHERE dependency %s (?)' % (item, sign,), (dep,)) 3520 if multi: 3521 return self.fetchall2set(self.cursor.fetchall()) 3522 else: 3523 iddep = self.cursor.fetchone() 3524 if iddep: 3525 iddep = iddep[0] 3526 else: 3527 iddep = -1 3528 return iddep

3529 3530 ''' search iddependency inside dependencies table and retrieve idpackages '''

3531 - def searchIdpackageFromIddependency(self, iddep):

3532 self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,)) 3533 return self.fetchall2set(self.cursor.fetchall())

3534

3535 - def searchSets(self, keyword):

3536 self.cursor.execute('SELECT DISTINCT(setname) FROM packagesets WHERE setname LIKE (?)', ("%"+keyword+"%",)) 3537 return self.fetchall2set(self.cursor.fetchall())

3538

3539 - def searchSimilarPackages(self, mystring, atom = False):

3540 s_item = 'name' 3541 if atom: s_item = 'atom' 3542 self.cursor.execute(""" 3543 SELECT idpackage FROM baseinfo 3544 WHERE soundex(%s) = soundex((?)) ORDER BY %s""" % (s_item, s_item,), (mystring,)) 3545 return self.fetchall2list(self.cursor.fetchall())

3546

3547 - def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None, order_by = 'atom', just_id = False):

3548 3549 searchkeywords = ["%"+keyword+"%"] 3550 slotstring = '' 3551 if slot: 3552 searchkeywords.append(slot) 3553 slotstring = ' and slot = (?)' 3554 tagstring = '' 3555 if tag: 3556 searchkeywords.append(tag) 3557 tagstring = ' and versiontag = (?)' 3558 branchstring = '' 3559 if branch: 3560 searchkeywords.append(branch) 3561 branchstring = ' and branch = (?)' 3562 order_by_string = '' 3563 if order_by in ("atom", "idpackage", "branch",): 3564 order_by_string = ' order by %s' % (order_by,) 3565 3566 search_elements = 'atom,idpackage,branch' 3567 if just_id: search_elements = 'idpackage' 3568 3569 if sensitive: 3570 self.cursor.execute(""" 3571 SELECT %s FROM baseinfo WHERE atom LIKE (?) %s %s %s %s""" % ( 3572 search_elements,slotstring,tagstring,branchstring,order_by_string,), 3573 searchkeywords 3574 ) 3575 else: 3576 self.cursor.execute(""" 3577 SELECT %s FROM baseinfo WHERE 3578 LOWER(atom) LIKE (?) %s %s %s %s""" % ( 3579 search_elements,slotstring,tagstring,branchstring,order_by_string,), 3580 searchkeywords 3581 ) 3582 if just_id: 3583 return self.fetchall2list(self.cursor.fetchall()) 3584 return self.cursor.fetchall()

3585

3586 - def searchProvide(self, keyword, slot = None, tag = None, branch = None, justid = False):

3587 3588 slotstring = '' 3589 searchkeywords = [keyword] 3590 if slot: 3591 searchkeywords.append(slot) 3592 slotstring = ' and baseinfo.slot = (?)' 3593 tagstring = '' 3594 if tag: 3595 searchkeywords.append(tag) 3596 tagstring = ' and baseinfo.versiontag = (?)' 3597 branchstring = '' 3598 if branch: 3599 searchkeywords.append(branch) 3600 branchstring = ' and baseinfo.branch = (?)' 3601 atomstring = '' 3602 if not justid: 3603 atomstring = 'baseinfo.atom,' 3604 3605 self.cursor.execute(""" 3606 SELECT %s baseinfo.idpackage FROM baseinfo,provide 3607 WHERE provide.atom = (?) AND 3608 provide.idpackage = baseinfo.idpackage %s %s %s""" % ( 3609 atomstring,slotstring,tagstring,branchstring,), 3610 searchkeywords 3611 ) 3612 3613 if justid: 3614 results = self.fetchall2list(self.cursor.fetchall()) 3615 else: 3616 results = self.cursor.fetchall() 3617 return results

3618

3619 - def searchPackagesByDescription(self, keyword):

3620 self.cursor.execute(""" 3621 SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo 3622 WHERE LOWER(extrainfo.description) LIKE (?) AND 3623 baseinfo.idpackage = extrainfo.idpackage""", ("%"+keyword.lower()+"%",)) 3624 return self.cursor.fetchall()

3625

3626 - def searchPackagesByName(self, keyword, sensitive = False, branch = None, justid = False):

3627 3628 if sensitive: 3629 searchkeywords = [keyword] 3630 else: 3631 searchkeywords = [keyword.lower()] 3632 branchstring = '' 3633 atomstring = '' 3634 if not justid: 3635 atomstring = 'atom,' 3636 if branch: 3637 searchkeywords.append(branch) 3638 branchstring = ' and branch = (?)' 3639 3640 if sensitive: 3641 self.cursor.execute(""" 3642 SELECT %s idpackage FROM baseinfo 3643 WHERE name = (?) %s""" % (atomstring, branchstring,), searchkeywords) 3644 else: 3645 self.cursor.execute(""" 3646 SELECT %s idpackage FROM baseinfo 3647 WHERE LOWER(name) = (?) %s""" % (atomstring, branchstring,), searchkeywords) 3648 3649 if justid: 3650 results = self.fetchall2list(self.cursor.fetchall()) 3651 else: 3652 results = self.cursor.fetchall() 3653 return results

3654 3655

3656 - def searchPackagesByCategory(self, keyword, like = False, branch = None):

3657 3658 searchkeywords = [keyword] 3659 branchstring = '' 3660 if branch: 3661 searchkeywords.append(branch) 3662 branchstring = 'and branch = (?)' 3663 3664 if like: 3665 self.cursor.execute(""" 3666 SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories 3667 WHERE categories.category LIKE (?) AND 3668 baseinfo.idcategory = categories.idcategory %s""" % (branchstring,), searchkeywords) 3669 else: 3670 self.cursor.execute(""" 3671 SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories 3672 WHERE categories.category = (?) AND 3673 baseinfo.idcategory = categories.idcategory %s""" % (branchstring,), searchkeywords) 3674 3675 return self.cursor.fetchall()

3676

3677 - def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None, justid = False):

3678 3679 myname = name 3680 mycat = category 3681 if not sensitive: 3682 myname = name.lower() 3683 mycat = category.lower() 3684 3685 searchkeywords = [myname, mycat] 3686 branchstring = '' 3687 if branch: 3688 searchkeywords.append(branch) 3689 branchstring = ' and branch = (?)' 3690 atomstring = '' 3691 if not justid: 3692 atomstring = 'atom,' 3693 3694 if sensitive: 3695 self.cursor.execute(""" 3696 SELECT %s idpackage FROM baseinfo 3697 WHERE name = (?) AND 3698 idcategory IN ( 3699 SELECT idcategory FROM categories 3700 WHERE category = (?) 3701 ) %s""" % (atomstring, branchstring,), searchkeywords) 3702 else: 3703 self.cursor.execute(""" 3704 SELECT %s idpackage FROM baseinfo 3705 WHERE LOWER(name) = (?) AND 3706 idcategory IN ( 3707 SELECT idcategory FROM categories 3708 WHERE LOWER(category) = (?) 3709 ) %s""" % (atomstring, branchstring,), searchkeywords) 3710 3711 if justid: 3712 results = self.fetchall2list(self.cursor.fetchall()) 3713 else: 3714 results = self.cursor.fetchall() 3715 return results

3716

3717 - def isPackageScopeAvailable(self, atom, slot, revision):

3718 searchdata = (atom, slot, revision,) 3719 self.cursor.execute('SELECT idpackage FROM baseinfo where atom = (?) and slot = (?) and revision = (?)', searchdata) 3720 rslt = self.cursor.fetchone() 3721 idreason = 0 3722 idpackage = -1 3723 if rslt: 3724 # check if it's masked 3725 idpackage, idreason = self.idpackageValidator(rslt[0]) 3726 return idpackage, idreason

3727

3728 - def isBranchMigrationAvailable(self, repository, from_branch, to_branch):

3729 """ 3730 Returns whether branch migration metadata given by the provided key 3731 (repository, from_branch, to_branch,) is available. 3732 3733 @param repository: repository identifier 3734 @type repository: string 3735 @param from_branch: original branch 3736 @type from_branch: string 3737 @param to_branch: destination branch 3738 @type to_branch: string 3739 @return: tuple composed by (1)post migration script md5sum and 3740 (2)post upgrade script md5sum 3741 @rtype: tuple 3742 """ 3743 self.cursor.execute(""" 3744 SELECT post_migration_md5sum, post_upgrade_md5sum 3745 FROM entropy_branch_migration 3746 WHERE repository = (?) AND from_branch = (?) AND to_branch = (?) 3747 """, (repository, from_branch, to_branch,)) 3748 return self.cursor.fetchone()

3749

3750 - def listAllPackages(self, get_scope = False, order_by = None, branch = None, branch_operator = "="):

3751 3752 branchstring = '' 3753 searchkeywords = [] 3754 if branch: 3755 searchkeywords = [branch] 3756 branchstring = ' where branch %s (?)' % (branch_operator,) 3757 3758 order_txt = '' 3759 if order_by: 3760 order_txt = ' order by %s' % (order_by,) 3761 if get_scope: 3762 self.cursor.execute('SELECT idpackage,atom,slot,revision FROM baseinfo'+order_txt+branchstring, searchkeywords) 3763 else: 3764 self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo'+order_txt+branchstring, searchkeywords) 3765 return self.cursor.fetchall()

3766

3767 - def listAllInjectedPackages(self, justFiles = False):

3768 self.cursor.execute('SELECT idpackage FROM injected') 3769 injecteds = self.fetchall2set(self.cursor.fetchall()) 3770 results = set() 3771 # get download 3772 for injected in injecteds: 3773 download = self.retrieveDownloadURL(injected) 3774 if justFiles: 3775 results.add(download) 3776 else: 3777 results.add((download, injected)) 3778 return results

3779

3780 - def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="):

3781 3782 branchstring = '' 3783 if branch: 3784 branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"' 3785 if onlycounters: 3786 self.cursor.execute('SELECT counter FROM counters'+branchstring) 3787 return self.fetchall2set(self.cursor.fetchall()) 3788 else: 3789 self.cursor.execute('SELECT counter,idpackage FROM counters'+branchstring) 3790 return self.cursor.fetchall()

3791

3792 - def listAllIdpackages(self, branch = None, branch_operator = "=", order_by = None):

3793 3794 branchstring = '' 3795 orderbystring = '' 3796 searchkeywords = [] 3797 if branch: 3798 searchkeywords.append(branch) 3799 branchstring = ' where branch %s (?)' % (str(branch_operator),) 3800 if order_by: 3801 orderbystring = ' order by '+order_by 3802 3803 self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring+orderbystring, searchkeywords) 3804 3805 try: 3806 if order_by: 3807 results = self.fetchall2list(self.cursor.fetchall()) 3808 else: 3809 results = self.fetchall2set(self.cursor.fetchall()) 3810 return results 3811 except self.dbapi2.OperationalError: 3812 if order_by: 3813 return [] 3814 return set()

3815

3816 - def listAllDependencies(self, only_deps = False):

3817 if only_deps: 3818 self.cursor.execute('SELECT dependency FROM dependenciesreference') 3819 return self.fetchall2set(self.cursor.fetchall()) 3820 else: 3821 self.cursor.execute('SELECT * FROM dependenciesreference') 3822 return self.cursor.fetchall()

3823

3824 - def listAllBranches(self):

3825 3826 cache = self.live_cache.get('listAllBranches') 3827 if cache != None: 3828 return cache 3829 3830 self.cursor.execute('SELECT distinct branch FROM baseinfo') 3831 results = self.fetchall2set(self.cursor.fetchall()) 3832 3833 self.live_cache['listAllBranches'] = results.copy() 3834 return results

3835

3836 - def listIdPackagesInIdcategory(self, idcategory, order_by = 'atom'):

3837 order_by_string = '' 3838 if order_by in ("atom", "name", "version",): 3839 order_by_string = ' ORDER BY %s' % (order_by,) 3840 self.cursor.execute('SELECT idpackage FROM baseinfo where idcategory = (?)'+order_by_string, (idcategory,)) 3841 return self.fetchall2set(self.cursor.fetchall())

3842

3843 - def listIdpackageDependencies(self, idpackage):

3844 self.cursor.execute(""" 3845 SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies 3846 WHERE dependencies.idpackage = (?) AND 3847 dependenciesreference.iddependency = dependencies.iddependency""", (idpackage,)) 3848 return set(self.cursor.fetchall())

3849

3850 - def listAllDownloads(self, do_sort = True, full_path = False):

3851 3852 order_string = '' 3853 if do_sort: 3854 order_string = 'ORDER BY extrainfo.download' 3855 self.cursor.execute(""" 3856 SELECT extrainfo.download FROM baseinfo,extrainfo 3857 WHERE baseinfo.idpackage = extrainfo.idpackage %s""" % (order_string,)) 3858 3859 if do_sort: 3860 results = self.fetchall2list(self.cursor.fetchall()) 3861 else: 3862 results = self.fetchall2set(self.cursor.fetchall()) 3863 3864 if not full_path: 3865 results = [os.path.basename(x) for x in results] 3866 3867 return results

3868

3869 - def listAllFiles(self, clean = False, count = False):

3870 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3871 if count: 3872 self.cursor.execute('SELECT count(file) FROM content') 3873 else: 3874 self.cursor.execute('SELECT file FROM content') 3875 if count: 3876 return self.cursor.fetchone()[0] 3877 else: 3878 if clean: 3879 return self.fetchall2set(self.cursor.fetchall()) 3880 else: 3881 return self.fetchall2list(self.cursor.fetchall())

3882

3883 - def listAllCategories(self, order_by = ''):

3884 order_by_string = '' 3885 if order_by: order_by_string = ' order by %s' % (order_by,) 3886 self.cursor.execute('SELECT idcategory,category FROM categories %s' % ( 3887 order_by_string,)) 3888 return self.cursor.fetchall()

3889

3890 - def listConfigProtectDirectories(self, mask = False):

3891 mask_t = '' 3892 if mask: mask_t = 'mask' 3893 self.cursor.execute(""" 3894 SELECT DISTINCT(protect) FROM configprotectreference 3895 WHERE idprotect >= 1 AND 3896 idprotect <= (SELECT max(idprotect) FROM configprotect%s) 3897 ORDER BY protect""" % (mask_t,)) 3898 results = self.fetchall2set(self.cursor.fetchall()) 3899 dirs = set() 3900 for mystr in results: 3901 dirs |= set(map(unicode, mystr.split())) 3902 return sorted(list(dirs))

3903

3904 - def switchBranch(self, idpackage, tobranch):

3905 with self.__write_mutex: 3906 self.cursor.execute(""" 3907 UPDATE baseinfo SET branch = (?) 3908 WHERE idpackage = (?)""", (tobranch, idpackage,)) 3909 self.commitChanges() 3910 self.clearCache() 3911

3912 - def databaseStructureUpdates(self):

3913 3914 old_readonly = self.readOnly 3915 self.readOnly = False 3916 3917 if not self.doesTableExist("licenses_accepted"): 3918 self.createLicensesAcceptedTable() 3919 3920 if not self.doesTableExist("installedtable"): 3921 self.createInstalledTable() 3922 3923 if not self.doesColumnInTableExist("installedtable", "source"): 3924 self.createInstalledTableSource() 3925 3926 if not self.doesTableExist('packagesets'): 3927 self.createPackagesetsTable() 3928 3929 if not self.doesTableExist('packagechangelogs'): 3930 self.createPackagechangelogsTable() 3931 3932 if not self.doesTableExist('automergefiles'): 3933 self.createAutomergefilesTable() 3934 3935 if not self.doesTableExist('packagesignatures'): 3936 self.createPackagesignaturesTable() 3937 3938 if not self.doesTableExist('packagespmphases'): 3939 self.createPackagespmphases() 3940 3941 if not self.doesTableExist('entropy_branch_migration'): 3942 self.createEntropyBranchMigrationTable() 3943 3944 if not self.doesTableExist('neededlibrarypaths'): 3945 self.createNeededlibrarypathsTable() 3946 if not self.doesColumnInTableExist("neededlibrarypaths", "elfclass"): 3947 self.createNeededlibrarypathsTable() 3948 3949 if not self.doesTableExist('neededlibraryidpackages'): 3950 self.createNeededlibraryidpackagesTable() 3951 elif not self.doesColumnInTableExist("neededlibraryidpackages", "elfclass"): 3952 self.createNeededlibraryidpackagesTable() 3953 3954 if not self.doesTableExist('dependstable'): 3955 self.createDependsTable() 3956 3957 self.readOnly = old_readonly 3958 self.connection.commit()

3959

3960 - def validateDatabase(self):

3961 self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table", "baseinfo")) 3962 rslt = self.cursor.fetchone() 3963 if rslt == None: 3964 mytxt = _("baseinfo table not found. Either does not exist or corrupted.") 3965 raise SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,)) 3966 self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table", "extrainfo")) 3967 rslt = self.cursor.fetchone() 3968 if rslt == None: 3969 mytxt = _("extrainfo table not found. Either does not exist or corrupted.") 3970 raise SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,))

3971

3972 - def getIdpackagesDifferences(self, foreign_idpackages):

3973 myids = self.listAllIdpackages() 3974 if isinstance(foreign_idpackages, (list, tuple,)): 3975 outids = set(foreign_idpackages) 3976 else: 3977 outids = foreign_idpackages 3978 added_ids = outids - myids 3979 removed_ids = myids - outids 3980 return added_ids, removed_ids

3981

3982 - def uniformBranch(self, branch):

3983 with self.__write_mutex: 3984 self.cursor.execute('UPDATE baseinfo SET branch = (?)', (branch,)) 3985 self.commitChanges() 3986 self.clearCache()

3987

3988 - def alignDatabases(self, dbconn, force = False, output_header = " ", align_limit = 300):

3989 3990 added_ids, removed_ids = self.getIdpackagesDifferences(dbconn.listAllIdpackages()) 3991 3992 if not force: 3993 if len(added_ids) > align_limit: # too much hassle 3994 return 0 3995 if len(removed_ids) > align_limit: # too much hassle 3996 return 0 3997 3998 if not added_ids and not removed_ids: 3999 return -1 4000 4001 mytxt = red("%s, %s ...") % (_("Syncing current database"), _("please wait"),) 4002 self.updateProgress( 4003 mytxt, 4004 importance = 1, 4005 type = "info", 4006 header = output_header, 4007 back = True 4008 ) 4009 maxcount = len(removed_ids) 4010 mycount = 0 4011 for idpackage in removed_ids: 4012 mycount += 1 4013 mytxt = "%s: %s" % (red(_("Removing entry")), blue(str(self.retrieveAtom(idpackage))),) 4014 self.updateProgress( 4015 mytxt, 4016 importance = 0, 4017 type = "info", 4018 header = output_header, 4019 back = True, 4020 count = (mycount, maxcount) 4021 ) 4022 self.removePackage(idpackage, do_cleanup = False, do_commit = False) 4023 4024 maxcount = len(added_ids) 4025 mycount = 0 4026 for idpackage in added_ids: 4027 mycount += 1 4028 mytxt = "%s: %s" % (red(_("Adding entry")), blue(str(dbconn.retrieveAtom(idpackage))),) 4029 self.updateProgress( 4030 mytxt, 4031 importance = 0, 4032 type = "info", 4033 header = output_header, 4034 back = True, 4035 count = (mycount, maxcount) 4036 ) 4037 mydata = dbconn.getPackageData(idpackage, get_content = True, content_insert_formatted = True) 4038 self.addPackage( 4039 mydata, 4040 revision = mydata['revision'], 4041 idpackage = idpackage, 4042 do_remove = False, 4043 do_commit = False, 4044 formatted_content = True 4045 ) 4046 4047 # do some cleanups 4048 self.doCleanups() 4049 # clear caches 4050 self.clearCache() 4051 self.commitChanges() 4052 self.regenerateDependsTable(output = False) 4053 dbconn.clearCache() 4054 4055 # verify both checksums, if they don't match, bomb out 4056 mycheck = self.database_checksum(do_order = True, strict = False) 4057 outcheck = dbconn.database_checksum(do_order = True, strict = False) 4058 if mycheck == outcheck: 4059 return 1 4060 return 0

4061

4062 - def checkDatabaseApi(self):

4063 4064 dbapi = self.getApi() 4065 if int(dbapi) > int(etpConst['etpapi']): 4066 self.updateProgress( 4067 red(_("Repository EAPI > Entropy EAPI. Please update Equo/Entropy as soon as possible !")), 4068 importance = 1, 4069 type = "warning", 4070 header = " * ! * ! * ! * " 4071 )

4072

4073 - def doDatabaseImport(self, dumpfile, dbfile):

4074 import subprocess 4075 sqlite3_exec = "/usr/bin/sqlite3 %s < %s" % (dbfile, dumpfile,) 4076 retcode = subprocess.call(sqlite3_exec, shell = True) 4077 return retcode

4078

4079 - def doDatabaseExport(self, dumpfile, gentle_with_tables = True, 4080 exclude_tables = None):

4081 4082 if not exclude_tables: 4083 exclude_tables = [] 4084 4085 dumpfile.write("BEGIN TRANSACTION;\n") 4086 self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'") 4087 for name, x, sql in self.cursor.fetchall(): 4088 4089 self.updateProgress( 4090 red("%s " % (_("Exporting database table"),) )+"["+blue(str(name))+"]", 4091 importance = 0, 4092 type = "info", 4093 back = True, 4094 header = " " 4095 ) 4096 4097 if name == "sqlite_sequence": 4098 dumpfile.write("DELETE FROM sqlite_sequence;\n") 4099 elif name == "sqlite_stat1": 4100 dumpfile.write("ANALYZE sqlite_master;\n") 4101 elif name.startswith("sqlite_"): 4102 continue 4103 else: 4104 t_cmd = "CREATE TABLE" 4105 if sql.startswith(t_cmd) and gentle_with_tables: 4106 sql = "CREATE TABLE IF NOT EXISTS"+sql[len(t_cmd):] 4107 dumpfile.write("%s;\n" % sql) 4108 4109 if name in exclude_tables: 4110 continue 4111 4112 self.cursor.execute("PRAGMA table_info('%s')" % name) 4113 cols = [str(r[1]) for r in self.cursor.fetchall()] 4114 q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES(" 4115 q += ", ".join(["'||quote(" + x + ")||'" for x in cols]) 4116 q += ")' FROM '%(tbl_name)s'" 4117 self.cursor.execute(q % {'tbl_name': name}) 4118 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 4119 for row in self.cursor: 4120 dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape'))) 4121 4122 self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'") 4123 for name, x, sql in self.cursor.fetchall(): 4124 dumpfile.write("%s;\n" % sql) 4125 4126 dumpfile.write("COMMIT;\n") 4127 try: 4128 dumpfile.flush() 4129 except: 4130 pass 4131 self.updateProgress( 4132 red(_("Database Export completed.")), 4133 importance = 0, 4134 type = "info", 4135 header = " " 4136 )

4137 # remember to close the file 4138

4139 - def listAllTables(self):

4140 self.cursor.execute(""" 4141 SELECT name FROM SQLITE_MASTER WHERE type = "table" 4142 """) 4143 return self.fetchall2list(self.cursor.fetchall())

4144

4145 - def doesTableExist(self, table):

4146 self.cursor.execute('select name from SQLITE_MASTER where type = "table" and name = (?)', (table,)) 4147 rslt = self.cursor.fetchone() 4148 if rslt == None: 4149 return False 4150 return True

4151

4152 - def doesColumnInTableExist(self, table, column):

4153 self.cursor.execute('PRAGMA table_info( %s )' % (table,)) 4154 rslt = (x[1] for x in self.cursor.fetchall()) 4155 if column in rslt: 4156 return True 4157 return False

4158

4159 - def database_checksum(self, do_order = False, strict = True, strings = False):

4160 4161 c_tup = ("database_checksum", do_order, strict, strings,) 4162 cache = self.live_cache.get(c_tup) 4163 if cache != None: return cache 4164 4165 idpackage_order = '' 4166 category_order = '' 4167 license_order = '' 4168 flags_order = '' 4169 if do_order: 4170 idpackage_order = 'order by idpackage' 4171 category_order = 'order by category' 4172 license_order = 'order by license' 4173 flags_order = 'order by chost' 4174 4175 def do_update_md5(m, cursor): 4176 mydata = cursor.fetchall() 4177 for record in mydata: 4178 for item in record: 4179 m.update(str(item))

4180 4181 if strings: 4182 import hashlib 4183 m = hashlib.md5() 4184 4185 self.cursor.execute(""" 4186 SELECT idpackage,atom,name,version,versiontag, 4187 revision,branch,slot,etpapi,trigger FROM 4188 baseinfo %s""" % (idpackage_order,)) 4189 if strings: 4190 do_update_md5(m, self.cursor) 4191 else: 4192 a_hash = hash(tuple(self.cursor.fetchall())) 4193 self.cursor.execute(""" 4194 SELECT idpackage,description,homepage, 4195 download,size,digest,datecreation FROM 4196 extrainfo %s""" % (idpackage_order,)) 4197 if strings: 4198 do_update_md5(m, self.cursor) 4199 else: 4200 b_hash = hash(tuple(self.cursor.fetchall())) 4201 self.cursor.execute('select category from categories %s' % (category_order,)) 4202 if strings: 4203 do_update_md5(m, self.cursor) 4204 else: 4205 c_hash = hash(tuple(self.cursor.fetchall())) 4206 d_hash = '0' 4207 e_hash = '0' 4208 if strict: 4209 self.cursor.execute('select * from licenses %s' % (license_order,)) 4210 if strings: 4211 do_update_md5(m, self.cursor) 4212 else: 4213 d_hash = hash(tuple(self.cursor.fetchall())) 4214 self.cursor.execute('select * from flags %s' % (flags_order,)) 4215 if strings: 4216 do_update_md5(m, self.cursor) 4217 else: 4218 e_hash = hash(tuple(self.cursor.fetchall())) 4219 4220 if strings: 4221 result = m.hexdigest() 4222 else: 4223 result = "%s:%s:%s:%s:%s" % (a_hash, b_hash, c_hash, d_hash, e_hash,) 4224 4225 self.live_cache[c_tup] = result[:] 4226 return result 4227 4228 4229 ######################################################## 4230 #### 4231 ## Client Database API / but also used by server part 4232 # 4233

4234 - def updateInstalledTableSource(self, idpackage, source):

4235 with self.__write_mutex: 4236 self.cursor.execute(""" 4237 UPDATE installedtable SET source = (?) WHERE idpackage = (?) 4238 """, (source, idpackage,))

4239

4240 - def addPackageToInstalledTable(self, idpackage, repoid, source = 0):

4241 with self.__write_mutex: 4242 self.cursor.execute('INSERT into installedtable VALUES (?,?,?)', 4243 (idpackage, repoid, source,))

4244 # self.commitChanges() 4245

4246 - def retrievePackageFromInstalledTable(self, idpackage):

4247 with self.__write_mutex: 4248 try: 4249 self.cursor.execute(""" 4250 SELECT repositoryname FROM installedtable 4251 WHERE idpackage = (?)""", (idpackage,)) 4252 return self.cursor.fetchone()[0] 4253 except (self.dbapi2.OperationalError,TypeError,): 4254 return 'Not available'

4255

4256 - def removePackageFromInstalledTable(self, idpackage):

4257 with self.__write_mutex: 4258 self.cursor.execute(""" 4259 DELETE FROM installedtable 4260 WHERE idpackage = (?)""", (idpackage,))

4261

4262 - def removePackageFromDependsTable(self, idpackage):

4263 with self.__write_mutex: 4264 try: 4265 self.cursor.execute('DELETE FROM dependstable WHERE idpackage = (?)', (idpackage,)) 4266 return 0 4267 except (self.dbapi2.OperationalError,): 4268 return 1 # need reinit

4269

4270 - def createDependsTable(self):

4271 with self.__write_mutex: 4272 self.cursor.executescript(""" 4273 CREATE TABLE IF NOT EXISTS dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER ); 4274 INSERT INTO dependstable VALUES (-1,-1); 4275 """) 4276 if self.indexing: 4277 self.cursor.execute('CREATE INDEX IF NOT EXISTS dependsindex_idpackage ON dependstable ( idpackage )') 4278 self.commitChanges()

4279

4280 - def sanitizeDependsTable(self):

4281 with self.__write_mutex: 4282 self.cursor.execute('DELETE FROM dependstable where iddependency = -1') 4283 self.commitChanges()

4284

4285 - def isDependsTableSane(self):

4286 try: 4287 self.cursor.execute('SELECT iddependency FROM dependstable WHERE iddependency = -1') 4288 except (self.dbapi2.OperationalError,): 4289 return False # table does not exist, please regenerate and re-run 4290 status = self.cursor.fetchone() 4291 if status: return False 4292 4293 self.cursor.execute('select count(*) from dependstable') 4294 dependstable_count = self.cursor.fetchone() 4295 if dependstable_count < 2: 4296 return False 4297 return True

4298

4299 - def createXpakTable(self):

4300 with self.__write_mutex: 4301 self.cursor.execute('CREATE TABLE xpakdata ( idpackage INTEGER PRIMARY KEY, data BLOB );') 4302 self.commitChanges()

4303

4304 - def storeXpakMetadata(self, idpackage, blob):

4305 with self.__write_mutex: 4306 self.cursor.execute('INSERT into xpakdata VALUES (?,?)', (int(idpackage), buffer(blob),)) 4307 self.commitChanges()

4308

4309 - def retrieveXpakMetadata(self, idpackage):

4310 try: 4311 self.cursor.execute('SELECT data from xpakdata where idpackage = (?)', (idpackage,)) 4312 mydata = self.cursor.fetchone() 4313 if not mydata: 4314 return "" 4315 return mydata[0] 4316 except: 4317 return ""

4318

4319 - def retrieveBranchMigration(self, to_branch):

4320 """ 4321 This method returns branch migration metadata stored in Entropy 4322 Client database (installed packages database). It is used to 4323 determine whether to run per-repository branch migration scripts. 4324 4325 @param to_branch: usually the current branch string 4326 @type to_branch: string 4327 @return: branch migration metadata contained in database 4328 @rtype: dict 4329 """ 4330 if not self.doesTableExist('entropy_branch_migration'): 4331 return None 4332 4333 self.cursor.execute(""" 4334 SELECT repository, from_branch, post_migration_md5sum, 4335 post_upgrade_md5sum FROM entropy_branch_migration WHERE to_branch = (?) 4336 """, (to_branch,)) 4337 4338 data = self.cursor.fetchall() 4339 meta = {} 4340 for repo, from_branch, post_migration_md5, post_upgrade_md5 in data: 4341 obj = meta.setdefault(repo, {}) 4342 obj[from_branch] = (post_migration_md5, post_upgrade_md5,) 4343 return meta

4344

4345 - def dropContent(self):

4346 with self.__write_mutex: 4347 self.cursor.execute('DELETE FROM content')

4348

4349 - def dropAllIndexes(self):

4350 self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"') 4351 indexes = self.fetchall2set(self.cursor.fetchall()) 4352 with self.__write_mutex: 4353 for index in indexes: 4354 if not index.startswith("sqlite"): 4355 self.cursor.execute('DROP INDEX IF EXISTS %s' % (index,))

4356

4357 - def listAllIndexes(self, only_entropy = True):

4358 self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"') 4359 indexes = self.fetchall2set(self.cursor.fetchall()) 4360 if not only_entropy: 4361 return indexes 4362 myindexes = set() 4363 for index in indexes: 4364 if index.startswith("sqlite"): 4365 continue 4366 myindexes.add(index) 4367 return myindexes

4368 4369

4370 - def createAllIndexes(self):

4371 self.createContentIndex() 4372 self.createBaseinfoIndex() 4373 self.createKeywordsIndex() 4374 self.createDependenciesIndex() 4375 self.createProvideIndex() 4376 self.createConflictsIndex() 4377 self.createExtrainfoIndex() 4378 self.createNeededIndex() 4379 self.createUseflagsIndex() 4380 self.createLicensedataIndex() 4381 self.createLicensesIndex() 4382 self.createConfigProtectReferenceIndex() 4383 self.createMessagesIndex() 4384 self.createSourcesIndex() 4385 self.createCountersIndex() 4386 self.createEclassesIndex() 4387 self.createCategoriesIndex() 4388 self.createCompileFlagsIndex() 4389 self.createPackagesetsIndex() 4390 self.createAutomergefilesIndex() 4391 self.createNeededlibrarypathsIndex() 4392 self.createNeededlibraryidpackagesIndex()

4393

4394 - def createPackagesetsIndex(self):

4395 if self.indexing: 4396 with self.__write_mutex: 4397 try: 4398 self.cursor.execute('CREATE INDEX IF NOT EXISTS packagesetsindex ON packagesets ( setname )') 4399 self.commitChanges() 4400 except self.dbapi2.OperationalError: 4401 pass

4402

4403 - def createNeededlibraryidpackagesIndex(self):

4404 if self.indexing: 4405 with self.__write_mutex: 4406 try: 4407 self.cursor.executescript(""" 4408 CREATE INDEX IF NOT EXISTS neededlibidpackages_library 4409 ON neededlibraryidpackages ( library ); 4410 CREATE INDEX IF NOT EXISTS neededlibidpackages_idpackage 4411 ON neededlibraryidpackages ( idpackage ); 4412 CREATE INDEX IF NOT EXISTS neededlibidpackages_lib_elf 4413 ON neededlibraryidpackages ( library, elfclass ); 4414 """) 4415 except self.dbapi2.OperationalError: 4416 pass

4417

4418 - def createNeededlibrarypathsIndex(self):

4419 if self.indexing: 4420 with self.__write_mutex: 4421 try: 4422 self.cursor.executescript(""" 4423 CREATE INDEX IF NOT EXISTS neededlibpaths_library 4424 ON neededlibrarypaths ( library ); 4425 CREATE INDEX IF NOT EXISTS neededlibpaths_elf 4426 ON neededlibrarypaths ( elfclass ); 4427 CREATE INDEX IF NOT EXISTS neededlibpaths_path 4428 ON neededlibrarypaths ( path ); 4429 CREATE INDEX IF NOT EXISTS neededlibpaths_library_elf 4430 ON neededlibrarypaths ( library, elfclass ); 4431 """) 4432 except self.dbapi2.OperationalError: 4433 pass

4434

4435 - def createAutomergefilesIndex(self):

4436 if self.indexing: 4437 with self.__write_mutex: 4438 try: 4439 self.cursor.executescript(""" 4440 CREATE INDEX IF NOT EXISTS automergefiles_idpackage 4441 ON automergefiles ( idpackage ); 4442 CREATE INDEX IF NOT EXISTS automergefiles_file_md5 4443 ON automergefiles ( configfile, md5 ); 4444 """) 4445 except self.dbapi2.OperationalError: 4446 pass

4447

4448 - def createNeededIndex(self):

4449 if self.indexing: 4450 with self.__write_mutex: 4451 self.cursor.executescript(""" 4452 CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library ); 4453 CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded ); 4454 CREATE INDEX IF NOT EXISTS neededindex_idpackage ON needed ( idpackage ); 4455 CREATE INDEX IF NOT EXISTS neededindex_elfclass ON needed ( elfclass ); 4456 """)

4457

4458 - def createMessagesIndex(self):

4459 if self.indexing: 4460 with self.__write_mutex: 4461 self.cursor.execute('CREATE INDEX IF NOT EXISTS messagesindex ON messages ( idpackage )')

4462

4463 - def createCompileFlagsIndex(self):

4464 if self.indexing: 4465 with self.__write_mutex: 4466 self.cursor.execute('CREATE INDEX IF NOT EXISTS flagsindex ON flags ( chost,cflags,cxxflags )')

4467

4468 - def createUseflagsIndex(self):

4469 if self.indexing: 4470 with self.__write_mutex: 4471 self.cursor.executescript(""" 4472 CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idpackage ON useflags ( idpackage ); 4473 CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idflag ON useflags ( idflag ); 4474 CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( flagname ); 4475 """)

4476

4477 - def dropContentIndex(self, only_file = False):

4478 with self.__write_mutex: 4479 self.cursor.execute("DROP INDEX IF EXISTS contentindex_file") 4480 if not only_file: 4481 self.cursor.executescript("DROP INDEX IF EXISTS contentindex_couple;")

4482

4483 - def createContentIndex(self):

4484 if self.indexing: 4485 with self.__write_mutex: 4486 if self.doesTableExist("content"): 4487 self.cursor.executescript(""" 4488 CREATE INDEX IF NOT EXISTS contentindex_couple ON content ( idpackage ); 4489 CREATE INDEX IF NOT EXISTS contentindex_file ON content ( file ); 4490 """)

4491

4492 - def createConfigProtectReferenceIndex(self):

4493 if self.indexing: 4494 with self.__write_mutex: 4495 self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )')

4496

4497 - def createBaseinfoIndex(self):

4498 if self.indexing: 4499 with self.__write_mutex: 4500 self.cursor.executescript(""" 4501 CREATE INDEX IF NOT EXISTS baseindex_atom ON baseinfo ( atom ); 4502 CREATE INDEX IF NOT EXISTS baseindex_branch_name ON baseinfo ( name,branch ); 4503 CREATE INDEX IF NOT EXISTS baseindex_branch_name_idcategory ON baseinfo ( name,idcategory,branch ); 4504 CREATE INDEX IF NOT EXISTS baseindex_idcategory ON baseinfo ( idcategory ); 4505 """)

4506

4507 - def createLicensedataIndex(self):

4508 if self.indexing: 4509 with self.__write_mutex: 4510 self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )')

4511

4512 - def createLicensesIndex(self):

4513 if self.indexing: 4514 with self.__write_mutex: 4515 self.cursor.execute('CREATE INDEX IF NOT EXISTS licensesindex ON licenses ( license )')

4516

4517 - def createCategoriesIndex(self):

4518 if self.indexing: 4519 with self.__write_mutex: 4520 self.cursor.execute('CREATE INDEX IF NOT EXISTS categoriesindex_category ON categories ( category )')

4521

4522 - def createKeywordsIndex(self):

4523 if self.indexing: 4524 with self.__write_mutex: 4525 self.cursor.executescript(""" 4526 CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname ); 4527 CREATE INDEX IF NOT EXISTS keywordsindex_idpackage ON keywords ( idpackage ); 4528 CREATE INDEX IF NOT EXISTS keywordsindex_idkeyword ON keywords ( idkeyword ); 4529 """)

4530

4531 - def createDependenciesIndex(self):

4532 if self.indexing: 4533 with self.__write_mutex: 4534 self.cursor.executescript(""" 4535 CREATE INDEX IF NOT EXISTS dependenciesindex_idpackage ON dependencies ( idpackage ); 4536 CREATE INDEX IF NOT EXISTS dependenciesindex_iddependency ON dependencies ( iddependency ); 4537 CREATE INDEX IF NOT EXISTS dependenciesreferenceindex_dependency ON dependenciesreference ( dependency ); 4538 """)

4539

4540 - def createCountersIndex(self):

4541 if self.indexing: 4542 with self.__write_mutex: 4543 self.cursor.executescript(""" 4544 CREATE INDEX IF NOT EXISTS countersindex_idpackage ON counters ( idpackage ); 4545 CREATE INDEX IF NOT EXISTS countersindex_counter ON counters ( counter ); 4546 """)

4547

4548 - def createSourcesIndex(self):

4549 if self.indexing: 4550 with self.__write_mutex: 4551 self.cursor.executescript(""" 4552 CREATE INDEX IF NOT EXISTS sourcesindex_idpackage ON sources ( idpackage ); 4553 CREATE INDEX IF NOT EXISTS sourcesindex_idsource ON sources ( idsource ); 4554 CREATE INDEX IF NOT EXISTS sourcesreferenceindex_source ON sourcesreference ( source ); 4555 """)

4556

4557 - def createProvideIndex(self):

4558 if self.indexing: 4559 with self.__write_mutex: 4560 self.cursor.executescript(""" 4561 CREATE INDEX IF NOT EXISTS provideindex_idpackage ON provide ( idpackage ); 4562 CREATE INDEX IF NOT EXISTS provideindex_atom ON provide ( atom ); 4563 """)

4564

4565 - def createConflictsIndex(self):

4566 if self.indexing: 4567 with self.__write_mutex: 4568 self.cursor.executescript(""" 4569 CREATE INDEX IF NOT EXISTS conflictsindex_idpackage ON conflicts ( idpackage ); 4570 CREATE INDEX IF NOT EXISTS conflictsindex_atom ON conflicts ( conflict ); 4571 """)

4572

4573 - def createExtrainfoIndex(self):

4574 if self.indexing: 4575 with self.__write_mutex: 4576 self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( description )') 4577 self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex_pkgindex ON extrainfo ( idpackage )')

4578

4579 - def createEclassesIndex(self):

4580 if self.indexing: 4581 with self.__write_mutex: 4582 self.cursor.executescript(""" 4583 CREATE INDEX IF NOT EXISTS eclassesindex_idpackage ON eclasses ( idpackage ); 4584 CREATE INDEX IF NOT EXISTS eclassesindex_idclass ON eclasses ( idclass ); 4585 CREATE INDEX IF NOT EXISTS eclassesreferenceindex_classname ON eclassesreference ( classname ); 4586 """)

4587

4588 - def regenerateCountersTable(self, vdb_path, output = False):

4589 4590 # this is necessary now, counters table should be empty 4591 self.cursor.execute("DELETE FROM counters;") 4592 # assign a counter to an idpackage 4593 myids = self.listAllIdpackages() 4594 counter_path = etpConst['spm']['xpak_entries']['counter'] 4595 for myid in myids: 4596 # get atom 4597 myatom = self.retrieveAtom(myid) 4598 mybranch = self.retrieveBranch(myid) 4599 myatom = self.entropyTools.remove_tag(myatom) 4600 myatomcounterpath = "%s%s/%s" % (vdb_path, myatom, counter_path,) 4601 if os.path.isfile(myatomcounterpath): 4602 try: 4603 with open(myatomcounterpath, "r") as f: 4604 counter = int(f.readline().strip()) 4605 except: 4606 if output: 4607 mytxt = "%s: %s: %s" % ( 4608 bold(_("ATTENTION")), 4609 red(_("cannot open Spm counter file for")), 4610 bold(myatom), 4611 ) 4612 self.updateProgress( 4613 mytxt, 4614 importance = 1, 4615 type = "warning" 4616 ) 4617 continue 4618 # insert id+counter 4619 with self.__write_mutex: 4620 try: 4621 self.cursor.execute( 4622 'INSERT into counters VALUES ' 4623 '(?,?,?)', ( counter, myid, mybranch ) 4624 ) 4625 except self.dbapi2.IntegrityError: 4626 if output: 4627 mytxt = "%s: %s: %s" % ( 4628 bold(_("ATTENTION")), 4629 red(_("counter for atom is duplicated, ignoring")), 4630 bold(myatom), 4631 ) 4632 self.updateProgress( 4633 mytxt, 4634 importance = 1, 4635 type = "warning" 4636 ) 4637 continue 4638 # don't trust counters, they might not be unique 4639 4640 self.commitChanges()

4641

4642 - def clearTreeupdatesEntries(self, repository):

4643 if not self.doesTableExist("treeupdates"): 4644 self.createTreeupdatesTable() 4645 # treeupdates 4646 with self.__write_mutex: 4647 self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,)) 4648 self.commitChanges()

4649

4650 - def resetTreeupdatesDigests(self):

4651 with self.__write_mutex: 4652 self.cursor.execute('UPDATE treeupdates SET digest = "-1"') 4653 self.commitChanges()

4654

4655 - def migrateCountersTable(self):

4656 with self.__write_mutex: 4657 self._migrateCountersTable()

4658

4659 - def _migrateCountersTable(self):

4660 self.cursor.executescript(""" 4661 DROP TABLE IF EXISTS counterstemp; 4662 CREATE TABLE counterstemp ( 4663 counter INTEGER, idpackage INTEGER, branch VARCHAR, 4664 PRIMARY KEY(idpackage,branch) 4665 ); 4666 INSERT INTO counterstemp (counter, idpackage, branch) 4667 SELECT counter, idpackage, branch FROM counters; 4668 DROP TABLE counters; 4669 ALTER TABLE counterstemp RENAME TO counters; 4670 """) 4671 self.commitChanges()

4672

4673 - def createNeededlibrarypathsTable(self):

4674 with self.__write_mutex: 4675 self.cursor.executescript(""" 4676 DROP TABLE IF EXISTS neededlibrarypaths; 4677 CREATE TABLE neededlibrarypaths ( 4678 library VARCHAR, 4679 path VARCHAR, 4680 elfclass INTEGER, 4681 PRIMARY KEY(library, path, elfclass) 4682 ); 4683 """)

4684

4685 - def createNeededlibraryidpackagesTable(self):

4686 with self.__write_mutex: 4687 self.cursor.executescript(""" 4688 DROP TABLE IF EXISTS neededlibraryidpackages; 4689 CREATE TABLE neededlibraryidpackages ( 4690 idpackage INTEGER, 4691 library VARCHAR, 4692 elfclass INTEGER 4693 ); 4694 """)

4695

4696 - def createInstalledTableSource(self):

4697 with self.__write_mutex: 4698 self.cursor.execute('ALTER TABLE installedtable ADD source INTEGER;') 4699 self.cursor.execute(""" 4700 UPDATE installedtable SET source = (?) 4701 """, (etpConst['install_sources']['unknown'],))

4702

4703 - def createPackagechangelogsTable(self):

4704 with self.__write_mutex: 4705 self.cursor.execute('CREATE TABLE packagechangelogs ( category VARCHAR, name VARCHAR, changelog BLOB, PRIMARY KEY (category, name));')

4706

4707 - def createAutomergefilesTable(self):

4708 with self.__write_mutex: 4709 self.cursor.execute('CREATE TABLE automergefiles ( idpackage INTEGER, configfile VARCHAR, md5 VARCHAR );')

4710

4711 - def createPackagesignaturesTable(self):

4712 with self.__write_mutex: 4713 self.cursor.execute('CREATE TABLE packagesignatures ( idpackage INTEGER PRIMARY KEY, sha1 VARCHAR, sha256 VARCHAR, sha512 VARCHAR );')

4714

4715 - def createPackagespmphases(self):

4716 with self.__write_mutex: 4717 self.cursor.execute(""" 4718 CREATE TABLE packagespmphases ( 4719 idpackage INTEGER PRIMARY KEY, 4720 phases VARCHAR 4721 ); 4722 """)

4723

4724 - def createEntropyBranchMigrationTable(self):

4725 with self.__write_mutex: 4726 self.cursor.execute(""" 4727 CREATE TABLE entropy_branch_migration ( 4728 repository VARCHAR, 4729 from_branch VARCHAR, 4730 to_branch VARCHAR, 4731 post_migration_md5sum VARCHAR, 4732 post_upgrade_md5sum VARCHAR, 4733 PRIMARY KEY (repository, from_branch, to_branch) 4734 ); 4735 """)

4736

4737 - def createPackagesetsTable(self):

4738 with self.__write_mutex: 4739 self.cursor.execute('CREATE TABLE packagesets ( setname VARCHAR, dependency VARCHAR );')

4740

4741 - def createCategoriesdescriptionTable(self):

4742 with self.__write_mutex: 4743 self.cursor.execute('CREATE TABLE categoriesdescription ( category VARCHAR, locale VARCHAR, description VARCHAR );')

4744

4745 - def createTreeupdatesTable(self):

4746 with self.__write_mutex: 4747 self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );')

4748

4749 - def createLicensedataTable(self):

4750 with self.__write_mutex: 4751 self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );')

4752

4753 - def createLicensesAcceptedTable(self):

4754 with self.__write_mutex: 4755 self.cursor.execute('CREATE TABLE licenses_accepted ( licensename VARCHAR UNIQUE );')

4756

4757 - def createInstalledTable(self):

4758 with self.__write_mutex: 4759 self.cursor.execute('DROP TABLE IF EXISTS installedtable;') 4760 self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR, source INTEGER );')

4761

4762 - def addDependsRelationToDependsTable(self, iterable):

4763 with self.__write_mutex: 4764 self.cursor.executemany('INSERT into dependstable VALUES (?,?)', iterable) 4765 if (self.entropyTools.is_user_in_entropy_group()) and \ 4766 (self.dbname.startswith(etpConst['serverdbid'])): 4767 # force commit even if readonly, this will allow to automagically fix dependstable server side 4768 self.connection.commit() # we don't care much about syncing the database since it's quite trivial

4769

4770 - def clearDependsTable(self):

4771 if not self.doesTableExist("dependstable"): 4772 return 4773 self.cursor.executescript(""" 4774 DELETE FROM dependstable; 4775 INSERT INTO dependstable VALUES (-1,-1); 4776 """)

4777

4778 - def regenerateDependsTable(self, output = True):

4779 4780 depends = self.listAllDependencies() 4781 count = 0 4782 total = len(depends) 4783 mydata = set() 4784 am = self.atomMatch 4785 up = self.updateProgress 4786 self.clearDependsTable() 4787 for iddep, atom in depends: 4788 count += 1 4789 4790 if output and ((count == 0) or (count % 150 == 0) or \ 4791 (count == total)): 4792 up( red("Resolving %s") % (atom,), importance = 0, 4793 type = "info", back = True, count = (count, total) 4794 ) 4795 4796 idpackage, rc = am(atom) 4797 if idpackage == -1: 4798 continue 4799 mydata.add((iddep, idpackage)) 4800 4801 if mydata: 4802 self.addDependsRelationToDependsTable(mydata) 4803 4804 # now validate dependstable 4805 self.sanitizeDependsTable()

4806

4807 - def regenerateLibrarypathsidpackageTable(self, output = True):

4808 4809 if output: 4810 self.updateProgress( 4811 "%s ..." % ( 4812 purple(_("Resolving libraries, please wait")), 4813 ), 4814 importance = 0, type = "info", back = True 4815 ) 4816 self.cursor.executescript(""" 4817 DELETE FROM neededlibraryidpackages; 4818 INSERT INTO neededlibraryidpackages (idpackage, library, elfclass) 4819 SELECT 4820 baseinfo.idpackage as idpackage, 4821 neededreference.library as library, 4822 neededlibrarypaths.elfclass as elfclass 4823 FROM 4824 baseinfo, neededlibrarypaths, needed, neededreference, content 4825 WHERE 4826 neededreference.idneeded = needed.idneeded AND 4827 needed.idpackage = content.idpackage AND 4828 baseinfo.idpackage = needed.idpackage AND 4829 neededlibrarypaths.library = neededreference.library AND 4830 neededlibrarypaths.elfclass = needed.elfclass AND 4831 content.file = neededlibrarypaths.path 4832 GROUP BY idpackage, library; 4833 4834 """) 4835 if output: 4836 self.updateProgress( 4837 "%s" % ( 4838 purple(_("Libraries solved, all fine")), 4839 ), 4840 importance = 0, type = "info" 4841 )

4842

4843 - def moveCountersToBranch(self, to_branch, from_branch = None):

4844 with self.__write_mutex: 4845 if from_branch is not None: 4846 self.cursor.execute('UPDATE counters SET branch = (?) WHERE branch = (?)', (to_branch, from_branch,)) 4847 else: 4848 self.cursor.execute('UPDATE counters SET branch = (?)', (to_branch,)) 4849 self.commitChanges() 4850 self.clearCache()

4851

4852 - def atomMatchFetchCache(self, *args):

4853 if self.xcache: 4854 cached = self.dumpTools.loadobj("%s/%s/%s" % (self.dbMatchCacheKey, self.dbname, hash(tuple(args)),)) 4855 if cached != None: return cached

4856

4857 - def atomMatchStoreCache(self, *args, **kwargs):

4858 if self.xcache: 4859 self.Cacher.push("%s/%s/%s" % ( 4860 self.dbMatchCacheKey,self.dbname,hash(tuple(args)),), 4861 kwargs.get('result') 4862 )

4863

4864 - def atomMatchValidateCache(self, cached_obj, multiMatch, extendedResults):

4865 4866 # time wasted for a reason 4867 data, rc = cached_obj 4868 if rc != 0: return cached_obj 4869 4870 if (not extendedResults) and (not multiMatch): 4871 if not self.isIDPackageAvailable(data): return None 4872 elif extendedResults and (not multiMatch): 4873 # ((idpackage,0,version,versiontag,revision,),0) 4874 if not self.isIDPackageAvailable(data[0]): return None 4875 elif extendedResults and multiMatch: 4876 # (set([(idpackage,0,version,version_tag,revision) for x in dbpkginfo]),0) 4877 idpackages = set([x[0] for x in data]) 4878 if not self.areIDPackagesAvailable(idpackages): return None 4879 elif (not extendedResults) and multiMatch: 4880 # (set([x[0] for x in dbpkginfo]),0) 4881 idpackages = set(data) 4882 if not self.areIDPackagesAvailable(idpackages): return None 4883 4884 return cached_obj

4885

4886 - def _idpackageValidator_live(self, idpackage, reponame):

4887 if (idpackage, reponame) in self.SystemSettings['live_packagemasking']['mask_matches']: 4888 # do not cache this 4889 return -1, self.SystemSettings['pkg_masking_reference']['user_live_mask'] 4890 elif (idpackage, reponame) in self.SystemSettings['live_packagemasking']['unmask_matches']: 4891 return idpackage, self.SystemSettings['pkg_masking_reference']['user_live_unmask']

4892

4893 - def _idpackageValidator_user_package_mask(self, idpackage, reponame, live):

4894 # check if user package.mask needs it masked 4895 4896 mykw = "%smask_ids" % (reponame,) 4897 user_package_mask_ids = self.SystemSettings.get(mykw) 4898 if not isinstance(user_package_mask_ids, (list, set,)): 4899 user_package_mask_ids = set() 4900 for atom in self.SystemSettings['mask']: 4901 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4902 if r != 0: 4903 continue 4904 user_package_mask_ids |= set(matches) 4905 self.SystemSettings[mykw] = user_package_mask_ids 4906 if idpackage in user_package_mask_ids: 4907 # sorry, masked 4908 myr = self.SystemSettings['pkg_masking_reference']['user_package_mask'] 4909 try: 4910 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4911 validator_cache[(idpackage, reponame, live)] = -1, myr 4912 except KeyError: # system settings client plugin not found 4913 pass 4914 return -1, myr

4915

4916 - def _idpackageValidator_user_package_unmask(self, idpackage, reponame, live):

4917 # see if we can unmask by just lookin into user package.unmask stuff -> self.SystemSettings['unmask'] 4918 mykw = "%sunmask_ids" % (reponame,) 4919 user_package_unmask_ids = self.SystemSettings.get(mykw) 4920 if not isinstance(user_package_unmask_ids, (list, set,)): 4921 user_package_unmask_ids = set() 4922 for atom in self.SystemSettings['unmask']: 4923 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4924 if r != 0: 4925 continue 4926 user_package_unmask_ids |= set(matches) 4927 self.SystemSettings[mykw] = user_package_unmask_ids 4928 if idpackage in user_package_unmask_ids: 4929 myr = self.SystemSettings['pkg_masking_reference']['user_package_unmask'] 4930 try: 4931 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4932 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4933 except KeyError: # system settings client plugin not found 4934 pass 4935 return idpackage, myr

4936

4937 - def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live):

4938 # check if repository packages.db.mask needs it masked 4939 repos_mask = {} 4940 client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin'] 4941 client_settings = self.SystemSettings.get(client_plg_id, {}) 4942 if client_settings: 4943 repos_mask = client_settings['repositories']['mask'] 4944 repomask = repos_mask.get(reponame) 4945 if isinstance(repomask, (list, set,)): 4946 # first, seek into generic masking, all branches 4947 mask_repo_id = "%s_ids@@:of:%s" % (reponame, reponame,) # avoid issues with repository names 4948 repomask_ids = repos_mask.get(mask_repo_id) 4949 if not isinstance(repomask_ids, set): 4950 repomask_ids = set() 4951 for atom in repomask: 4952 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4953 if r != 0: 4954 continue 4955 repomask_ids |= set(matches) 4956 repos_mask[mask_repo_id] = repomask_ids 4957 if idpackage in repomask_ids: 4958 myr = self.SystemSettings['pkg_masking_reference']['repository_packages_db_mask'] 4959 try: 4960 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4961 validator_cache[(idpackage, reponame, live)] = -1, myr 4962 except KeyError: # system settings client plugin not found 4963 pass 4964 return -1, myr

4965

4966 - def _idpackageValidator_package_license_mask(self, idpackage, reponame, live):

4967 if self.SystemSettings['license_mask']: 4968 mylicenses = self.retrieveLicense(idpackage) 4969 mylicenses = mylicenses.strip().split() 4970 lic_mask = self.SystemSettings['license_mask'] 4971 for mylicense in mylicenses: 4972 if mylicense not in lic_mask: continue 4973 myr = self.SystemSettings['pkg_masking_reference']['user_license_mask'] 4974 try: 4975 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4976 validator_cache[(idpackage, reponame, live)] = -1, myr 4977 except KeyError: # system settings client plugin not found 4978 pass 4979 return -1, myr

4980

4981 - def _idpackageValidator_keyword_mask(self, idpackage, reponame, live):

4982 4983 mykeywords = self.retrieveKeywords(idpackage) 4984 # WORKAROUND for buggy entries 4985 if not mykeywords: mykeywords = [''] # ** is fine then 4986 # firstly, check if package keywords are in etpConst['keywords'] 4987 # (universal keywords have been merged from package.mask) 4988 for key in etpConst['keywords']: 4989 if key not in mykeywords: continue 4990 myr = self.SystemSettings['pkg_masking_reference']['system_keyword'] 4991 try: 4992 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4993 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4994 except KeyError: # system settings client plugin not found 4995 pass 4996 return idpackage, myr 4997 4998 # if we get here, it means we didn't find mykeywords in etpConst['keywords'] 4999 # we need to seek self.SystemSettings['keywords'] 5000 # seek in repository first 5001 if reponame in self.SystemSettings['keywords']['repositories']: 5002 for keyword in self.SystemSettings['keywords']['repositories'][reponame]: 5003 if keyword not in mykeywords: continue 5004 keyword_data = self.SystemSettings['keywords']['repositories'][reponame].get(keyword) 5005 if not keyword_data: continue 5006 if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok 5007 myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords_all'] 5008 try: 5009 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 5010 validator_cache[(idpackage, reponame, live)] = idpackage, myr 5011 except KeyError: # system settings client plugin not found 5012 pass 5013 return idpackage, myr 5014 kwd_key = "%s_ids" % (keyword,) 5015 keyword_data_ids = self.SystemSettings['keywords']['repositories'][reponame].get(kwd_key) 5016 if not isinstance(keyword_data_ids, set): 5017 keyword_data_ids = set() 5018 for atom in keyword_data: 5019 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 5020 if r != 0: 5021 continue 5022 keyword_data_ids |= matches 5023 self.SystemSettings['keywords']['repositories'][reponame][kwd_key] = keyword_data_ids 5024 if idpackage in keyword_data_ids: 5025 myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords'] 5026 try: 5027 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 5028 validator_cache[(idpackage, reponame, live)] = idpackage, myr 5029 except KeyError: # system settings client plugin not found 5030 pass 5031 return idpackage, myr 5032 5033 # if we get here, it means we didn't find a match in repositories 5034 # so we scan packages, last chance 5035 for keyword in self.SystemSettings['keywords']['packages']: 5036 # first of all check if keyword is in mykeywords 5037 if keyword not in mykeywords: continue 5038 keyword_data = self.SystemSettings['keywords']['packages'].get(keyword) 5039 if not keyword_data: continue 5040 kwd_key = "%s_ids" % (keyword,) 5041 keyword_data_ids = self.SystemSettings['keywords']['packages'].get(reponame+kwd_key) 5042 if not isinstance(keyword_data_ids, (list, set,)): 5043 keyword_data_ids = set() 5044 for atom in keyword_data: 5045 # match atom 5046 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 5047 if r != 0: 5048 continue 5049 keyword_data_ids |= matches 5050 self.SystemSettings['keywords']['packages'][reponame+kwd_key] = keyword_data_ids 5051 if idpackage in keyword_data_ids: 5052 # valid! 5053 myr = self.SystemSettings['pkg_masking_reference']['user_package_keywords'] 5054 try: 5055 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 5056 validator_cache[(idpackage, reponame, live)] = idpackage, myr 5057 except KeyError: # system settings client plugin not found 5058 pass 5059 return idpackage, myr

5060 5061 5062 5063 # function that validate one atom by reading keywords settings 5064 # validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']

5065 - def idpackageValidator(self, idpackage, live = True):

5066 5067 if self.dbname == etpConst['clientdbid']: 5068 return idpackage, 0 5069 elif self.dbname.startswith(etpConst['serverdbid']): 5070 return idpackage, 0 5071 5072 reponame = self.dbname[len(etpConst['dbnamerepoprefix']):] 5073 try: 5074 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 5075 cached = validator_cache.get((idpackage, reponame, live)) 5076 if cached != None: 5077 return cached 5078 # avoid memleaks 5079 if len(validator_cache) > 10000: 5080 validator_cache.clear() 5081 except KeyError: # plugin does not exist 5082 pass 5083 5084 if live: 5085 data = self._idpackageValidator_live(idpackage, reponame) 5086 if data: return data 5087 5088 data = self._idpackageValidator_user_package_mask(idpackage, reponame, live) 5089 if data: return data 5090 5091 data = self._idpackageValidator_user_package_unmask(idpackage, reponame, live) 5092 if data: return data 5093 5094 data = self._idpackageValidator_packages_db_mask(idpackage, reponame, live) 5095 if data: return data 5096 5097 data = self._idpackageValidator_package_license_mask(idpackage, reponame, live) 5098 if data: return data 5099 5100 data = self._idpackageValidator_keyword_mask(idpackage, reponame, live) 5101 if data: return data 5102 5103 # holy crap, can't validate 5104 myr = self.SystemSettings['pkg_masking_reference']['completely_masked'] 5105 validator_cache[(idpackage, reponame, live)] = -1, myr 5106 return -1, myr

5107 5108 # packages filter used by atomMatch, input must me foundIDs, a list like this: 5109 # [608,1867]

5110 - def packagesFilter(self, results):

5111 # keywordsFilter ONLY FILTERS results if 5112 # self.dbname.startswith(etpConst['dbnamerepoprefix']) => repository database is open 5113 if not self.dbname.startswith(etpConst['dbnamerepoprefix']): 5114 return results 5115 5116 newresults = set() 5117 for idpackage in results: 5118 idpackage, reason = self.idpackageValidator(idpackage) 5119 if idpackage == -1: continue 5120 newresults.add(idpackage) 5121 return newresults

5122

5123 - def __filterSlot(self, idpackage, slot):

5124 if slot == None: 5125 return idpackage 5126 dbslot = self.retrieveSlot(idpackage) 5127 if str(dbslot) == str(slot): 5128 return idpackage

5129

5130 - def __filterTag(self, idpackage, tag, operators):

5131 if tag == None: 5132 return idpackage 5133 dbtag = self.retrieveVersionTag(idpackage) 5134 compare = cmp(tag, dbtag) 5135 if not operators or operators == "=": 5136 if compare == 0: 5137 return idpackage 5138 else: 5139 return self.__do_operator_compare(idpackage, operators, compare)

5140

5141 - def __filterUse(self, idpackage, use):

5142 if not use: 5143 return idpackage 5144 pkguse = self.retrieveUseflags(idpackage) 5145 disabled = set([x[1:] for x in use if x.startswith("-")]) 5146 enabled = set([x for x in use if not x.startswith("-")]) 5147 enabled_not_satisfied = enabled - pkguse 5148 # check enabled 5149 if enabled_not_satisfied: 5150 return None 5151 # check disabled 5152 disabled_not_satisfied = disabled - pkguse 5153 if len(disabled_not_satisfied) != len(disabled): 5154 return None 5155 return idpackage

5156

5157 - def __do_operator_compare(self, token, operators, compare):

5158 if operators == ">" and compare == -1: 5159 return token 5160 elif operators == ">=" and compare < 1: 5161 return token 5162 elif operators == "<" and compare == 1: 5163 return token 5164 elif operators == "<=" and compare > -1: 5165 return token

5166

5167 - def __filterSlotTagUse(self, foundIDs, slot, tag, use, operators):

5168 5169 def myfilter(idpackage): 5170 5171 idpackage = self.__filterSlot(idpackage, slot) 5172 if not idpackage: 5173 return False 5174 5175 idpackage = self.__filterUse(idpackage, use) 5176 if not idpackage: 5177 return False 5178 5179 idpackage = self.__filterTag(idpackage, tag, operators) 5180 if not idpackage: 5181 return False 5182 5183 return True

5184 5185 return set(filter(myfilter, foundIDs)) 5186

5187 - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, 5188 matchBranches = (), matchTag = None, matchUse = (), packagesFilter = True, 5189 matchRevision = None, extendedResults = False, useCache = True ):

5190 5191 """ 5192 5193 @description: matches the user chosen package name+ver, if possibile, in a single repository 5194 @input atom: string, atom to match 5195 @input caseSensitive: bool, should the atom be parsed case sensitive? 5196 @input matchSlot: string, match atoms with the provided slot 5197 @input multiMatch: bool, return all the available atoms 5198 @input matchBranches: tuple or list, match packages only in the specified branches 5199 @input matchTag: match packages only for the specified tag 5200 @input matchUse: match packages only if it owns the specified use flags 5201 @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter 5202 @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error 5203 5204 """ 5205 5206 if not atom: 5207 return -1, 1 5208 5209 if useCache: 5210 cached = self.atomMatchFetchCache( 5211 atom, caseSensitive, matchSlot, 5212 multiMatch, matchBranches, matchTag, 5213 matchUse, packagesFilter, matchRevision, 5214 extendedResults 5215 ) 5216 if isinstance(cached, tuple): 5217 try: 5218 cached = self.atomMatchValidateCache(cached, multiMatch, extendedResults) 5219 except (TypeError, ValueError, IndexError, KeyError,): 5220 cached = None 5221 if isinstance(cached, tuple): 5222 return cached 5223 5224 atomTag = self.entropyTools.dep_gettag(atom) 5225 try: 5226 atomUse = self.entropyTools.dep_getusedeps(atom) 5227 except InvalidAtom: 5228 atomUse = () 5229 atomSlot = self.entropyTools.dep_getslot(atom) 5230 atomRev = self.entropyTools.dep_get_entropy_revision(atom) 5231 if isinstance(atomRev, (int, long,)): 5232 if atomRev < 0: atomRev = None 5233 5234 # use match 5235 scan_atom = self.entropyTools.remove_usedeps(atom) 5236 if (not matchUse) and (atomUse): 5237 matchUse = atomUse 5238 5239 # tag match 5240 scan_atom = self.entropyTools.remove_tag(scan_atom) 5241 if (matchTag == None) and (atomTag != None): 5242 matchTag = atomTag 5243 5244 # slot match 5245 scan_atom = self.entropyTools.remove_slot(scan_atom) 5246 if (matchSlot == None) and (atomSlot != None): 5247 matchSlot = atomSlot 5248 5249 # revision match 5250 scan_atom = self.entropyTools.remove_entropy_revision(scan_atom) 5251 if (matchRevision == None) and (atomRev != None): 5252 matchRevision = atomRev 5253 5254 branch_list = () 5255 direction = '' 5256 justname = True 5257 pkgkey = '' 5258 strippedAtom = '' 5259 foundIDs = [] 5260 dbpkginfo = set() 5261 5262 if scan_atom: 5263 5264 while 1: 5265 pkgversion = '' 5266 # check for direction 5267 strippedAtom = self.entropyTools.dep_getcpv(scan_atom) 5268 if scan_atom[-1] == "*": 5269 strippedAtom += "*" 5270 direction = scan_atom[0:len(scan_atom)-len(strippedAtom)] 5271 5272 justname = self.entropyTools.isjustname(strippedAtom) 5273 pkgkey = strippedAtom 5274 if justname == 0: 5275 # get version 5276 data = self.entropyTools.catpkgsplit(strippedAtom) 5277 if data == None: break # badly formatted 5278 pkgversion = data[2]+"-"+data[3] 5279 pkgkey = self.entropyTools.dep_getkey(strippedAtom) 5280 5281 splitkey = pkgkey.split("/") 5282 if (len(splitkey) == 2): 5283 pkgname = splitkey[1] 5284 pkgcat = splitkey[0] 5285 else: 5286 pkgname = splitkey[0] 5287 pkgcat = "null" 5288 5289 branch_list = (self.db_branch,) 5290 if matchBranches: 5291 # force to tuple for security 5292 branch_list = tuple(matchBranches) 5293 break 5294 5295 5296 if branch_list: 5297 # IDs found in the database that match our search 5298 foundIDs = self.__generate_found_ids_match(branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch) 5299 5300 ### FILTERING 5301 # filter slot and tag 5302 if foundIDs: 5303 foundIDs = self.__filterSlotTagUse(foundIDs, matchSlot, matchTag, matchUse, direction) 5304 if packagesFilter: 5305 foundIDs = self.packagesFilter(foundIDs) 5306 ### END FILTERING 5307 5308 if foundIDs: 5309 dbpkginfo = self.__handle_found_ids_match(foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, pkgversion) 5310 5311 if not dbpkginfo: 5312 if extendedResults: 5313 if multiMatch: 5314 x = set() 5315 else: 5316 x = (-1, 1, None, None, None,) 5317 self.atomMatchStoreCache( 5318 atom, caseSensitive, matchSlot, 5319 multiMatch, matchBranches, matchTag, 5320 matchUse, packagesFilter, matchRevision, 5321 extendedResults, result = (x, 1) 5322 ) 5323 return x, 1 5324 else: 5325 if multiMatch: 5326 x = set() 5327 else: 5328 x = -1 5329 self.atomMatchStoreCache( 5330 atom, caseSensitive, matchSlot, 5331 multiMatch, matchBranches, matchTag, 5332 matchUse, packagesFilter, matchRevision, 5333 extendedResults, result = (x, 1) 5334 ) 5335 return x, 1 5336 5337 if multiMatch: 5338 if extendedResults: 5339 x = set([(x[0], 0, x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) for x in dbpkginfo]) 5340 self.atomMatchStoreCache( 5341 atom, caseSensitive, matchSlot, 5342 multiMatch, matchBranches, matchTag, 5343 matchUse, packagesFilter, matchRevision, 5344 extendedResults, result = (x, 0) 5345 ) 5346 return x, 0 5347 else: 5348 x = set([x[0] for x in dbpkginfo]) 5349 self.atomMatchStoreCache( 5350 atom, caseSensitive, matchSlot, 5351 multiMatch, matchBranches, matchTag, 5352 matchUse, packagesFilter, matchRevision, 5353 extendedResults, result = (x, 0) 5354 ) 5355 return x, 0 5356 5357 if len(dbpkginfo) == 1: 5358 x = dbpkginfo.pop() 5359 if extendedResults: 5360 x = (x[0], 0, x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) 5361 self.atomMatchStoreCache( 5362 atom, caseSensitive, matchSlot, 5363 multiMatch, matchBranches, matchTag, 5364 matchUse, packagesFilter, matchRevision, 5365 extendedResults, result = (x, 0) 5366 ) 5367 return x, 0 5368 else: 5369 self.atomMatchStoreCache( 5370 atom, caseSensitive, matchSlot, 5371 multiMatch, matchBranches, matchTag, 5372 matchUse, packagesFilter, matchRevision, 5373 extendedResults, result = (x[0], 0) 5374 ) 5375 return x[0], 0 5376 5377 dbpkginfo = list(dbpkginfo) 5378 pkgdata = {} 5379 versions = set() 5380 for x in dbpkginfo: 5381 info_tuple = (x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) 5382 versions.add(info_tuple) 5383 pkgdata[info_tuple] = x[0] 5384 newer = self.entropyTools.get_entropy_newer_version(list(versions))[0] 5385 x = pkgdata[newer] 5386 if extendedResults: 5387 x = (x, 0, newer[0], newer[1], newer[2]) 5388 self.atomMatchStoreCache( 5389 atom, caseSensitive, matchSlot, 5390 multiMatch, matchBranches, matchTag, 5391 matchUse, packagesFilter, matchRevision, 5392 extendedResults, result = (x, 0) 5393 ) 5394 return x, 0 5395 else: 5396 self.atomMatchStoreCache( 5397 atom, caseSensitive, matchSlot, 5398 multiMatch, matchBranches, matchTag, 5399 matchUse, packagesFilter, matchRevision, 5400 extendedResults, result = (x, 0) 5401 ) 5402 return x, 0

5403

5404 - def __generate_found_ids_match(self, branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch):

5405 foundIDs = set() 5406 for idx in branch_list: 5407 5408 if pkgcat == "null": 5409 results = self.searchPackagesByName( 5410 pkgname, sensitive = caseSensitive, 5411 branch = idx, justid = True 5412 ) 5413 else: 5414 results = self.searchPackagesByNameAndCategory( 5415 name = pkgname, category = pkgcat, branch = idx, 5416 sensitive = caseSensitive, justid = True 5417 ) 5418 5419 mypkgcat = pkgcat 5420 mypkgname = pkgname 5421 virtual = False 5422 # if it's a PROVIDE, search with searchProvide 5423 # there's no package with that name 5424 if (not results) and (mypkgcat == "virtual"): 5425 virtuals = self.searchProvide(pkgkey, branch = idx, justid = True) 5426 if virtuals: 5427 virtual = True 5428 mypkgname = self.retrieveName(virtuals[0]) 5429 mypkgcat = self.retrieveCategory(virtuals[0]) 5430 results = virtuals 5431 5432 # now validate 5433 if not results: 5434 continue # search into a stabler branch 5435 5436 elif (len(results) > 1): 5437 5438 # if it's because category differs, it's a problem 5439 foundCat = None 5440 cats = set() 5441 for idpackage in results: 5442 cat = self.retrieveCategory(idpackage) 5443 cats.add(cat) 5444 if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): 5445 # in case of virtual packages only (that they're not stored as provide) 5446 foundCat = cat 5447 5448 # if we found something at least... 5449 if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual", "null")): 5450 foundCat = sorted(cats)[0] 5451 5452 if not foundCat: 5453 # got the issue 5454 continue 5455 5456 # we can use foundCat 5457 mypkgcat = foundCat 5458 5459 # we need to search using the category 5460 if (not multiMatch) and (pkgcat == "null" or virtual): 5461 # we searched by name, we need to search using category 5462 results = self.searchPackagesByNameAndCategory( 5463 name = mypkgname, category = mypkgcat, 5464 branch = idx, sensitive = caseSensitive, justid = True 5465 ) 5466 5467 # validate again 5468 if not results: 5469 continue # search into another branch 5470 5471 # if we get here, we have found the needed IDs 5472 foundIDs |= set(results) 5473 break 5474 5475 else: 5476 5477 idpackage = results[0] 5478 # if mypkgcat is virtual, we can force 5479 if (mypkgcat == "virtual") and (not virtual): 5480 # in case of virtual packages only (that they're not stored as provide) 5481 mypkgcat = self.retrieveCategory(idpackage) 5482 5483 # check if category matches 5484 if mypkgcat != "null": 5485 foundCat = self.retrieveCategory(idpackage) 5486 if mypkgcat == foundCat: 5487 foundIDs.add(idpackage) 5488 continue 5489 foundIDs.add(idpackage) 5490 break 5491 5492 return foundIDs

5493 5494

5495 - def __handle_found_ids_match(self, foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, pkgversion):

5496 5497 dbpkginfo = set() 5498 # now we have to handle direction 5499 if ((direction) or ((not direction) and (not justname)) or ((not direction) and (not justname) and strippedAtom.endswith("*"))) and foundIDs: 5500 5501 if (not justname) and \ 5502 ((direction == "~") or (direction == "=") or \ 5503 (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*"))): 5504 # any revision within the version specified OR the specified version 5505 5506 if (direction == '' and not justname): 5507 direction = "=" 5508 5509 # remove gentoo revision (-r0 if none) 5510 if (direction == "="): 5511 if (pkgversion.split("-")[-1] == "r0"): 5512 pkgversion = self.entropyTools.remove_revision(pkgversion) 5513 if (direction == "~"): 5514 pkgrevision = self.entropyTools.dep_get_portage_revision(pkgversion) 5515 pkgversion = self.entropyTools.remove_revision(pkgversion) 5516 5517 for idpackage in foundIDs: 5518 5519 dbver = self.retrieveVersion(idpackage) 5520 if (direction == "~"): 5521 myrev = self.entropyTools.dep_get_portage_revision(dbver) 5522 myver = self.entropyTools.remove_revision(dbver) 5523 if myver == pkgversion and pkgrevision <= myrev: 5524 # found 5525 dbpkginfo.add((idpackage, dbver)) 5526 else: 5527 # media-libs/test-1.2* support 5528 if pkgversion[-1] == "*": 5529 if dbver.startswith(pkgversion[:-1]): 5530 dbpkginfo.add((idpackage, dbver)) 5531 elif (matchRevision != None) and (pkgversion == dbver): 5532 dbrev = self.retrieveRevision(idpackage) 5533 if dbrev == matchRevision: 5534 dbpkginfo.add((idpackage, dbver)) 5535 elif (pkgversion == dbver) and (matchRevision == None): 5536 dbpkginfo.add((idpackage, dbver)) 5537 5538 elif (direction.find(">") != -1) or (direction.find("<") != -1): 5539 5540 if not justname: 5541 5542 # remove revision (-r0 if none) 5543 if pkgversion.endswith("r0"): 5544 # remove 5545 self.entropyTools.remove_revision(pkgversion) 5546 5547 for idpackage in foundIDs: 5548 5549 revcmp = 0 5550 tagcmp = 0 5551 if matchRevision != None: 5552 dbrev = self.retrieveRevision(idpackage) 5553 revcmp = cmp(matchRevision, dbrev) 5554 if matchTag != None: 5555 dbtag = self.retrieveVersionTag(idpackage) 5556 tagcmp = cmp(matchTag, dbtag) 5557 dbver = self.retrieveVersion(idpackage) 5558 pkgcmp = self.entropyTools.compare_versions(pkgversion, dbver) 5559 if pkgcmp == None: 5560 import warnings 5561 warnings.warn("WARNING, invalid version string stored in %s: %s <-> %s" % (self.dbname, pkgversion, dbver,)) 5562 continue 5563 if direction == ">": 5564 if pkgcmp < 0: 5565 dbpkginfo.add((idpackage, dbver)) 5566 elif (matchRevision != None) and pkgcmp <= 0 and revcmp < 0: 5567 dbpkginfo.add((idpackage, dbver)) 5568 elif (matchTag != None) and tagcmp < 0: 5569 dbpkginfo.add((idpackage, dbver)) 5570 elif direction == "<": 5571 if pkgcmp > 0: 5572 dbpkginfo.add((idpackage, dbver)) 5573 elif (matchRevision != None) and pkgcmp >= 0 and revcmp > 0: 5574 dbpkginfo.add((idpackage, dbver)) 5575 elif (matchTag != None) and tagcmp > 0: 5576 dbpkginfo.add((idpackage, dbver)) 5577 elif direction == ">=": 5578 if (matchRevision != None) and pkgcmp <= 0: 5579 if pkgcmp == 0: 5580 if revcmp <= 0: 5581 dbpkginfo.add((idpackage, dbver)) 5582 else: 5583 dbpkginfo.add((idpackage, dbver)) 5584 elif pkgcmp <= 0 and matchRevision == None: 5585 dbpkginfo.add((idpackage, dbver)) 5586 elif (matchTag != None) and tagcmp <= 0: 5587 dbpkginfo.add((idpackage, dbver)) 5588 elif direction == "<=": 5589 if (matchRevision != None) and pkgcmp >= 0: 5590 if pkgcmp == 0: 5591 if revcmp >= 0: 5592 dbpkginfo.add((idpackage, dbver)) 5593 else: 5594 dbpkginfo.add((idpackage, dbver)) 5595 elif pkgcmp >= 0 and matchRevision == None: 5596 dbpkginfo.add((idpackage, dbver)) 5597 elif (matchTag != None) and tagcmp >= 0: 5598 dbpkginfo.add((idpackage, dbver)) 5599 5600 else: # just the key 5601 5602 dbpkginfo = set([(x, self.retrieveVersion(x),) for x in foundIDs]) 5603 5604 return dbpkginfo

5605