entropy.db

Source Code for Module entropy.db

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework repository database module}. 10 Entropy repositories (server and client) are implemented as relational 11 databases. Currently, LocalRepository class is the object that wraps 12 sqlite3 database queries and repository logic: there are no more 13 abstractions between the two because there is only one implementation 14 available at this time. In future, entropy.db will feature more backends 15 such as MySQL embedded, SparQL, remote repositories support via TCP socket, 16 etc. This will require a new layer between the repository interface now 17 offered by LocalRepository and the underlying data retrieval logic. 18 Every repository interface available inherits from EntropyRepository 19 class and has to reimplement its own Schema subclass and its get_init 20 method (see LocalRepository documentation for more information). 21 22 I{LocalRepository} is the sqlite3 implementation of the repository 23 interface, as written above. 24 25 I{ServerRepositoryStatus} is a singleton containing the status of 26 server-side repositories. It is used to determine if repository has 27 been modified (tainted) or has been revision bumped already. 28 Revision bumps are automatic and happen on the very first data "commit". 29 Every repository features a revision number which is stored into the 30 "packages.db.revision" file. Only server-side (or community) repositories 31 are subject to this automation (revision file update on commit). 32 33 """ 34 35 from __future__ import with_statement 36 import os 37 import shutil 38 from entropy.const import etpConst, etpCache 39 from entropy.exceptions import IncorrectParameter, InvalidAtom, \ 40 SystemDatabaseError, OperationNotPermitted 41 from entropy.i18n import _ 42 from entropy.output import brown, bold, red, blue, purple, darkred, darkgreen, \ 43 TextInterface 44 from entropy.cache import EntropyCacher 45 from entropy.core import Singleton, SystemSettings 46 47 try: # try with sqlite3 from >=python 2.5 48 from sqlite3 import dbapi2 49 except ImportError: # fallback to pysqlite 50 try: 51 from pysqlite2 import dbapi2 52 except ImportError, e: 53 raise SystemError( 54 "%s. %s: %s" % ( 55 _("Entropy needs Python compiled with sqlite3 support"), 56 _("Error"), 57 e, 58 ) 59 ) 60

61 -class ServerRepositoryStatus(Singleton):

62

63 - def init_singleton(self):

64 self.__data = {}

65

66 - def __create_if_necessary(self, db):

67 if db not in self.__data: 68 self.__data[db] = {} 69 self.__data[db]['tainted'] = False 70 self.__data[db]['bumped'] = False 71 self.__data[db]['unlock_msg'] = False

72

73 - def set_unlock_msg(self, db):

74 self.__create_if_necessary(db) 75 self.__data[db]['unlock_msg'] = True

76

77 - def unset_unlock_msg(self, db):

78 self.__create_if_necessary(db) 79 self.__data[db]['unlock_msg'] = False

80

81 - def set_tainted(self, db):

82 self.__create_if_necessary(db) 83 self.__data[db]['tainted'] = True

84

85 - def unset_tainted(self, db):

86 self.__create_if_necessary(db) 87 self.__data[db]['tainted'] = False

88

89 - def set_bumped(self, db):

90 self.__create_if_necessary(db) 91 self.__data[db]['bumped'] = True

92

93 - def unset_bumped(self, db):

94 self.__create_if_necessary(db) 95 self.__data[db]['bumped'] = False

96

97 - def is_tainted(self, db):

98 self.__create_if_necessary(db) 99 return self.__data[db]['tainted']

100

101 - def is_bumped(self, db):

102 self.__create_if_necessary(db) 103 return self.__data[db]['bumped']

104

105 - def is_unlock_msg(self, db):

106 self.__create_if_necessary(db) 107 return self.__data[db]['unlock_msg']

108 109

110 -class EntropyRepository:

111 112 """ 113 Entropy repository interface base class. 114 This class contains database implementation independent logic, and 115 is inherited by all the Entropy repository interfaces implementations 116 (actually only LocalRepository). 117 """ 118

119 - class Schema:

120

121 - def get_init(self):

122 raise NotImplementedError

123

124 - def __init__(self):

125 """ 126 EntropyRepository base repository interface class constructor. 127 128 @return: None 129 @rtype: None 130 """ 131 self.SystemSettings = SystemSettings() 132 self.srv_sys_settings_plugin = \ 133 etpConst['system_settings_plugins_ids']['server_plugin'] 134 self.dbMatchCacheKey = etpCache['dbMatch'] 135 self.client_settings_plugin_id = etpConst['system_settings_plugins_ids']['client_plugin'] 136 self.db_branch = self.SystemSettings['repositories']['branch'] 137 self.Cacher = EntropyCacher()

138 139

140 -class LocalRepository(EntropyRepository):

141

142 - class Schema:

143

144 - def get_init(self):

145 return """ 146 CREATE TABLE baseinfo ( 147 idpackage INTEGER PRIMARY KEY AUTOINCREMENT, 148 atom VARCHAR, 149 idcategory INTEGER, 150 name VARCHAR, 151 version VARCHAR, 152 versiontag VARCHAR, 153 revision INTEGER, 154 branch VARCHAR, 155 slot VARCHAR, 156 idlicense INTEGER, 157 etpapi INTEGER, 158 trigger INTEGER 159 ); 160 161 CREATE TABLE extrainfo ( 162 idpackage INTEGER PRIMARY KEY, 163 description VARCHAR, 164 homepage VARCHAR, 165 download VARCHAR, 166 size VARCHAR, 167 idflags INTEGER, 168 digest VARCHAR, 169 datecreation VARCHAR 170 ); 171 172 CREATE TABLE content ( 173 idpackage INTEGER, 174 file VARCHAR, 175 type VARCHAR 176 ); 177 178 CREATE TABLE provide ( 179 idpackage INTEGER, 180 atom VARCHAR 181 ); 182 183 CREATE TABLE dependencies ( 184 idpackage INTEGER, 185 iddependency INTEGER, 186 type INTEGER 187 ); 188 189 CREATE TABLE dependenciesreference ( 190 iddependency INTEGER PRIMARY KEY AUTOINCREMENT, 191 dependency VARCHAR 192 ); 193 194 CREATE TABLE dependstable ( 195 iddependency INTEGER PRIMARY KEY, 196 idpackage INTEGER 197 ); 198 199 CREATE TABLE conflicts ( 200 idpackage INTEGER, 201 conflict VARCHAR 202 ); 203 204 CREATE TABLE mirrorlinks ( 205 mirrorname VARCHAR, 206 mirrorlink VARCHAR 207 ); 208 209 CREATE TABLE sources ( 210 idpackage INTEGER, 211 idsource INTEGER 212 ); 213 214 CREATE TABLE sourcesreference ( 215 idsource INTEGER PRIMARY KEY AUTOINCREMENT, 216 source VARCHAR 217 ); 218 219 CREATE TABLE useflags ( 220 idpackage INTEGER, 221 idflag INTEGER 222 ); 223 224 CREATE TABLE useflagsreference ( 225 idflag INTEGER PRIMARY KEY AUTOINCREMENT, 226 flagname VARCHAR 227 ); 228 229 CREATE TABLE keywords ( 230 idpackage INTEGER, 231 idkeyword INTEGER 232 ); 233 234 CREATE TABLE keywordsreference ( 235 idkeyword INTEGER PRIMARY KEY AUTOINCREMENT, 236 keywordname VARCHAR 237 ); 238 239 CREATE TABLE categories ( 240 idcategory INTEGER PRIMARY KEY AUTOINCREMENT, 241 category VARCHAR 242 ); 243 244 CREATE TABLE licenses ( 245 idlicense INTEGER PRIMARY KEY AUTOINCREMENT, 246 license VARCHAR 247 ); 248 249 CREATE TABLE flags ( 250 idflags INTEGER PRIMARY KEY AUTOINCREMENT, 251 chost VARCHAR, 252 cflags VARCHAR, 253 cxxflags VARCHAR 254 ); 255 256 CREATE TABLE configprotect ( 257 idpackage INTEGER PRIMARY KEY, 258 idprotect INTEGER 259 ); 260 261 CREATE TABLE configprotectmask ( 262 idpackage INTEGER PRIMARY KEY, 263 idprotect INTEGER 264 ); 265 266 CREATE TABLE configprotectreference ( 267 idprotect INTEGER PRIMARY KEY AUTOINCREMENT, 268 protect VARCHAR 269 ); 270 271 CREATE TABLE systempackages ( 272 idpackage INTEGER PRIMARY KEY 273 ); 274 275 CREATE TABLE injected ( 276 idpackage INTEGER PRIMARY KEY 277 ); 278 279 CREATE TABLE installedtable ( 280 idpackage INTEGER PRIMARY KEY, 281 repositoryname VARCHAR, 282 source INTEGER 283 ); 284 285 CREATE TABLE sizes ( 286 idpackage INTEGER PRIMARY KEY, 287 size INTEGER 288 ); 289 290 CREATE TABLE messages ( 291 idpackage INTEGER, 292 message VARCHAR 293 ); 294 295 CREATE TABLE counters ( 296 counter INTEGER, 297 idpackage INTEGER, 298 branch VARCHAR, 299 PRIMARY KEY(idpackage,branch) 300 ); 301 302 CREATE TABLE trashedcounters ( 303 counter INTEGER 304 ); 305 306 CREATE TABLE eclasses ( 307 idpackage INTEGER, 308 idclass INTEGER 309 ); 310 311 CREATE TABLE eclassesreference ( 312 idclass INTEGER PRIMARY KEY AUTOINCREMENT, 313 classname VARCHAR 314 ); 315 316 CREATE TABLE needed ( 317 idpackage INTEGER, 318 idneeded INTEGER, 319 elfclass INTEGER 320 ); 321 322 CREATE TABLE neededreference ( 323 idneeded INTEGER PRIMARY KEY AUTOINCREMENT, 324 library VARCHAR 325 ); 326 327 CREATE TABLE neededlibrarypaths ( 328 library VARCHAR, 329 path VARCHAR, 330 elfclass INTEGER, 331 PRIMARY KEY(library, path, elfclass) 332 ); 333 334 CREATE TABLE neededlibraryidpackages ( 335 idpackage INTEGER, 336 library VARCHAR, 337 elfclass INTEGER 338 ); 339 340 CREATE TABLE treeupdates ( 341 repository VARCHAR PRIMARY KEY, 342 digest VARCHAR 343 ); 344 345 CREATE TABLE treeupdatesactions ( 346 idupdate INTEGER PRIMARY KEY AUTOINCREMENT, 347 repository VARCHAR, 348 command VARCHAR, 349 branch VARCHAR, 350 date VARCHAR 351 ); 352 353 CREATE TABLE licensedata ( 354 licensename VARCHAR UNIQUE, 355 text BLOB, 356 compressed INTEGER 357 ); 358 359 CREATE TABLE licenses_accepted ( 360 licensename VARCHAR UNIQUE 361 ); 362 363 CREATE TABLE triggers ( 364 idpackage INTEGER PRIMARY KEY, 365 data BLOB 366 ); 367 368 CREATE TABLE entropy_misc_counters ( 369 idtype INTEGER PRIMARY KEY, 370 counter INTEGER 371 ); 372 373 CREATE TABLE categoriesdescription ( 374 category VARCHAR, 375 locale VARCHAR, 376 description VARCHAR 377 ); 378 379 CREATE TABLE packagesets ( 380 setname VARCHAR, 381 dependency VARCHAR 382 ); 383 384 CREATE TABLE packagechangelogs ( 385 category VARCHAR, 386 name VARCHAR, 387 changelog BLOB, 388 PRIMARY KEY (category, name) 389 ); 390 391 CREATE TABLE automergefiles ( 392 idpackage INTEGER, 393 configfile VARCHAR, 394 md5 VARCHAR 395 ); 396 397 CREATE TABLE packagesignatures ( 398 idpackage INTEGER PRIMARY KEY, 399 sha1 VARCHAR, 400 sha256 VARCHAR, 401 sha512 VARCHAR 402 ); 403 404 CREATE TABLE packagespmphases ( 405 idpackage INTEGER PRIMARY KEY, 406 phases VARCHAR 407 ); 408 409 """

410 411 import entropy.tools as entropyTools 412 import entropy.dump as dumpTools 413 import threading

414 - def __init__(self, readOnly = False, noUpload = False, dbFile = None, 415 clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], 416 indexing = True, OutputInterface = None, ServiceInterface = None, 417 skipChecks = False, useBranch = None, lockRemote = True):

418 419 EntropyRepository.__init__(self) 420 421 self.dbname = dbname 422 self.lockRemote = lockRemote 423 if self.dbname == etpConst['clientdbid']: 424 self.db_branch = None 425 if useBranch != None: 426 self.db_branch = useBranch 427 428 if OutputInterface == None: 429 OutputInterface = TextInterface() 430 431 if dbFile == None: 432 raise IncorrectParameter("IncorrectParameter: %s" % ( 433 _("valid database path needed"),) ) 434 435 self.__write_mutex = self.threading.Lock() 436 self.dbapi2 = dbapi2 437 # setup output interface 438 self.OutputInterface = OutputInterface 439 self.updateProgress = self.OutputInterface.updateProgress 440 self.askQuestion = self.OutputInterface.askQuestion 441 # setup service interface 442 self.ServiceInterface = ServiceInterface 443 self.readOnly = readOnly 444 self.noUpload = noUpload 445 self.clientDatabase = clientDatabase 446 self.xcache = xcache 447 self.indexing = indexing 448 self.skipChecks = skipChecks 449 if not self.skipChecks: 450 if not self.entropyTools.is_user_in_entropy_group(): 451 # forcing since we won't have write access to db 452 self.indexing = False 453 # live systems don't like wasting RAM 454 if self.entropyTools.islive(): 455 self.indexing = False 456 self.dbFile = dbFile 457 self.dbclosed = True 458 self.server_repo = None 459 460 if not self.clientDatabase: 461 self.server_repo = self.dbname[len(etpConst['serverdbid']):] 462 self.create_dbstatus_data() 463 464 if not self.skipChecks: 465 # no caching for non root and server connections 466 if (self.dbname.startswith(etpConst['serverdbid'])) or \ 467 (not self.entropyTools.is_user_in_entropy_group()): 468 self.xcache = False 469 self.live_cache = {} 470 471 # create connection 472 self.connection = self.dbapi2.connect(dbFile, timeout=300.0, 473 check_same_thread = False) 474 self.cursor = self.connection.cursor() 475 476 if not self.skipChecks: 477 try: 478 if os.access(self.dbFile, os.W_OK) and \ 479 self.doesTableExist('baseinfo') and \ 480 self.doesTableExist('extrainfo'): 481 482 if self.entropyTools.islive() and etpConst['systemroot']: 483 self.databaseStructureUpdates() 484 else: 485 self.databaseStructureUpdates() 486 except self.dbapi2.Error: 487 self.cursor.close() 488 self.connection.close() 489 raise 490 491 # now we can set this to False 492 self.dbclosed = False

493

494 - def setCacheSize(self, size):

495 self.cursor.execute('PRAGMA cache_size = '+str(size))

496

497 - def setDefaultCacheSize(self, size):

498 self.cursor.execute('PRAGMA default_cache_size = '+str(size))

499 500

501 - def __del__(self):

502 if not self.dbclosed: 503 self.closeDB()

504

505 - def create_dbstatus_data(self):

506 taint_file = self.ServiceInterface.get_local_database_taint_file( 507 self.server_repo) 508 if os.path.isfile(taint_file): 509 dbs = ServerRepositoryStatus() 510 dbs.set_tainted(self.dbFile) 511 dbs.set_bumped(self.dbFile)

512

513 - def closeDB(self):

514 515 self.dbclosed = True 516 517 # if the class is opened readOnly, close and forget 518 if self.readOnly: 519 self.cursor.close() 520 self.connection.close() 521 return 522 523 if self.clientDatabase: 524 self.commitChanges() 525 self.cursor.close() 526 self.connection.close() 527 return 528 529 sts = ServerRepositoryStatus() 530 if not sts.is_tainted(self.dbFile): 531 # we can unlock it, no changes were made 532 self.ServiceInterface.MirrorsService.lock_mirrors(False, 533 repo = self.server_repo) 534 elif not sts.is_unlock_msg(self.dbFile): 535 u_msg = _("Mirrors have not been unlocked. Remember to sync them.") 536 self.updateProgress( 537 darkgreen(u_msg), 538 importance = 1, 539 type = "info", 540 header = brown(" * ") 541 ) 542 sts.set_unlock_msg(self.dbFile) # avoid spamming 543 544 self.commitChanges() 545 self.cursor.close() 546 self.connection.close()

547

548 - def vacuum(self):

549 self.cursor.execute("vacuum")

550

551 - def commitChanges(self):

552 553 if self.readOnly: 554 return 555 556 try: 557 self.connection.commit() 558 except self.dbapi2.Error: 559 pass 560 561 if not self.clientDatabase: 562 self.taintDatabase() 563 dbs = ServerRepositoryStatus() 564 if (dbs.is_tainted(self.dbFile)) and \ 565 (not dbs.is_bumped(self.dbFile)): 566 # bump revision, setting DatabaseBump causes 567 # the session to just bump once 568 dbs.set_bumped(self.dbFile) 569 self.revisionBump()

570

571 - def taintDatabase(self):

572 # if it's equo to open it, this should be avoided 573 if self.clientDatabase: 574 return 575 # taint the database status 576 taint_file = self.ServiceInterface.get_local_database_taint_file( 577 repo = self.server_repo) 578 f = open(taint_file, "w") 579 f.write(etpConst['currentarch']+" database tainted\n") 580 f.flush() 581 f.close() 582 ServerRepositoryStatus().set_tainted(self.dbFile)

583

584 - def untaintDatabase(self):

585 # if it's equo to open it, this should be avoided 586 if self.clientDatabase: 587 return 588 ServerRepositoryStatus().unset_tainted(self.dbFile) 589 # untaint the database status 590 taint_file = self.ServiceInterface.get_local_database_taint_file( 591 repo = self.server_repo) 592 if os.path.isfile(taint_file): 593 os.remove(taint_file)

594

595 - def revisionBump(self):

596 revision_file = self.ServiceInterface.get_local_database_revision_file( 597 repo = self.server_repo) 598 if not os.path.isfile(revision_file): 599 revision = 1 600 else: 601 f = open(revision_file, "r") 602 revision = int(f.readline().strip()) 603 revision += 1 604 f.close() 605 f = open(revision_file, "w") 606 f.write(str(revision)+"\n") 607 f.flush() 608 f.close()

609

610 - def isDatabaseTainted(self):

611 taint_file = self.ServiceInterface.get_local_database_taint_file( 612 repo = self.server_repo) 613 if os.path.isfile(taint_file): 614 return True 615 return False

616 617 # never use this unless you know what you're doing

618 - def initializeDatabase(self):

619 self.checkReadOnly() 620 my = self.Schema() 621 for table in self.listAllTables(): 622 try: 623 self.cursor.execute("DROP TABLE %s" % (table,)) 624 except self.dbapi2.OperationalError: 625 # skip tables that can't be dropped 626 continue 627 self.cursor.executescript(my.get_init()) 628 self.databaseStructureUpdates() 629 # set cache size 630 self.setCacheSize(8192) 631 self.setDefaultCacheSize(8192) 632 self.commitChanges()

633

634 - def checkReadOnly(self):

635 if self.readOnly: 636 raise OperationNotPermitted("OperationNotPermitted: %s." % ( 637 _("can't do that on a readonly database"), 638 ) 639 )

640 641 # check for /usr/portage/profiles/updates changes

642 - def serverUpdatePackagesData(self):

643 644 etpConst['server_treeupdatescalled'].add(self.server_repo) 645 646 repo_updates_file = self.ServiceInterface.get_local_database_treeupdates_file(self.server_repo) 647 doRescan = False 648 649 stored_digest = self.retrieveRepositoryUpdatesDigest(self.server_repo) 650 if stored_digest == -1: 651 doRescan = True 652 653 # check portage files for changes if doRescan is still false 654 portage_dirs_digest = "0" 655 if not doRescan: 656 657 treeupdates_dict = self.ServiceInterface.repository_treeupdate_digests 658 659 if treeupdates_dict.has_key(self.server_repo): 660 portage_dirs_digest = treeupdates_dict.get(self.server_repo) 661 else: 662 663 spm = self.ServiceInterface.SpmService 664 # grab portdir 665 updates_dir = etpConst['systemroot'] + \ 666 spm.get_spm_setting("PORTDIR") + "/profiles/updates" 667 if os.path.isdir(updates_dir): 668 # get checksum 669 mdigest = self.entropyTools.md5sum_directory(updates_dir, 670 get_obj = True) 671 # also checksum etpConst['etpdatabaseupdatefile'] 672 if os.path.isfile(repo_updates_file): 673 f = open(repo_updates_file) 674 block = f.read(1024) 675 while block: 676 mdigest.update(block) 677 block = f.read(1024) 678 f.close() 679 portage_dirs_digest = mdigest.hexdigest() 680 treeupdates_dict[self.server_repo] = portage_dirs_digest 681 del updates_dir 682 683 if doRescan or (str(stored_digest) != str(portage_dirs_digest)): 684 685 # force parameters 686 self.readOnly = False 687 self.noUpload = True 688 689 # reset database tables 690 self.clearTreeupdatesEntries(self.server_repo) 691 692 spm = self.ServiceInterface.SpmService 693 updates_dir = etpConst['systemroot'] + \ 694 spm.get_spm_setting("PORTDIR") + "/profiles/updates" 695 update_files = self.entropyTools.sort_update_files( 696 os.listdir(updates_dir)) 697 update_files = [os.path.join(updates_dir, x) for x in update_files] 698 # now load actions from files 699 update_actions = [] 700 for update_file in update_files: 701 f = open(update_file, "r") 702 mycontent = f.readlines() 703 f.close() 704 lines = [x.strip() for x in mycontent if x.strip()] 705 update_actions.extend(lines) 706 707 # add entropy packages.db.repo_updates content 708 if os.path.isfile(repo_updates_file): 709 f = open(repo_updates_file, "r") 710 mycontent = f.readlines() 711 f.close() 712 lines = [x.strip() for x in mycontent if x.strip() and \ 713 not x.strip().startswith("#")] 714 update_actions.extend(lines) 715 # now filter the required actions 716 update_actions = self.filterTreeUpdatesActions(update_actions) 717 if update_actions: 718 719 mytxt = "%s: %s. %s %s" % ( 720 bold(_("ATTENTION")), 721 red(_("forcing package updates")), 722 red(_("Syncing with")), 723 blue(updates_dir), 724 ) 725 self.updateProgress( 726 mytxt, 727 importance = 1, 728 type = "info", 729 header = brown(" * ") 730 ) 731 # lock database 732 if self.lockRemote: 733 self.ServiceInterface.do_server_repository_sync_lock( 734 self.server_repo, self.noUpload) 735 # now run queue 736 try: 737 self.runTreeUpdatesActions(update_actions) 738 except: 739 # destroy digest 740 self.setRepositoryUpdatesDigest(self.server_repo, "-1") 741 raise 742 743 # store new actions 744 self.addRepositoryUpdatesActions( 745 self.server_repo, update_actions, self.db_branch) 746 747 # store new digest into database 748 self.setRepositoryUpdatesDigest( 749 self.server_repo, portage_dirs_digest) 750 self.commitChanges()

751

752 - def clientUpdatePackagesData(self, clientDbconn, force = False):

753 754 if clientDbconn == None: 755 return 756 757 repository = self.dbname[len(etpConst['dbnamerepoprefix']):] 758 etpConst['client_treeupdatescalled'].add(repository) 759 760 doRescan = False 761 shell_rescan = os.getenv("ETP_TREEUPDATES_RESCAN") 762 if shell_rescan: doRescan = True 763 764 # check database digest 765 stored_digest = self.retrieveRepositoryUpdatesDigest(repository) 766 if stored_digest == -1: 767 doRescan = True 768 769 # check stored value in client database 770 client_digest = "0" 771 if not doRescan: 772 client_digest = clientDbconn.retrieveRepositoryUpdatesDigest( 773 repository) 774 775 if doRescan or (str(stored_digest) != str(client_digest)) or force: 776 777 # reset database tables 778 clientDbconn.clearTreeupdatesEntries(repository) 779 780 # load updates 781 update_actions = self.retrieveTreeUpdatesActions(repository) 782 # now filter the required actions 783 update_actions = clientDbconn.filterTreeUpdatesActions( 784 update_actions) 785 786 if update_actions: 787 788 mytxt = "%s: %s." % ( 789 bold(_("ATTENTION")), 790 red(_("forcing packages metadata update")), 791 ) 792 self.updateProgress( 793 mytxt, 794 importance = 1, 795 type = "info", 796 header = darkred(" * ") 797 ) 798 mytxt = "%s %s." % ( 799 red(_("Updating system database using repository")), 800 blue(repository), 801 ) 802 self.updateProgress( 803 mytxt, 804 importance = 1, 805 type = "info", 806 header = darkred(" * ") 807 ) 808 # run stuff 809 clientDbconn.runTreeUpdatesActions(update_actions) 810 811 # store new digest into database 812 clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest) 813 # store new actions 814 clientDbconn.addRepositoryUpdatesActions(etpConst['clientdbid'], 815 update_actions, self.SystemSettings['repositories']['branch']) 816 clientDbconn.commitChanges() 817 # clear client cache 818 clientDbconn.clearCache() 819 return True

820 821 # this functions will filter either data from /usr/portage/profiles/updates/* 822 # or repository database returning only the needed actions

823 - def filterTreeUpdatesActions(self, actions):

824 new_actions = [] 825 for action in actions: 826 827 if action in new_actions: # skip dupies 828 continue 829 830 doaction = action.split() 831 if doaction[0] == "slotmove": 832 833 # slot move 834 atom = doaction[1] 835 from_slot = doaction[2] 836 to_slot = doaction[3] 837 atom_key = self.entropyTools.dep_getkey(atom) 838 category = atom_key.split("/")[0] 839 matches = self.atomMatch(atom, matchSlot = from_slot, 840 multiMatch = True) 841 found = False 842 # found atoms, check category 843 for idpackage in matches[0]: 844 myslot = self.retrieveSlot(idpackage) 845 mycategory = self.retrieveCategory(idpackage) 846 if mycategory == category: 847 if (myslot != to_slot) and \ 848 (action not in new_actions): 849 new_actions.append(action) 850 found = True 851 break 852 if found: 853 continue 854 # if we get here it means found == False 855 # search into dependencies 856 dep_atoms = self.searchDependency(atom_key, like = True, 857 multi = True, strings = True) 858 dep_atoms = [x for x in dep_atoms if x.endswith(":"+from_slot) \ 859 and self.entropyTools.dep_getkey(x) == atom_key] 860 if dep_atoms: 861 new_actions.append(action) 862 863 elif doaction[0] == "move": 864 atom = doaction[1] # usually a key 865 atom_key = self.entropyTools.dep_getkey(atom) 866 category = atom_key.split("/")[0] 867 matches = self.atomMatch(atom, multiMatch = True) 868 found = False 869 for idpackage in matches[0]: 870 mycategory = self.retrieveCategory(idpackage) 871 if (mycategory == category) and (action \ 872 not in new_actions): 873 new_actions.append(action) 874 found = True 875 break 876 if found: 877 continue 878 # if we get here it means found == False 879 # search into dependencies 880 dep_atoms = self.searchDependency(atom_key, like = True, 881 multi = True, strings = True) 882 dep_atoms = [x for x in dep_atoms if \ 883 self.entropyTools.dep_getkey(x) == atom_key] 884 if dep_atoms: 885 new_actions.append(action) 886 return new_actions

887 888 # this is the place to add extra actions support

889 - def runTreeUpdatesActions(self, actions):

890 891 mytxt = "%s: %s, %s." % ( 892 bold(_("SPM")), 893 blue(_("Running fixpackages")), 894 red(_("it could take a while")), 895 ) 896 self.updateProgress( 897 mytxt, 898 importance = 1, 899 type = "warning", 900 header = darkred(" * ") 901 ) 902 if self.clientDatabase: 903 try: 904 spm = self.ServiceInterface.Spm() 905 spm.run_fixpackages() 906 except: 907 pass 908 else: 909 self.ServiceInterface.SpmService.run_fixpackages() 910 911 spm_moves = set() 912 quickpkg_atoms = set() 913 for action in actions: 914 command = action.split() 915 mytxt = "%s: %s: %s." % ( 916 bold(_("ENTROPY")), 917 red(_("action")), 918 blue(action), 919 ) 920 self.updateProgress( 921 mytxt, 922 importance = 1, 923 type = "warning", 924 header = darkred(" * ") 925 ) 926 if command[0] == "move": 927 spm_moves.add(action) 928 quickpkg_atoms |= self.runTreeUpdatesMoveAction(command[1:], 929 quickpkg_atoms) 930 elif command[0] == "slotmove": 931 quickpkg_atoms |= self.runTreeUpdatesSlotmoveAction(command[1:], 932 quickpkg_atoms) 933 934 mytxt = "%s: %s." % ( 935 bold(_("ENTROPY")), 936 blue(_("package move actions complete")), 937 ) 938 self.updateProgress( 939 mytxt, 940 importance = 1, 941 type = "info", 942 header = purple(" @@ ") 943 ) 944 945 if quickpkg_atoms and not self.clientDatabase: 946 # quickpkg package and packages owning it as a dependency 947 try: 948 self.runTreeUpdatesQuickpkgAction(quickpkg_atoms) 949 except: 950 self.entropyTools.print_traceback() 951 mytxt = "%s: %s: %s, %s." % ( 952 bold(_("WARNING")), 953 red(_("Cannot complete quickpkg for atoms")), 954 blue(str(list(quickpkg_atoms))), 955 _("do it manually"), 956 ) 957 self.updateProgress( 958 mytxt, 959 importance = 1, 960 type = "warning", 961 header = darkred(" * ") 962 ) 963 self.commitChanges() 964 965 if spm_moves: 966 try: 967 self.doTreeupdatesSpmCleanup(spm_moves) 968 except Exception, e: 969 mytxt = "%s: %s: %s, %s." % ( 970 bold(_("WARNING")), 971 red(_("Cannot run SPM cleanup, error")), 972 Exception, 973 e, 974 ) 975 self.entropyTools.print_traceback() 976 977 mytxt = "%s: %s." % ( 978 bold(_("ENTROPY")), 979 blue(_("package moves completed successfully")), 980 ) 981 self.updateProgress( 982 mytxt, 983 importance = 1, 984 type = "info", 985 header = brown(" @@ ") 986 ) 987 988 # discard cache 989 self.clearCache()

990 991 992 # -- move action: 993 # 1) move package key to the new name: category + name + atom 994 # 2) update all the dependencies in dependenciesreference to the new key 995 # 3) run fixpackages which will update /var/db/pkg files 996 # 4) automatically run quickpkg() to build the new binary and 997 # tainted binaries owning tainted iddependency and taint database

998 - def runTreeUpdatesMoveAction(self, move_command, quickpkg_queue):

999 1000 dep_from = move_command[0] 1001 key_from = self.entropyTools.dep_getkey(dep_from) 1002 key_to = move_command[1] 1003 cat_to = key_to.split("/")[0] 1004 name_to = key_to.split("/")[1] 1005 matches = self.atomMatch(dep_from, multiMatch = True) 1006 iddependencies = set() 1007 1008 for idpackage in matches[0]: 1009 1010 slot = self.retrieveSlot(idpackage) 1011 old_atom = self.retrieveAtom(idpackage) 1012 new_atom = old_atom.replace(key_from, key_to) 1013 1014 ### UPDATE DATABASE 1015 # update category 1016 self.setCategory(idpackage, cat_to) 1017 # update name 1018 self.setName(idpackage, name_to) 1019 # update atom 1020 self.setAtom(idpackage, new_atom) 1021 1022 # look for packages we need to quickpkg again 1023 # note: quickpkg_queue is simply ignored if self.clientDatabase 1024 quickpkg_queue.add(key_to+":"+str(slot)) 1025 1026 if not self.clientDatabase: 1027 1028 # check for injection and warn the developer 1029 injected = self.isInjected(idpackage) 1030 if injected: 1031 mytxt = "%s: %s %s. %s !!! %s." % ( 1032 bold(_("INJECT")), 1033 blue(str(new_atom)), 1034 red(_("has been injected")), 1035 red(_("quickpkg manually to update embedded db")), 1036 red(_("Repository database updated anyway")), 1037 ) 1038 self.updateProgress( 1039 mytxt, 1040 importance = 1, 1041 type = "warning", 1042 header = darkred(" * ") 1043 ) 1044 1045 iddeps = self.searchDependency(key_from, like = True, multi = True) 1046 for iddep in iddeps: 1047 # update string 1048 mydep = self.retrieveDependencyFromIddependency(iddep) 1049 mydep_key = self.entropyTools.dep_getkey(mydep) 1050 # avoid changing wrong atoms -> dev-python/qscintilla-python would 1051 # become x11-libs/qscintilla if we don't do this check 1052 if mydep_key != key_from: 1053 continue 1054 mydep = mydep.replace(key_from, key_to) 1055 # now update 1056 # dependstable on server is always re-generated 1057 self.setDependency(iddep, mydep) 1058 # we have to repackage also package owning this iddep 1059 iddependencies |= self.searchIdpackageFromIddependency(iddep) 1060 1061 self.commitChanges() 1062 quickpkg_queue = list(quickpkg_queue) 1063 for x in range(len(quickpkg_queue)): 1064 myatom = quickpkg_queue[x] 1065 myatom = myatom.replace(key_from, key_to) 1066 quickpkg_queue[x] = myatom 1067 quickpkg_queue = set(quickpkg_queue) 1068 for idpackage_owner in iddependencies: 1069 myatom = self.retrieveAtom(idpackage_owner) 1070 myatom = myatom.replace(key_from, key_to) 1071 quickpkg_queue.add(myatom) 1072 return quickpkg_queue

1073 1074 1075 # -- slotmove action: 1076 # 1) move package slot 1077 # 2) update all the dependencies in dependenciesreference owning 1078 # same matched atom + slot 1079 # 3) run fixpackages which will update /var/db/pkg files 1080 # 4) automatically run quickpkg() to build the new 1081 # binary and tainted binaries owning tainted iddependency 1082 # and taint database

1083 - def runTreeUpdatesSlotmoveAction(self, slotmove_command, quickpkg_queue):

1084 1085 atom = slotmove_command[0] 1086 atomkey = self.entropyTools.dep_getkey(atom) 1087 slot_from = slotmove_command[1] 1088 slot_to = slotmove_command[2] 1089 matches = self.atomMatch(atom, multiMatch = True) 1090 iddependencies = set() 1091 1092 matched_idpackages = matches[0] 1093 for idpackage in matched_idpackages: 1094 1095 ### UPDATE DATABASE 1096 # update slot 1097 self.setSlot(idpackage, slot_to) 1098 1099 # look for packages we need to quickpkg again 1100 # note: quickpkg_queue is simply ignored if self.clientDatabase 1101 quickpkg_queue.add(atom+":"+str(slot_to)) 1102 1103 if not self.clientDatabase: 1104 1105 # check for injection and warn the developer 1106 injected = self.isInjected(idpackage) 1107 if injected: 1108 mytxt = "%s: %s %s. %s !!! %s." % ( 1109 bold(_("INJECT")), 1110 blue(str(atom)), 1111 red(_("has been injected")), 1112 red(_("quickpkg manually to update embedded db")), 1113 red(_("Repository database updated anyway")), 1114 ) 1115 self.updateProgress( 1116 mytxt, 1117 importance = 1, 1118 type = "warning", 1119 header = darkred(" * ") 1120 ) 1121 1122 # only if we've found VALID matches ! 1123 iddeps = self.searchDependency(atomkey, like = True, multi = True) 1124 for iddep in iddeps: 1125 # update string 1126 mydep = self.retrieveDependencyFromIddependency(iddep) 1127 mydep_key = self.entropyTools.dep_getkey(mydep) 1128 if mydep_key != atomkey: 1129 continue 1130 if not mydep.endswith(":"+slot_from): # probably slotted dep 1131 continue 1132 mydep_match = self.atomMatch(mydep) 1133 if mydep_match not in matched_idpackages: 1134 continue 1135 mydep = mydep.replace(":"+slot_from, ":"+slot_to) 1136 # now update 1137 # dependstable on server is always re-generated 1138 self.setDependency(iddep, mydep) 1139 # we have to repackage also package owning this iddep 1140 iddependencies |= self.searchIdpackageFromIddependency(iddep) 1141 1142 self.commitChanges() 1143 for idpackage_owner in iddependencies: 1144 myatom = self.retrieveAtom(idpackage_owner) 1145 quickpkg_queue.add(myatom) 1146 return quickpkg_queue

1147

1148 - def runTreeUpdatesQuickpkgAction(self, atoms):

1149 1150 self.commitChanges() 1151 1152 package_paths = set() 1153 runatoms = set() 1154 for myatom in atoms: 1155 mymatch = self.atomMatch(myatom) 1156 if mymatch[0] == -1: 1157 continue 1158 myatom = self.retrieveAtom(mymatch[0]) 1159 runatoms.add(myatom) 1160 1161 for myatom in runatoms: 1162 self.updateProgress( 1163 red("%s: " % (_("repackaging"),) )+blue(myatom), 1164 importance = 1, 1165 type = "warning", 1166 header = blue(" # ") 1167 ) 1168 mydest = self.ServiceInterface.get_local_store_directory( 1169 repo = self.server_repo) 1170 try: 1171 mypath = self.ServiceInterface.quickpkg(myatom, mydest) 1172 except: 1173 # remove broken bin before raising 1174 mypath = os.path.join(mydest, 1175 os.path.basename(myatom)+etpConst['packagesext']) 1176 if os.path.isfile(mypath): 1177 os.remove(mypath) 1178 self.entropyTools.print_traceback() 1179 mytxt = "%s: %s: %s, %s." % ( 1180 bold(_("WARNING")), 1181 red(_("Cannot complete quickpkg for atom")), 1182 blue(myatom), 1183 _("do it manually"), 1184 ) 1185 self.updateProgress( 1186 mytxt, 1187 importance = 1, 1188 type = "warning", 1189 header = darkred(" * ") 1190 ) 1191 continue 1192 package_paths.add(mypath) 1193 packages_data = [(x, False,) for x in package_paths] 1194 idpackages = self.ServiceInterface.add_packages_to_repository( 1195 packages_data, repo = self.server_repo) 1196 1197 if not idpackages: 1198 1199 mytxt = "%s: %s. %s." % ( 1200 bold(_("ATTENTION")), 1201 red(_("runTreeUpdatesQuickpkgAction did not run properly")), 1202 red(_("Please update packages manually")), 1203 ) 1204 self.updateProgress( 1205 mytxt, 1206 importance = 1, 1207 type = "warning", 1208 header = darkred(" * ") 1209 )

1210

1211 - def doTreeupdatesSpmCleanup(self, spm_moves):

1212 1213 # now erase Spm entries if necessary 1214 for action in spm_moves: 1215 command = action.split() 1216 if len(command) < 2: 1217 continue 1218 key = command[1] 1219 name = key.split("/")[1] 1220 if self.clientDatabase: 1221 try: 1222 spm = self.ServiceInterface.Spm() 1223 except: 1224 continue 1225 else: 1226 spm = self.ServiceInterface.SpmService 1227 vdb_path = spm.get_vdb_path() 1228 pkg_path = os.path.join(vdb_path, key.split("/")[0]) 1229 try: 1230 mydirs = [os.path.join(pkg_path, x) for x in \ 1231 os.listdir(pkg_path) if x.startswith(name)] 1232 except OSError: # no dir, no party! 1233 continue 1234 mydirs = [x for x in mydirs if os.path.isdir(x)] 1235 # now move these dirs 1236 for mydir in mydirs: 1237 to_path = os.path.join(etpConst['packagestmpdir'], 1238 os.path.basename(mydir)) 1239 mytxt = "%s: %s '%s' %s '%s'" % ( 1240 bold(_("SPM")), 1241 red(_("Moving old entry")), 1242 blue(mydir), 1243 red(_("to")), 1244 blue(to_path), 1245 ) 1246 self.updateProgress( 1247 mytxt, 1248 importance = 1, 1249 type = "warning", 1250 header = darkred(" * ") 1251 ) 1252 if os.path.isdir(to_path): 1253 shutil.rmtree(to_path, True) 1254 try: 1255 os.rmdir(to_path) 1256 except OSError: 1257 pass 1258 shutil.move(mydir, to_path)

1259 1260

1261 - def handlePackage(self, etpData, forcedRevision = -1, 1262 formattedContent = False):

1263 1264 self.checkReadOnly() 1265 1266 if self.clientDatabase: 1267 return self.addPackage(etpData, revision = forcedRevision, 1268 formatted_content = formattedContent) 1269 1270 # build atom string, server side 1271 pkgatom = self.entropyTools.create_package_atom_string( 1272 etpData['category'], etpData['name'], etpData['version'], 1273 etpData['versiontag']) 1274 foundid = self.isPackageAvailable(pkgatom) 1275 if foundid < 0: # same atom doesn't exist in any branch 1276 return self.addPackage(etpData, revision = forcedRevision, 1277 formatted_content = formattedContent) 1278 1279 idpackage = self.getIDPackage(pkgatom) 1280 curRevision = forcedRevision 1281 if forcedRevision == -1: 1282 curRevision = 0 1283 if idpackage != -1: 1284 curRevision = self.retrieveRevision(idpackage) 1285 1286 # remove old package atom, we do it here because othersie 1287 if idpackage != -1: 1288 # injected packages wouldn't be removed by addPackages 1289 self.removePackage(idpackage) 1290 if forcedRevision == -1: curRevision += 1 1291 1292 # add the new one 1293 return self.addPackage(etpData, revision = curRevision, 1294 formatted_content = formattedContent)

1295

1296 - def retrieve_packages_to_remove(self, name, category, slot, injected):

1297 1298 removelist = set() 1299 1300 # support for expiration-based packages handling, also internally 1301 # called Fat Scope. 1302 filter_similar = False 1303 srv_ss_plg = etpConst['system_settings_plugins_ids']['server_plugin'] 1304 srv_ss_fs_plg = \ 1305 etpConst['system_settings_plugins_ids']['server_plugin_fatscope'] 1306 1307 if not self.clientDatabase: # server-side db 1308 srv_plug_settings = self.SystemSettings.get(srv_ss_plg) 1309 if srv_plug_settings != None: 1310 if srv_plug_settings['server']['exp_based_scope']: 1311 # in case support is enabled, return an empty set 1312 filter_similar = True 1313 1314 searchsimilar = self.searchPackagesByNameAndCategory( 1315 name = name, 1316 category = category, 1317 sensitive = True 1318 ) 1319 if filter_similar: 1320 # filter out packages in the same scope that are allowed to stay 1321 idpkgs = self.SystemSettings[srv_ss_fs_plg]['repos'].get( 1322 self.server_repo) 1323 if idpkgs: 1324 if -1 in idpkgs: 1325 del searchsimilar[:] 1326 else: 1327 searchsimilar = [x for x in searchsimilar if x[1] \ 1328 not in idpkgs] 1329 1330 if not injected: 1331 # read: if package has been injected, we'll skip 1332 # the removal of packages in the same slot, 1333 # usually used server side btw 1334 for atom, idpackage in searchsimilar: 1335 # get the package slot 1336 myslot = self.retrieveSlot(idpackage) 1337 # we merely ignore packages with 1338 # negative counters, since they're the injected ones 1339 if self.isInjected(idpackage): continue 1340 if slot == myslot: 1341 # remove! 1342 removelist.add(idpackage) 1343 1344 return removelist

1345

1346 - def addPackage(self, etpData, revision = -1, idpackage = None, 1347 do_remove = True, do_commit = True, formatted_content = False):

1348 1349 self.checkReadOnly() 1350 1351 if revision == -1: 1352 try: 1353 revision = int(etpData['revision']) 1354 except (KeyError, ValueError): 1355 etpData['revision'] = 0 # revision not specified 1356 revision = 0 1357 elif not etpData.has_key('revision'): 1358 etpData['revision'] = revision 1359 1360 manual_deps = set() 1361 if do_remove: 1362 removelist = self.retrieve_packages_to_remove( 1363 etpData['name'], 1364 etpData['category'], 1365 etpData['slot'], 1366 etpData['injected'] 1367 ) 1368 for r_idpackage in removelist: 1369 manual_deps |= self.retrieveManualDependencies(r_idpackage) 1370 self.removePackage(r_idpackage, do_cleanup = False, 1371 do_commit = False) 1372 1373 # create new category if it doesn't exist 1374 catid = self.isCategoryAvailable(etpData['category']) 1375 if catid == -1: catid = self.addCategory(etpData['category']) 1376 1377 # create new license if it doesn't exist 1378 licid = self.isLicenseAvailable(etpData['license']) 1379 if licid == -1: licid = self.addLicense(etpData['license']) 1380 1381 idprotect = self.isProtectAvailable(etpData['config_protect']) 1382 if idprotect == -1: idprotect = self.addProtect( 1383 etpData['config_protect']) 1384 1385 idprotect_mask = self.isProtectAvailable( 1386 etpData['config_protect_mask']) 1387 if idprotect_mask == -1: idprotect_mask = self.addProtect( 1388 etpData['config_protect_mask']) 1389 1390 idflags = self.areCompileFlagsAvailable( 1391 etpData['chost'],etpData['cflags'],etpData['cxxflags']) 1392 if idflags == -1: idflags = self.addCompileFlags( 1393 etpData['chost'],etpData['cflags'],etpData['cxxflags']) 1394 1395 1396 trigger = 0 1397 if etpData['trigger']: 1398 trigger = 1 1399 1400 # baseinfo 1401 pkgatom = self.entropyTools.create_package_atom_string( 1402 etpData['category'], etpData['name'], etpData['version'], 1403 etpData['versiontag']) 1404 # add atom metadatum 1405 etpData['atom'] = pkgatom 1406 1407 mybaseinfo_data = (pkgatom, catid, etpData['name'], etpData['version'], 1408 etpData['versiontag'], revision, etpData['branch'], etpData['slot'], 1409 licid, etpData['etpapi'], trigger,) 1410 1411 myidpackage_string = 'NULL' 1412 if isinstance(idpackage, int): 1413 manual_deps |= self.retrieveManualDependencies(idpackage) 1414 # does it exist? 1415 self.removePackage(idpackage, do_cleanup = False, 1416 do_commit = False, do_rss = False) 1417 myidpackage_string = '?' 1418 mybaseinfo_data = (idpackage,)+mybaseinfo_data 1419 else: 1420 idpackage = None 1421 1422 # merge old manual dependencies 1423 for manual_dep in manual_deps: 1424 if manual_dep in etpData['dependencies']: continue 1425 etpData['dependencies'][manual_dep] = etpConst['spm']['mdepend_id'] 1426 1427 with self.__write_mutex: 1428 1429 self.cursor.execute(""" 1430 INSERT INTO baseinfo VALUES (%s,?,?,?,?,?,?,?,?,?,?,?)""" % ( 1431 myidpackage_string,), mybaseinfo_data) 1432 if idpackage == None: 1433 idpackage = self.cursor.lastrowid 1434 1435 # extrainfo 1436 self.cursor.execute( 1437 'INSERT INTO extrainfo VALUES (?,?,?,?,?,?,?,?)', 1438 ( idpackage, 1439 etpData['description'], 1440 etpData['homepage'], 1441 etpData['download'], 1442 etpData['size'], 1443 idflags, 1444 etpData['digest'], 1445 etpData['datecreation'], 1446 ) 1447 ) 1448 ### other information iserted below are not as critical as these above 1449 1450 # tables using a select 1451 self.insertEclasses(idpackage, etpData['eclasses']) 1452 self.insertNeeded(idpackage, etpData['needed']) 1453 self.insertDependencies(idpackage, etpData['dependencies']) 1454 self.insertSources(idpackage, etpData['sources']) 1455 self.insertUseflags(idpackage, etpData['useflags']) 1456 self.insertKeywords(idpackage, etpData['keywords']) 1457 self.insertLicenses(etpData['licensedata']) 1458 self.insertMirrors(etpData['mirrorlinks']) 1459 # package ChangeLog 1460 if etpData.get('changelog'): 1461 self.insertChangelog(etpData['category'], etpData['name'], 1462 etpData['changelog']) 1463 # package signatures 1464 if etpData.get('signatures'): 1465 self.insertSignatures(idpackage, etpData['signatures']) 1466 # needed libraries paths 1467 if etpData.get('needed_paths'): 1468 for lib in sorted(etpData['needed_paths']): 1469 self.insertNeededPaths(lib, etpData['needed_paths'][lib]) 1470 1471 # spm phases 1472 if etpData.get('spm_phases') != None: 1473 self.insertSpmPhases(idpackage, etpData['spm_phases']) 1474 1475 # not depending on other tables == no select done 1476 self.insertContent(idpackage, etpData['content'], 1477 already_formatted = formatted_content) 1478 etpData['counter'] = int(etpData['counter']) # cast to integer 1479 etpData['counter'] = self.insertPortageCounter( 1480 idpackage, 1481 etpData['counter'], 1482 etpData['branch'], 1483 etpData['injected'] 1484 ) 1485 self.insertOnDiskSize(idpackage, etpData['disksize']) 1486 if etpData['trigger']: 1487 self.insertTrigger(idpackage, etpData['trigger']) 1488 self.insertConflicts(idpackage, etpData['conflicts']) 1489 self.insertProvide(idpackage, etpData['provide']) 1490 self.insertMessages(idpackage, etpData['messages']) 1491 self.insertConfigProtect(idpackage, idprotect) 1492 self.insertConfigProtect(idpackage, idprotect_mask, mask = True) 1493 # injected? 1494 if etpData.get('injected'): 1495 self.setInjected(idpackage, do_commit = False) 1496 # is it a system package? 1497 if etpData.get('systempackage'): 1498 self.setSystemPackage(idpackage, do_commit = False) 1499 1500 self.clearCache() # we do live_cache.clear() here too 1501 if do_commit: 1502 self.commitChanges() 1503 1504 ### RSS Atom support 1505 ### dictionary will be elaborated by activator 1506 if self.SystemSettings.has_key(self.srv_sys_settings_plugin): 1507 if self.SystemSettings[self.srv_sys_settings_plugin]['server']['rss']['enabled'] and \ 1508 not self.clientDatabase: 1509 1510 self._write_rss_for_added_package(pkgatom, revision, 1511 etpData['description'], etpData['homepage']) 1512 1513 # Update category description 1514 if not self.clientDatabase: 1515 mycategory = etpData['category'] 1516 descdata = {} 1517 try: 1518 descdata = self.get_category_description_from_disk(mycategory) 1519 except (IOError, OSError, EOFError,): 1520 pass 1521 if descdata: 1522 self.setCategoryDescription(mycategory, descdata) 1523 1524 return idpackage, revision, etpData

1525

1526 - def _write_rss_for_added_package(self, pkgatom, revision, description, 1527 homepage):

1528 1529 rssAtom = pkgatom+"~"+str(revision) 1530 rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name']) 1531 if rssObj: 1532 self.ServiceInterface.rssMessages = rssObj.copy() 1533 if not isinstance(self.ServiceInterface.rssMessages, dict): 1534 self.ServiceInterface.rssMessages = {} 1535 if not self.ServiceInterface.rssMessages.has_key('added'): 1536 self.ServiceInterface.rssMessages['added'] = {} 1537 if not self.ServiceInterface.rssMessages.has_key('removed'): 1538 self.ServiceInterface.rssMessages['removed'] = {} 1539 if rssAtom in self.ServiceInterface.rssMessages['removed']: 1540 del self.ServiceInterface.rssMessages['removed'][rssAtom] 1541 1542 self.ServiceInterface.rssMessages['added'][rssAtom] = {} 1543 self.ServiceInterface.rssMessages['added'][rssAtom]['description'] = \ 1544 description 1545 self.ServiceInterface.rssMessages['added'][rssAtom]['homepage'] = \ 1546 homepage 1547 self.ServiceInterface.rssMessages['light'][rssAtom] = {} 1548 self.ServiceInterface.rssMessages['light'][rssAtom]['description'] = \ 1549 description 1550 self.dumpTools.dumpobj(etpConst['rss-dump-name'], 1551 self.ServiceInterface.rssMessages)

1552

1553 - def _write_rss_for_removed_package(self, idpackage):

1554 1555 rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name']) 1556 if rssObj: 1557 self.ServiceInterface.rssMessages = rssObj.copy() 1558 rssAtom = self.retrieveAtom(idpackage) 1559 rssRevision = self.retrieveRevision(idpackage) 1560 rssAtom += "~"+str(rssRevision) 1561 if not isinstance(self.ServiceInterface.rssMessages, dict): 1562 self.ServiceInterface.rssMessages = {} 1563 if not self.ServiceInterface.rssMessages.has_key('added'): 1564 self.ServiceInterface.rssMessages['added'] = {} 1565 if not self.ServiceInterface.rssMessages.has_key('removed'): 1566 self.ServiceInterface.rssMessages['removed'] = {} 1567 if rssAtom in self.ServiceInterface.rssMessages['added']: 1568 del self.ServiceInterface.rssMessages['added'][rssAtom] 1569 if rssAtom in self.ServiceInterface.rssMessages.get('light', []): 1570 del self.ServiceInterface.rssMessages['light'][rssAtom] 1571 1572 mydict = {} 1573 try: 1574 mydict['description'] = self.retrieveDescription(idpackage) 1575 except TypeError: 1576 mydict['description'] = "N/A" 1577 try: 1578 mydict['homepage'] = self.retrieveHomepage(idpackage) 1579 except TypeError: 1580 mydict['homepage'] = "" 1581 1582 self.dumpTools.dumpobj(etpConst['rss-dump-name'], 1583 self.ServiceInterface.rssMessages) 1584 1585 self.ServiceInterface.rssMessages['removed'][rssAtom] = mydict

1586

1587 - def removePackage(self, idpackage, do_cleanup = True, do_commit = True, 1588 do_rss = True):

1589 1590 self.checkReadOnly() 1591 # clear caches 1592 self.clearCache() 1593 1594 ### RSS Atom support 1595 ### dictionary will be elaborated by activator 1596 if self.SystemSettings.has_key(self.srv_sys_settings_plugin): 1597 if self.SystemSettings[self.srv_sys_settings_plugin]['server']['rss']['enabled'] and \ 1598 (not self.clientDatabase) and do_rss: 1599 1600 # store addPackage action 1601 self._write_rss_for_removed_package(idpackage) 1602 1603 with self.__write_mutex: 1604 1605 r_tup = (idpackage,)*20 1606 self.cursor.executescript(""" 1607 DELETE FROM baseinfo WHERE idpackage = %d; 1608 DELETE FROM extrainfo WHERE idpackage = %d; 1609 DELETE FROM dependencies WHERE idpackage = %d; 1610 DELETE FROM provide WHERE idpackage = %d; 1611 DELETE FROM conflicts WHERE idpackage = %d; 1612 DELETE FROM configprotect WHERE idpackage = %d; 1613 DELETE FROM configprotectmask WHERE idpackage = %d; 1614 DELETE FROM sources WHERE idpackage = %d; 1615 DELETE FROM useflags WHERE idpackage = %d; 1616 DELETE FROM keywords WHERE idpackage = %d; 1617 DELETE FROM content WHERE idpackage = %d; 1618 DELETE FROM messages WHERE idpackage = %d; 1619 DELETE FROM counters WHERE idpackage = %d; 1620 DELETE FROM sizes WHERE idpackage = %d; 1621 DELETE FROM eclasses WHERE idpackage = %d; 1622 DELETE FROM needed WHERE idpackage = %d; 1623 DELETE FROM triggers WHERE idpackage = %d; 1624 DELETE FROM systempackages WHERE idpackage = %d; 1625 DELETE FROM injected WHERE idpackage = %d; 1626 DELETE FROM installedtable WHERE idpackage = %d; 1627 """ % r_tup) 1628 1629 # not yet possible to add these calls above 1630 try: 1631 self.removeAutomergefiles(idpackage) 1632 except self.dbapi2.OperationalError: 1633 pass 1634 try: 1635 self.removeSignatures(idpackage) 1636 except self.dbapi2.OperationalError: 1637 pass 1638 try: 1639 self.removeSpmPhases(idpackage) 1640 except self.dbapi2.OperationalError: 1641 pass 1642 1643 # Remove from dependstable if exists 1644 self.removePackageFromDependsTable(idpackage) 1645 1646 if do_cleanup: 1647 # Cleanups if at least one package has been removed 1648 self.doCleanups() 1649 1650 if do_commit: 1651 self.commitChanges()

1652

1653 - def removeMirrorEntries(self, mirrorname):

1654 with self.__write_mutex: 1655 self.cursor.execute(""" 1656 DELETE FROM mirrorlinks WHERE mirrorname = (?) 1657 """,(mirrorname,))

1658

1659 - def addMirrors(self, mirrorname, mirrorlist):

1660 with self.__write_mutex: 1661 data = [(mirrorname, x,) for x in mirrorlist] 1662 self.cursor.executemany(""" 1663 INSERT into mirrorlinks VALUES (?,?) 1664 """, data)

1665

1666 - def addCategory(self, category):

1667 with self.__write_mutex: 1668 self.cursor.execute(""" 1669 INSERT into categories VALUES (NULL,?) 1670 """, (category,)) 1671 return self.cursor.lastrowid

1672

1673 - def addProtect(self, protect):

1674 with self.__write_mutex: 1675 self.cursor.execute(""" 1676 INSERT into configprotectreference VALUES (NULL,?) 1677 """, (protect,)) 1678 return self.cursor.lastrowid

1679 1680

1681 - def addSource(self, source):

1682 with self.__write_mutex: 1683 self.cursor.execute(""" 1684 INSERT into sourcesreference VALUES (NULL,?) 1685 """, (source,)) 1686 return self.cursor.lastrowid

1687

1688 - def addDependency(self, dependency):

1689 with self.__write_mutex: 1690 self.cursor.execute('INSERT into dependenciesreference VALUES (NULL,?)', (dependency,)) 1691 return self.cursor.lastrowid

1692

1693 - def addKeyword(self, keyword):

1694 with self.__write_mutex: 1695 self.cursor.execute('INSERT into keywordsreference VALUES (NULL,?)', (keyword,)) 1696 return self.cursor.lastrowid

1697

1698 - def addUseflag(self, useflag):

1699 with self.__write_mutex: 1700 self.cursor.execute('INSERT into useflagsreference VALUES (NULL,?)', (useflag,)) 1701 return self.cursor.lastrowid

1702

1703 - def addEclass(self, eclass):

1704 with self.__write_mutex: 1705 self.cursor.execute('INSERT into eclassesreference VALUES (NULL,?)', (eclass,)) 1706 return self.cursor.lastrowid

1707

1708 - def addNeeded(self, needed):

1709 with self.__write_mutex: 1710 self.cursor.execute('INSERT into neededreference VALUES (NULL,?)', (needed,)) 1711 return self.cursor.lastrowid

1712

1713 - def addLicense(self, pkglicense):

1714 if not self.entropyTools.is_valid_string(pkglicense): 1715 pkglicense = ' ' # workaround for broken license entries 1716 with self.__write_mutex: 1717 self.cursor.execute('INSERT into licenses VALUES (NULL,?)', (pkglicense,)) 1718 return self.cursor.lastrowid

1719

1720 - def addCompileFlags(self, chost, cflags, cxxflags):

1721 with self.__write_mutex: 1722 self.cursor.execute('INSERT into flags VALUES (NULL,?,?,?)', (chost,cflags,cxxflags,)) 1723 return self.cursor.lastrowid

1724

1725 - def setSystemPackage(self, idpackage, do_commit = True):

1726 with self.__write_mutex: 1727 self.cursor.execute('INSERT into systempackages VALUES (?)', (idpackage,)) 1728 if do_commit: 1729 self.commitChanges()

1730

1731 - def setInjected(self, idpackage, do_commit = True):

1732 with self.__write_mutex: 1733 if not self.isInjected(idpackage): 1734 self.cursor.execute('INSERT into injected VALUES (?)', (idpackage,)) 1735 if do_commit: 1736 self.commitChanges()

1737 1738 # date expressed the unix way

1739 - def setDateCreation(self, idpackage, date):

1740 with self.__write_mutex: 1741 self.cursor.execute('UPDATE extrainfo SET datecreation = (?) WHERE idpackage = (?)', (str(date), idpackage,)) 1742 self.commitChanges()

1743

1744 - def setDigest(self, idpackage, digest):

1745 with self.__write_mutex: 1746 self.cursor.execute('UPDATE extrainfo SET digest = (?) WHERE idpackage = (?)', (digest, idpackage,)) 1747 self.commitChanges()

1748

1749 - def setSignatures(self, idpackage, signatures):

1750 with self.__write_mutex: 1751 sha1, sha256, sha512 = signatures['sha1'], signatures['sha256'], \ 1752 signatures['sha512'] 1753 self.cursor.execute(""" 1754 UPDATE packagesignatures SET sha1 = (?), sha256 = (?), sha512 = (?) 1755 WHERE idpackage = (?) 1756 """, (sha1, sha256, sha512, idpackage))

1757

1758 - def setDownloadURL(self, idpackage, url):

1759 with self.__write_mutex: 1760 self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (url, idpackage,)) 1761 self.commitChanges()

1762

1763 - def setCategory(self, idpackage, category):

1764 # create new category if it doesn't exist 1765 catid = self.isCategoryAvailable(category) 1766 if (catid == -1): 1767 # create category 1768 catid = self.addCategory(category) 1769 with self.__write_mutex: 1770 self.cursor.execute('UPDATE baseinfo SET idcategory = (?) WHERE idpackage = (?)', (catid, idpackage,)) 1771 self.commitChanges()

1772

1773 - def setCategoryDescription(self, category, description_data):

1774 with self.__write_mutex: 1775 self.cursor.execute('DELETE FROM categoriesdescription WHERE category = (?)', (category,)) 1776 for locale in description_data: 1777 mydesc = description_data[locale] 1778 #if type(mydesc) is unicode: 1779 # mydesc = mydesc.encode('raw_unicode_escape') 1780 self.cursor.execute('INSERT INTO categoriesdescription VALUES (?,?,?)', (category, locale, mydesc,)) 1781 self.commitChanges()

1782

1783 - def setName(self, idpackage, name):

1784 with self.__write_mutex: 1785 self.cursor.execute('UPDATE baseinfo SET name = (?) WHERE idpackage = (?)', (name, idpackage,)) 1786 self.commitChanges()

1787

1788 - def setDependency(self, iddependency, dependency):

1789 with self.__write_mutex: 1790 self.cursor.execute('UPDATE dependenciesreference SET dependency = (?) WHERE iddependency = (?)', (dependency, iddependency,)) 1791 self.commitChanges()

1792

1793 - def setAtom(self, idpackage, atom):

1794 with self.__write_mutex: 1795 self.cursor.execute('UPDATE baseinfo SET atom = (?) WHERE idpackage = (?)', (atom, idpackage,)) 1796 self.commitChanges()

1797

1798 - def setSlot(self, idpackage, slot):

1799 with self.__write_mutex: 1800 self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot, idpackage,)) 1801 self.commitChanges()

1802

1803 - def removeLicensedata(self, license_name):

1804 with self.__write_mutex: 1805 self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,))

1806

1807 - def removeDependencies(self, idpackage):

1808 with self.__write_mutex: 1809 self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,)) 1810 self.commitChanges()

1811

1812 - def insertDependencies(self, idpackage, depdata):

1813 1814 dcache = set() 1815 add_dep = self.addDependency 1816 is_dep_avail = self.isDependencyAvailable 1817 def mymf(dep): 1818 1819 if dep in dcache: return 0 1820 iddep = is_dep_avail(dep) 1821 if iddep == -1: iddep = add_dep(dep) 1822 1823 deptype = 0 1824 if isinstance(depdata, dict): 1825 deptype = depdata[dep] 1826 1827 dcache.add(dep) 1828 return (idpackage, iddep, deptype,)

1829 1830 # do not place inside the with statement, otherwise there'll be an obvious lockup 1831 deps = [x for x in map(mymf, depdata) if type(x) != int] 1832 with self.__write_mutex: 1833 self.cursor.executemany('INSERT into dependencies VALUES (?,?,?)', deps)

1834

1835 - def insertManualDependencies(self, idpackage, manual_deps):

1836 mydict = {} 1837 for manual_dep in manual_deps: 1838 mydict[manual_dep] = etpConst['spm']['mdepend_id'] 1839 return self.insertDependencies(idpackage, mydict)

1840

1841 - def removeContent(self, idpackage):

1842 with self.__write_mutex: 1843 self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,)) 1844 self.commitChanges()

1845

1846 - def insertContent(self, idpackage, content, already_formatted = False):

1847 1848 with self.__write_mutex: 1849 if already_formatted: 1850 self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',((idpackage, x, y,) for a, x, y in content)) 1851 return 1852 def my_cmap(xfile): 1853 return (idpackage, xfile, content[xfile],)

1854 self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',map(my_cmap, content)) 1855

1856 - def insertNeededPaths(self, library, paths):

1857 with self.__write_mutex: 1858 self.cursor.executemany('INSERT OR IGNORE INTO neededlibrarypaths VALUES (?,?,?)', 1859 ((library, path, elfclass,) for path, elfclass in paths))

1860

1861 - def insertAutomergefiles(self, idpackage, automerge_data):

1862 with self.__write_mutex: 1863 self.cursor.executemany('INSERT INTO automergefiles VALUES (?,?,?)', 1864 ((idpackage, x, y,) for x, y in automerge_data))

1865

1866 - def removeAutomergefiles(self, idpackage):

1867 with self.__write_mutex: 1868 self.cursor.execute('DELETE FROM automergefiles WHERE idpackage = (?)', (idpackage,))

1869

1870 - def removeSignatures(self, idpackage):

1871 with self.__write_mutex: 1872 self.cursor.execute('DELETE FROM packagesignatures WHERE idpackage = (?)', (idpackage,))

1873

1874 - def removeSpmPhases(self, idpackage):

1875 with self.__write_mutex: 1876 self.cursor.execute('DELETE FROM packagespmphases WHERE idpackage = (?)', (idpackage,))

1877

1878 - def insertChangelog(self, category, name, changelog_txt):

1879 with self.__write_mutex: 1880 mytxt = changelog_txt.encode('raw_unicode_escape') 1881 self.cursor.execute('DELETE FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,)) 1882 self.cursor.execute('INSERT INTO packagechangelogs VALUES (?,?,?)', (category, name, buffer(mytxt),))

1883

1884 - def removeChangelog(self, category, name):

1885 with self.__write_mutex: 1886 self.cursor.execute('DELETE FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,))

1887

1888 - def insertLicenses(self, licenses_data):

1889 1890 mylicenses = licenses_data.keys() 1891 is_lic_avail = self.isLicensedataKeyAvailable 1892 def my_mf(mylicense): 1893 return not is_lic_avail(mylicense)

1894 1895 def my_mm(mylicense): 1896 lic_data = licenses_data.get(mylicense,'') 1897 # support both utf8 and str input 1898 if isinstance(lic_data, unicode): # encode to str 1899 try: 1900 lic_data = lic_data.encode('raw_unicode_escape') 1901 except (UnicodeDecodeError,): 1902 lic_data = lic_data.encode('utf-8') 1903 return (mylicense, buffer(lic_data), 0,) 1904 1905 with self.__write_mutex: 1906 self.cursor.executemany('INSERT into licensedata VALUES (?,?,?)',map(my_mm, list(set(filter(my_mf, mylicenses))))) 1907

1908 - def insertConfigProtect(self, idpackage, idprotect, mask = False):

1909 1910 mytable = 'configprotect' 1911 if mask: mytable += 'mask' 1912 with self.__write_mutex: 1913 self.cursor.execute('INSERT into %s VALUES (?,?)' % (mytable,), (idpackage, idprotect,))

1914

1915 - def insertMirrors(self, mirrors):

1916 1917 for mirrorname, mirrorlist in mirrors: 1918 # remove old 1919 self.removeMirrorEntries(mirrorname) 1920 # add new 1921 self.addMirrors(mirrorname, mirrorlist)

1922

1923 - def insertKeywords(self, idpackage, keywords):

1924 1925 mydata = set() 1926 for key in keywords: 1927 idkeyword = self.isKeywordAvailable(key) 1928 if (idkeyword == -1): 1929 # create category 1930 idkeyword = self.addKeyword(key) 1931 mydata.add((idpackage, idkeyword,)) 1932 1933 with self.__write_mutex: 1934 self.cursor.executemany('INSERT into keywords VALUES (?,?)', mydata)

1935

1936 - def insertUseflags(self, idpackage, useflags):

1937 1938 mydata = set() 1939 for flag in useflags: 1940 iduseflag = self.isUseflagAvailable(flag) 1941 if (iduseflag == -1): 1942 # create category 1943 iduseflag = self.addUseflag(flag) 1944 mydata.add((idpackage, iduseflag,)) 1945 1946 with self.__write_mutex: 1947 self.cursor.executemany('INSERT into useflags VALUES (?,?)', mydata)

1948

1949 - def insertSignatures(self, idpackage, signatures):

1950 with self.__write_mutex: 1951 sha1, sha256, sha512 = signatures['sha1'], signatures['sha256'], \ 1952 signatures['sha512'] 1953 self.cursor.execute(""" 1954 INSERT INTO packagesignatures VALUES (?,?,?,?) 1955 """, (idpackage, sha1, sha256, sha512))

1956

1957 - def insertSpmPhases(self, idpackage, phases):

1958 with self.__write_mutex: 1959 self.cursor.execute(""" 1960 INSERT INTO packagespmphases VALUES (?,?) 1961 """, (idpackage,phases,))

1962

1963 - def insertSources(self, idpackage, sources):

1964 1965 mydata = set() 1966 for source in sources: 1967 if (not source) or (source == "") or (not self.entropyTools.is_valid_string(source)): 1968 continue 1969 idsource = self.isSourceAvailable(source) 1970 if (idsource == -1): 1971 # create category 1972 idsource = self.addSource(source) 1973 mydata.add((idpackage, idsource,)) 1974 1975 with self.__write_mutex: 1976 self.cursor.executemany('INSERT into sources VALUES (?,?)', mydata)

1977

1978 - def insertConflicts(self, idpackage, conflicts):

1979 1980 def myiter(): 1981 for conflict in conflicts: 1982 yield (idpackage, conflict,)

1983 1984 with self.__write_mutex: 1985 self.cursor.executemany('INSERT into conflicts VALUES (?,?)', myiter()) 1986

1987 - def insertMessages(self, idpackage, messages):

1988 1989 def myiter(): 1990 for message in messages: 1991 yield (idpackage, message,)

1992 1993 with self.__write_mutex: 1994 self.cursor.executemany('INSERT into messages VALUES (?,?)', myiter()) 1995

1996 - def insertProvide(self, idpackage, provides):

1997 1998 def myiter(): 1999 for atom in provides: 2000 yield (idpackage, atom,)

2001 2002 with self.__write_mutex: 2003 self.cursor.executemany('INSERT into provide VALUES (?,?)', myiter()) 2004

2005 - def insertNeeded(self, idpackage, neededs):

2006 2007 mydata = set() 2008 for needed, elfclass in neededs: 2009 idneeded = self.isNeededAvailable(needed) 2010 if idneeded == -1: 2011 # create eclass 2012 idneeded = self.addNeeded(needed) 2013 mydata.add((idpackage, idneeded, elfclass)) 2014 2015 with self.__write_mutex: 2016 self.cursor.executemany('INSERT into needed VALUES (?,?,?)', mydata)

2017

2018 - def insertEclasses(self, idpackage, eclasses):

2019 2020 mydata = set() 2021 for eclass in eclasses: 2022 idclass = self.isEclassAvailable(eclass) 2023 if (idclass == -1): 2024 # create eclass 2025 idclass = self.addEclass(eclass) 2026 mydata.add((idpackage, idclass,)) 2027 2028 with self.__write_mutex: 2029 self.cursor.executemany('INSERT into eclasses VALUES (?,?)', mydata)

2030

2031 - def insertOnDiskSize(self, idpackage, mysize):

2032 with self.__write_mutex: 2033 self.cursor.execute('INSERT into sizes VALUES (?,?)', (idpackage, mysize,))

2034

2035 - def insertTrigger(self, idpackage, trigger):

2036 with self.__write_mutex: 2037 self.cursor.execute('INSERT into triggers VALUES (?,?)', (idpackage, buffer(trigger),))

2038

2039 - def insertPortageCounter(self, idpackage, counter, branch, injected):

2040 2041 if (counter != -1) and not injected: 2042 2043 if counter <= -2: 2044 # special cases 2045 counter = self.getNewNegativeCounter() 2046 2047 with self.__write_mutex: 2048 try: 2049 self.cursor.execute( 2050 'INSERT into counters VALUES ' 2051 '(?,?,?)' 2052 , ( counter, 2053 idpackage, 2054 branch, 2055 ) 2056 ) 2057 except self.dbapi2.IntegrityError: 2058 # we have a PRIMARY KEY we need to remove 2059 self._migrateCountersTable() 2060 self.cursor.execute( 2061 'INSERT into counters VALUES ' 2062 '(?,?,?)' 2063 , ( counter, 2064 idpackage, 2065 branch, 2066 ) 2067 ) 2068 except: 2069 if self.dbname == etpConst['clientdbid']: 2070 # force only for client database 2071 if self.doesTableExist("counters"): 2072 raise 2073 self.cursor.execute( 2074 'INSERT into counters VALUES ' 2075 '(?,?,?)' 2076 , ( counter, 2077 idpackage, 2078 branch, 2079 ) 2080 ) 2081 elif self.dbname.startswith(etpConst['serverdbid']): 2082 raise 2083 2084 return counter

2085

2086 - def insertCounter(self, idpackage, counter, branch = None):

2087 if not branch: branch = self.db_branch 2088 if not branch: branch = self.SystemSettings['repositories']['branch'] 2089 with self.__write_mutex: 2090 self.cursor.execute(""" 2091 DELETE FROM counters 2092 WHERE (counter = (?) OR 2093 idpackage = (?)) AND 2094 branch = (?)""", (counter, idpackage, branch,)) 2095 self.cursor.execute('INSERT INTO counters VALUES (?,?,?)', (counter, idpackage, branch,)) 2096 self.commitChanges()

2097

2098 - def setTrashedCounter(self, counter):

2099 with self.__write_mutex: 2100 self.cursor.execute('DELETE FROM trashedcounters WHERE counter = (?)', (counter,)) 2101 self.cursor.execute('INSERT INTO trashedcounters VALUES (?)', (counter,)) 2102 self.commitChanges()

2103 2104

2105 - def setCounter(self, idpackage, counter, branch = None):

2106 2107 branchstring = '' 2108 insertdata = [counter, idpackage] 2109 if branch: 2110 branchstring = ', branch = (?)' 2111 insertdata.insert(1, branch) 2112 2113 with self.__write_mutex: 2114 try: 2115 self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata) 2116 self.commitChanges() 2117 except: 2118 if self.dbname == etpConst['clientdbid']: 2119 raise

2120

2121 - def contentDiff(self, idpackage, dbconn, dbconn_idpackage):

2122 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2123 # create a random table and fill 2124 randomtable = "cdiff%s" % (self.entropyTools.get_random_number(),) 2125 while self.doesTableExist(randomtable): 2126 randomtable = "cdiff%s" % (self.entropyTools.get_random_number(),) 2127 self.cursor.execute('CREATE TEMPORARY TABLE %s ( file VARCHAR )' % (randomtable,)) 2128 2129 try: 2130 dbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2131 dbconn.cursor.execute('select file from content where idpackage = (?)', (dbconn_idpackage,)) 2132 xfile = dbconn.cursor.fetchone() 2133 while xfile: 2134 self.cursor.execute('INSERT INTO %s VALUES (?)' % (randomtable,), (xfile[0],)) 2135 xfile = dbconn.cursor.fetchone() 2136 2137 # now compare 2138 self.cursor.execute(""" 2139 SELECT file FROM content 2140 WHERE content.idpackage = (?) AND 2141 content.file NOT IN (SELECT file from %s)""" % (randomtable,), (idpackage,)) 2142 diff = self.fetchall2set(self.cursor.fetchall()) 2143 return diff 2144 finally: 2145 self.cursor.execute('DROP TABLE IF EXISTS %s' % (randomtable,))

2146

2147 - def doCleanups(self):

2148 self.cleanupUseflags() 2149 self.cleanupSources() 2150 self.cleanupEclasses() 2151 self.cleanupNeeded() 2152 self.cleanupNeededPaths() 2153 self.cleanupDependencies() 2154 self.cleanupChangelogs()

2155

2156 - def cleanupUseflags(self):

2157 with self.__write_mutex: 2158 self.cursor.execute(""" 2159 DELETE FROM useflagsreference 2160 WHERE idflag NOT IN (SELECT idflag FROM useflags)""")

2161

2162 - def cleanupSources(self):

2163 with self.__write_mutex: 2164 self.cursor.execute(""" 2165 DELETE FROM sourcesreference 2166 WHERE idsource NOT IN (SELECT idsource FROM sources)""")

2167

2168 - def cleanupEclasses(self):

2169 with self.__write_mutex: 2170 self.cursor.execute(""" 2171 DELETE FROM eclassesreference 2172 WHERE idclass NOT IN (SELECT idclass FROM eclasses)""")

2173

2174 - def cleanupNeeded(self):

2175 with self.__write_mutex: 2176 self.cursor.execute(""" 2177 DELETE FROM neededreference 2178 WHERE idneeded NOT IN (SELECT idneeded FROM needed)""")

2179

2180 - def cleanupNeededPaths(self):

2181 with self.__write_mutex: 2182 self.cursor.execute(""" 2183 DELETE FROM neededlibrarypaths 2184 WHERE library NOT IN (SELECT library FROM neededreference)""")

2185

2186 - def cleanupDependencies(self):

2187 with self.__write_mutex: 2188 self.cursor.execute(""" 2189 DELETE FROM dependenciesreference 2190 WHERE iddependency NOT IN (SELECT iddependency FROM dependencies)""")

2191

2192 - def cleanupChangelogs(self):

2193 with self.__write_mutex: 2194 self.cursor.execute(""" 2195 DELETE FROM packagechangelogs 2196 WHERE category || "/" || name NOT IN 2197 (SELECT categories.category || "/" || baseinfo.name FROM baseinfo,categories 2198 WHERE baseinfo.idcategory = categories.idcategory 2199 )""")

2200

2201 - def getNewNegativeCounter(self):

2202 counter = -2 2203 try: 2204 self.cursor.execute('SELECT min(counter) FROM counters') 2205 dbcounter = self.cursor.fetchone() 2206 mycounter = 0 2207 if dbcounter: 2208 mycounter = dbcounter[0] 2209 2210 if mycounter >= -1: 2211 counter = -2 2212 else: 2213 counter = mycounter-1 2214 2215 except: 2216 pass 2217 return counter

2218

2219 - def getApi(self):

2220 self.cursor.execute('SELECT max(etpapi) FROM baseinfo') 2221 api = self.cursor.fetchone() 2222 if api: api = api[0] 2223 else: api = -1 2224 return api

2225

2226 - def getCategory(self, idcategory):

2227 self.cursor.execute('SELECT category from categories WHERE idcategory = (?)', (idcategory,)) 2228 cat = self.cursor.fetchone() 2229 if cat: cat = cat[0] 2230 return cat

2231

2232 - def get_category_description_from_disk(self, category):

2233 if not self.ServiceInterface: 2234 return {} 2235 return self.ServiceInterface.SpmService.get_category_description_data(category)

2236

2237 - def getIDPackage(self, atom, branch = None):

2238 branch_string = '' 2239 params = [atom] 2240 if branch: 2241 branch_string = ' AND branch = (?)' 2242 params.append(branch) 2243 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)'+branch_string, params) 2244 idpackage = self.cursor.fetchone() 2245 if idpackage: return idpackage[0] 2246 return -1

2247

2248 - def getIDPackageFromDownload(self, download_relative_path, endswith = False):

2249 if endswith: 2250 self.cursor.execute(""" 2251 SELECT baseinfo.idpackage FROM baseinfo,extrainfo 2252 WHERE extrainfo.download LIKE (?)""", ("%"+download_relative_path,)) 2253 else: 2254 self.cursor.execute(""" 2255 SELECT baseinfo.idpackage FROM baseinfo,extrainfo 2256 WHERE extrainfo.download = (?)""", (download_relative_path,)) 2257 idpackage = self.cursor.fetchone() 2258 if idpackage: return idpackage[0] 2259 return -1

2260

2261 - def getIDPackagesFromFile(self, file):

2262 self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (file,)) 2263 return self.fetchall2list(self.cursor.fetchall())

2264

2265 - def getIDCategory(self, category):

2266 self.cursor.execute('SELECT "idcategory" FROM categories WHERE category = (?)', (category,)) 2267 idcat = self.cursor.fetchone() 2268 if idcat: return idcat[0] 2269 return -1

2270

2271 - def getVersioningData(self, idpackage):

2272 self.cursor.execute('SELECT version,versiontag,revision FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2273 return self.cursor.fetchone()

2274

2275 - def getStrictData(self, idpackage):

2276 self.cursor.execute(""" 2277 SELECT categories.category || "/" || baseinfo.name, 2278 baseinfo.slot,baseinfo.version,baseinfo.versiontag, 2279 baseinfo.revision,baseinfo.atom FROM baseinfo,categories 2280 WHERE baseinfo.idpackage = (?) AND 2281 baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2282 return self.cursor.fetchone()

2283

2284 - def getStrictScopeData(self, idpackage):

2285 self.cursor.execute(""" 2286 SELECT atom,slot,revision FROM baseinfo 2287 WHERE idpackage = (?)""", (idpackage,)) 2288 rslt = self.cursor.fetchone() 2289 return rslt

2290

2291 - def getScopeData(self, idpackage):

2292 self.cursor.execute(""" 2293 SELECT 2294 baseinfo.atom, 2295 categories.category, 2296 baseinfo.name, 2297 baseinfo.version, 2298 baseinfo.slot, 2299 baseinfo.versiontag, 2300 baseinfo.revision, 2301 baseinfo.branch, 2302 baseinfo.etpapi 2303 FROM 2304 baseinfo, 2305 categories 2306 WHERE 2307 baseinfo.idpackage = (?) 2308 and baseinfo.idcategory = categories.idcategory 2309 """, (idpackage,)) 2310 return self.cursor.fetchone()

2311

2312 - def getBaseData(self, idpackage):

2313 2314 sql = """ 2315 SELECT 2316 baseinfo.atom, 2317 baseinfo.name, 2318 baseinfo.version, 2319 baseinfo.versiontag, 2320 extrainfo.description, 2321 categories.category, 2322 flags.chost, 2323 flags.cflags, 2324 flags.cxxflags, 2325 extrainfo.homepage, 2326 licenses.license, 2327 baseinfo.branch, 2328 extrainfo.download, 2329 extrainfo.digest, 2330 baseinfo.slot, 2331 baseinfo.etpapi, 2332 extrainfo.datecreation, 2333 extrainfo.size, 2334 baseinfo.revision 2335 FROM 2336 baseinfo, 2337 extrainfo, 2338 categories, 2339 flags, 2340 licenses 2341 WHERE 2342 baseinfo.idpackage = (?) 2343 and baseinfo.idpackage = extrainfo.idpackage 2344 and baseinfo.idcategory = categories.idcategory 2345 and extrainfo.idflags = flags.idflags 2346 and baseinfo.idlicense = licenses.idlicense 2347 """ 2348 self.cursor.execute(sql, (idpackage,)) 2349 return self.cursor.fetchone()

2350

2351 - def getTriggerInfo(self, idpackage, content = True):

2352 2353 atom, category, name, \ 2354 version, slot, versiontag, \ 2355 revision, branch, etpapi = self.getScopeData(idpackage) 2356 chost, cflags, cxxflags = self.retrieveCompileFlags(idpackage) 2357 2358 pkg_content = set() 2359 if content: 2360 pkg_content = self.retrieveContent(idpackage) 2361 2362 data = { 2363 'atom': atom, 2364 'category': category, 2365 'name': name, 2366 'version': version, 2367 'versiontag': versiontag, 2368 'revision': revision, 2369 'branch': branch, 2370 'chost': chost, 2371 'cflags': cflags, 2372 'cxxflags': cxxflags, 2373 'etpapi': etpapi, 2374 'trigger': self.retrieveTrigger(idpackage), 2375 'eclasses': self.retrieveEclasses(idpackage), 2376 'content': pkg_content, 2377 'spm_phases': self.retrieveSpmPhases(idpackage), 2378 } 2379 return data

2380

2381 - def getPackageData(self, idpackage, get_content = True, 2382 content_insert_formatted = False, trigger_unicode = True):

2383 data = {} 2384 2385 try: 2386 atom, name, version, versiontag, \ 2387 description, category, chost, \ 2388 cflags, cxxflags,homepage, \ 2389 mylicense, branch, download, \ 2390 digest, slot, etpapi, \ 2391 datecreation, size, revision = self.getBaseData(idpackage) 2392 except TypeError: 2393 return None 2394 2395 content = {} 2396 if get_content: 2397 content = self.retrieveContent( 2398 idpackage, extended = True, 2399 formatted = True, insert_formatted = content_insert_formatted 2400 ) 2401 2402 sources = self.retrieveSources(idpackage) 2403 mirrornames = set() 2404 for x in sources: 2405 if x.startswith("mirror://"): 2406 mirrornames.add(x.split("/")[2]) 2407 2408 data = { 2409 'atom': atom, 2410 'name': name, 2411 'version': version, 2412 'versiontag':versiontag, 2413 'description': description, 2414 'category': category, 2415 'chost': chost, 2416 'cflags': cflags, 2417 'cxxflags': cxxflags, 2418 'homepage': homepage, 2419 'license': mylicense, 2420 'branch': branch, 2421 'download': download, 2422 'digest': digest, 2423 'slot': slot, 2424 'etpapi': etpapi, 2425 'datecreation': datecreation, 2426 'size': size, 2427 'revision': revision, 2428 # risky to add to the sql above, still 2429 'counter': self.retrieveCounter(idpackage), 2430 'messages': self.retrieveMessages(idpackage), 2431 'trigger': self.retrieveTrigger(idpackage, get_unicode = trigger_unicode), 2432 'disksize': self.retrieveOnDiskSize(idpackage), 2433 'changelog': self.retrieveChangelog(idpackage), 2434 'injected': self.isInjected(idpackage), 2435 'systempackage': self.isSystemPackage(idpackage), 2436 'config_protect': self.retrieveProtect(idpackage), 2437 'config_protect_mask': self.retrieveProtectMask(idpackage), 2438 'useflags': self.retrieveUseflags(idpackage), 2439 'keywords': self.retrieveKeywords(idpackage), 2440 'sources': sources, 2441 'eclasses': self.retrieveEclasses(idpackage), 2442 'needed': self.retrieveNeeded(idpackage, extended = True), 2443 'needed_paths': self.retrieveNeededPaths(idpackage), 2444 'provide': self.retrieveProvide(idpackage), 2445 'conflicts': self.retrieveConflicts(idpackage), 2446 'licensedata': self.retrieveLicensedata(idpackage), 2447 'content': content, 2448 'dependencies': dict((x, y,) for x, y in \ 2449 self.retrieveDependencies(idpackage, extended = True)), 2450 'mirrorlinks': [[x,self.retrieveMirrorInfo(x)] for x in mirrornames], 2451 'signatures': self.retrieveSignatures(idpackage), 2452 'spm_phases': self.retrieveSpmPhases(idpackage), 2453 } 2454 2455 return data

2456

2457 - def fetchall2set(self, item):

2458 mycontent = set() 2459 for x in item: 2460 mycontent |= set(x) 2461 return mycontent

2462

2463 - def fetchall2list(self, item):

2464 content = [] 2465 for x in item: 2466 content += list(x) 2467 return content

2468

2469 - def fetchone2list(self, item):

2470 return list(item)

2471

2472 - def fetchone2set(self, item):

2473 return set(item)

2474

2475 - def clearCache(self, depends = False):

2476 2477 self.live_cache.clear() 2478 def do_clear(name): 2479 dump_path = os.path.join(etpConst['dumpstoragedir'], name) 2480 dump_dir = os.path.dirname(dump_path) 2481 if os.path.isdir(dump_dir): 2482 for item in os.listdir(dump_dir): 2483 try: os.remove(os.path.join(dump_dir, item)) 2484 except OSError: pass

2485 2486 do_clear("%s/%s/" % (self.dbMatchCacheKey, self.dbname,)) 2487 if depends: 2488 do_clear(etpCache['depends_tree']) 2489 do_clear(etpCache['dep_tree']) 2490 do_clear(etpCache['filter_satisfied_deps']) 2491

2492 - def retrieveRepositoryUpdatesDigest(self, repository):

2493 if not self.doesTableExist("treeupdates"): 2494 return -1 2495 self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,)) 2496 mydigest = self.cursor.fetchone() 2497 if mydigest: 2498 return mydigest[0] 2499 else: 2500 return -1

2501

2502 - def listAllTreeUpdatesActions(self, no_ids_repos = False):

2503 if no_ids_repos: 2504 self.cursor.execute('SELECT command,branch,date FROM treeupdatesactions') 2505 else: 2506 self.cursor.execute('SELECT * FROM treeupdatesactions') 2507 return self.cursor.fetchall()

2508

2509 - def retrieveTreeUpdatesActions(self, repository, forbranch = None):

2510 2511 if not self.doesTableExist("treeupdatesactions"): return [] 2512 if forbranch == None: forbranch = self.db_branch 2513 params = [repository] 2514 branch_string = '' 2515 if forbranch: 2516 branch_string = 'and branch = (?)' 2517 params.append(forbranch) 2518 2519 self.cursor.execute(""" 2520 SELECT command FROM treeupdatesactions WHERE 2521 repository = (?) %s order by date""" % (branch_string,), params) 2522 return self.fetchall2list(self.cursor.fetchall())

2523 2524 # mainly used to restore a previous table, used by reagent in --initialize

2525 - def bumpTreeUpdatesActions(self, updates):

2526 with self.__write_mutex: 2527 self.cursor.execute('DELETE FROM treeupdatesactions') 2528 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (?,?,?,?,?)', updates) 2529 self.commitChanges()

2530

2531 - def removeTreeUpdatesActions(self, repository):

2532 with self.__write_mutex: 2533 self.cursor.execute('DELETE FROM treeupdatesactions WHERE repository = (?)', (repository,)) 2534 self.commitChanges()

2535

2536 - def insertTreeUpdatesActions(self, updates, repository):

2537 with self.__write_mutex: 2538 myupdates = [[repository]+list(x) for x in updates] 2539 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', myupdates) 2540 self.commitChanges()

2541

2542 - def setRepositoryUpdatesDigest(self, repository, digest):

2543 with self.__write_mutex: 2544 self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety 2545 self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository, digest,))

2546

2547 - def addRepositoryUpdatesActions(self, repository, actions, branch):

2548 2549 mytime = str(self.entropyTools.get_current_unix_time()) 2550 with self.__write_mutex: 2551 myupdates = [ 2552 (repository, x, branch, mytime,) for x in actions \ 2553 if not self.doesTreeupdatesActionExist(repository, x, branch) 2554 ] 2555 self.cursor.executemany('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', myupdates)

2556

2557 - def doesTreeupdatesActionExist(self, repository, command, branch):

2558 self.cursor.execute(""" 2559 SELECT * FROM treeupdatesactions 2560 WHERE repository = (?) and command = (?) and branch = (?)""", (repository, command, branch,)) 2561 result = self.cursor.fetchone() 2562 if result: 2563 return True 2564 return False

2565

2566 - def clearPackageSets(self):

2567 self.cursor.execute('DELETE FROM packagesets')

2568

2569 - def insertPackageSets(self, sets_data):

2570 2571 mysets = [] 2572 for setname in sorted(sets_data.keys()): 2573 for dependency in sorted(sets_data[setname]): 2574 try: 2575 mysets.append((unicode(setname), unicode(dependency),)) 2576 except (UnicodeDecodeError, UnicodeEncodeError,): 2577 continue 2578 2579 with self.__write_mutex: 2580 self.cursor.executemany('INSERT INTO packagesets VALUES (?,?)', mysets)

2581

2582 - def retrievePackageSets(self):

2583 if not self.doesTableExist("packagesets"): return {} 2584 self.cursor.execute('SELECT setname,dependency FROM packagesets') 2585 data = self.cursor.fetchall() 2586 sets = {} 2587 for setname, dependency in data: 2588 if not sets.has_key(setname): 2589 sets[setname] = set() 2590 sets[setname].add(dependency) 2591 return sets

2592

2593 - def retrievePackageSet(self, setname):

2594 self.cursor.execute('SELECT dependency FROM packagesets WHERE setname = (?)', (setname,)) 2595 return self.fetchall2set(self.cursor.fetchall())

2596

2597 - def retrieveSystemPackages(self):

2598 self.cursor.execute('SELECT idpackage FROM systempackages') 2599 return self.fetchall2set(self.cursor.fetchall())

2600

2601 - def retrieveAtom(self, idpackage):

2602 self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2603 atom = self.cursor.fetchone() 2604 if atom: return atom[0]

2605

2606 - def retrieveBranch(self, idpackage):

2607 self.cursor.execute('SELECT branch FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2608 br = self.cursor.fetchone() 2609 if br: return br[0]

2610

2611 - def retrieveTrigger(self, idpackage, get_unicode = False):

2612 self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,)) 2613 trigger = self.cursor.fetchone() 2614 if not trigger: 2615 return '' # FIXME backward compatibility with <=0.52.x 2616 if not get_unicode: 2617 return trigger[0] 2618 return unicode(trigger[0], 'raw_unicode_escape')

2619

2620 - def retrieveDownloadURL(self, idpackage):

2621 self.cursor.execute('SELECT download FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2622 download = self.cursor.fetchone() 2623 if download: return download[0]

2624

2625 - def retrieveDescription(self, idpackage):

2626 self.cursor.execute('SELECT description FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2627 description = self.cursor.fetchone() 2628 if description: return description[0]

2629

2630 - def retrieveHomepage(self, idpackage):

2631 self.cursor.execute('SELECT homepage FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2632 home = self.cursor.fetchone() 2633 if home: return home[0]

2634

2635 - def retrieveCounter(self, idpackage):

2636 self.cursor.execute(""" 2637 SELECT counters.counter FROM counters,baseinfo 2638 WHERE counters.idpackage = (?) AND 2639 baseinfo.idpackage = counters.idpackage AND 2640 baseinfo.branch = counters.branch""", (idpackage,)) 2641 mycounter = self.cursor.fetchone() 2642 if mycounter: return mycounter[0] 2643 return -1

2644

2645 - def retrieveMessages(self, idpackage):

2646 self.cursor.execute('SELECT message FROM messages WHERE idpackage = (?)', (idpackage,)) 2647 return self.fetchall2list(self.cursor.fetchall())

2648 2649 # in bytes

2650 - def retrieveSize(self, idpackage):

2651 self.cursor.execute('SELECT size FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2652 size = self.cursor.fetchone() 2653 if size: return size[0]

2654 2655 # in bytes

2656 - def retrieveOnDiskSize(self, idpackage):

2657 self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,)) 2658 size = self.cursor.fetchone() # do not use [0]! 2659 if not size: size = 0 2660 else: size = size[0] 2661 return size

2662

2663 - def retrieveDigest(self, idpackage):

2664 self.cursor.execute('SELECT digest FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2665 digest = self.cursor.fetchone() 2666 if digest: return digest[0]

2667

2668 - def retrieveSignatures(self, idpackage):

2669 mydict = { 2670 'sha1': None, 2671 'sha256': None, 2672 'sha512': None, 2673 } 2674 # FIXME: remove this check in future 2675 if self.doesTableExist('packagesignatures'): 2676 self.cursor.execute(""" 2677 SELECT sha1, sha256, sha512 FROM packagesignatures 2678 WHERE idpackage = (?)""", (idpackage,)) 2679 data = self.cursor.fetchone() 2680 if data: 2681 mydict['sha1'], mydict['sha256'], mydict['sha512'] = data 2682 return mydict

2683

2684 - def retrieveName(self, idpackage):

2685 self.cursor.execute('SELECT name FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2686 name = self.cursor.fetchone() 2687 if name: return name[0]

2688

2689 - def retrieveKeySlot(self, idpackage):

2690 self.cursor.execute(""" 2691 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories 2692 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2693 data = self.cursor.fetchone() 2694 return data

2695

2696 - def retrieveKeySlotAggregated(self, idpackage):

2697 self.cursor.execute(""" 2698 SELECT categories.category || "/" || baseinfo.name || ":" || baseinfo.slot FROM baseinfo,categories 2699 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2700 data = self.cursor.fetchone() 2701 if data: return data[0]

2702

2703 - def retrieveKeySlotTag(self, idpackage):

2704 self.cursor.execute(""" 2705 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot,baseinfo.versiontag FROM baseinfo,categories 2706 WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory""", (idpackage,)) 2707 data = self.cursor.fetchone() 2708 return data

2709

2710 - def retrieveVersion(self, idpackage):

2711 self.cursor.execute('SELECT version FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2712 ver = self.cursor.fetchone() 2713 if ver: return ver[0]

2714

2715 - def retrieveRevision(self, idpackage):

2716 self.cursor.execute('SELECT revision FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2717 rev = self.cursor.fetchone() 2718 if rev: return rev[0]

2719

2720 - def retrieveDateCreation(self, idpackage):

2721 self.cursor.execute('SELECT datecreation FROM extrainfo WHERE idpackage = (?)', (idpackage,)) 2722 date = self.cursor.fetchone() 2723 if date: return date[0]

2724

2725 - def retrieveApi(self, idpackage):

2726 self.cursor.execute('SELECT etpapi FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 2727 api = self.cursor.fetchone() 2728 if api: return api[0]

2729

2730 - def retrieveUseflags(self, idpackage):

2731 self.cursor.execute(""" 2732 SELECT flagname FROM useflags,useflagsreference 2733 WHERE useflags.idpackage = (?) AND 2734 useflags.idflag = useflagsreference.idflag""", (idpackage,)) 2735 return self.fetchall2set(self.cursor.fetchall())

2736

2737 - def retrieveEclasses(self, idpackage):

2738 self.cursor.execute(""" 2739 SELECT classname FROM eclasses,eclassesreference 2740 WHERE eclasses.idpackage = (?) AND 2741 eclasses.idclass = eclassesreference.idclass""", (idpackage,)) 2742 return self.fetchall2set(self.cursor.fetchall())

2743

2744 - def retrieveSpmPhases(self, idpackage):

2745 # FIXME: remove this check in future: 2746 if not self.doesTableExist('packagespmphases'): 2747 return None 2748 self.cursor.execute(""" 2749 SELECT phases FROM packagespmphases 2750 WHERE idpackage = (?) 2751 """, (idpackage,)) 2752 rslt = self.cursor.fetchone() 2753 if rslt: return rslt[0]

2754

2755 - def retrieveNeededRaw(self, idpackage):

2756 self.cursor.execute(""" 2757 SELECT library FROM needed,neededreference 2758 WHERE needed.idpackage = (?) AND 2759 needed.idneeded = neededreference.idneeded""", (idpackage,)) 2760 return self.fetchall2set(self.cursor.fetchall())

2761

2762 - def retrieveNeeded(self, idpackage, extended = False, format = False):

2763 2764 if extended: 2765 self.cursor.execute(""" 2766 SELECT library,elfclass FROM needed,neededreference 2767 WHERE needed.idpackage = (?) AND 2768 needed.idneeded = neededreference.idneeded order by library""", (idpackage,)) 2769 needed = self.cursor.fetchall() 2770 else: 2771 self.cursor.execute(""" 2772 SELECT library FROM needed,neededreference 2773 WHERE needed.idpackage = (?) AND 2774 needed.idneeded = neededreference.idneeded ORDER BY library""", (idpackage,)) 2775 needed = self.fetchall2list(self.cursor.fetchall()) 2776 2777 if extended and format: 2778 data = {} 2779 for lib, elfclass in needed: 2780 data[lib] = elfclass 2781 needed = data 2782 2783 return needed

2784

2785 - def retrieveNeededPaths(self, idpackage):

2786 if not self.doesTableExist('neededlibrarypaths'): 2787 return {} 2788 self.cursor.execute(""" 2789 SELECT neededlibrarypaths.library, neededlibrarypaths.path, 2790 neededlibrarypaths.elfclass FROM 2791 neededlibrarypaths, neededreference, needed WHERE 2792 needed.idpackage = (?) AND needed.idneeded = neededreference.idneeded 2793 AND neededreference.library = neededlibrarypaths.library 2794 """, (idpackage,)) 2795 data = {} 2796 for lib, path, elfclass in self.cursor.fetchall(): 2797 obj = data.setdefault(lib, set()) 2798 obj.add((path, elfclass)) 2799 return data

2800

2801 - def retrieveNeededLibraryPaths(self, needed_library_name, elfclass):

2802 if not self.doesTableExist('neededlibrarypaths'): 2803 return set() 2804 self.cursor.execute(""" 2805 SELECT path FROM neededlibrarypaths, neededreference, needed 2806 WHERE neededlibrarypaths.library = (?) AND 2807 neededlibrarypaths.elfclass = (?) AND 2808 neededreference.library = neededlibrarypaths.library AND 2809 needed.elfclass = neededlibrarypaths.elfclass AND 2810 needed.idneeded = neededreference.idneeded 2811 """, (needed_library_name, elfclass,)) 2812 return self.fetchall2set(self.cursor.fetchall())

2813

2814 - def retrieveNeededLibraryIdpackages(self):

2815 if not self.doesTableExist('neededlibraryidpackages'): 2816 return [] 2817 self.cursor.execute('SELECT idpackage, library, elfclass FROM neededlibraryidpackages') 2818 return self.cursor.fetchall()

2819

2820 - def clearNeededLibraryIdpackages(self):

2821 if not self.doesTableExist('neededlibraryidpackages'): 2822 return 2823 self.cursor.execute('DELETE FROM neededlibraryidpackages')

2824

2825 - def setNeededLibraryIdpackages(self, library_map):

2826 if not self.doesTableExist('neededlibraryidpackages'): 2827 return 2828 self.cursor.executemany('INSERT INTO neededlibraryidpackages VALUES (?,?,?)', library_map)

2829

2830 - def retrieveConflicts(self, idpackage):

2831 self.cursor.execute('SELECT conflict FROM conflicts WHERE idpackage = (?)', (idpackage,)) 2832 return self.fetchall2set(self.cursor.fetchall())

2833

2834 - def retrieveProvide(self, idpackage):

2835 self.cursor.execute('SELECT atom FROM provide WHERE idpackage = (?)', (idpackage,)) 2836 return self.fetchall2set(self.cursor.fetchall())

2837

2838 - def retrieveDependenciesList(self, idpackage):

2839 self.cursor.execute(""" 2840 SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference 2841 WHERE dependencies.idpackage = (?) AND 2842 dependencies.iddependency = dependenciesreference.iddependency 2843 UNION SELECT "!" || conflict FROM conflicts 2844 WHERE idpackage = (?)""", (idpackage, idpackage,)) 2845 return self.fetchall2set(self.cursor.fetchall())

2846

2847 - def retrievePostDependencies(self, idpackage, extended = False):

2848 return self.retrieveDependencies(idpackage, extended = extended, deptype = etpConst['spm']['pdepend_id'])

2849

2850 - def retrieveManualDependencies(self, idpackage, extended = False):

2851 return self.retrieveDependencies(idpackage, extended = extended, deptype = etpConst['spm']['mdepend_id'])

2852

2853 - def retrieveDependencies(self, idpackage, extended = False, deptype = None, 2854 exclude_deptypes = None):

2855 2856 searchdata = [idpackage] 2857 2858 depstring = '' 2859 if deptype != None: 2860 depstring = ' and dependencies.type = (?)' 2861 searchdata.append(deptype) 2862 2863 excluded_deptypes_query = "" 2864 if exclude_deptypes != None: 2865 for dep_type in exclude_deptypes: 2866 excluded_deptypes_query += " AND dependencies.type != %s" % ( 2867 dep_type,) 2868 2869 if extended: 2870 self.cursor.execute(""" 2871 SELECT dependenciesreference.dependency,dependencies.type 2872 FROM dependencies,dependenciesreference 2873 WHERE dependencies.idpackage = (?) AND 2874 dependencies.iddependency = dependenciesreference.iddependency %s %s""" % ( 2875 depstring,excluded_deptypes_query,), searchdata) 2876 deps = self.cursor.fetchall() 2877 else: 2878 self.cursor.execute(""" 2879 SELECT dependenciesreference.dependency 2880 FROM dependencies,dependenciesreference 2881 WHERE dependencies.idpackage = (?) AND 2882 dependencies.iddependency = dependenciesreference.iddependency %s %s""" % ( 2883 depstring,excluded_deptypes_query,), searchdata) 2884 deps = self.fetchall2set(self.cursor.fetchall()) 2885 2886 return deps

2887

2888 - def retrieveIdDependencies(self, idpackage):

2889 self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = (?)', (idpackage,)) 2890 return self.fetchall2set(self.cursor.fetchall())

2891

2892 - def retrieveDependencyFromIddependency(self, iddependency):

2893 self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = (?)', (iddependency,)) 2894 dep = self.cursor.fetchone() 2895 if dep: dep = dep[0] 2896 return dep

2897

2898 - def retrieveKeywords(self, idpackage):

2899 self.cursor.execute(""" 2900 SELECT keywordname FROM keywords,keywordsreference 2901 WHERE keywords.idpackage = (?) AND 2902 keywords.idkeyword = keywordsreference.idkeyword""", (idpackage,)) 2903 return self.fetchall2set(self.cursor.fetchall())

2904

2905 - def retrieveProtect(self, idpackage):

2906 self.cursor.execute(""" 2907 SELECT protect FROM configprotect,configprotectreference 2908 WHERE configprotect.idpackage = (?) AND 2909 configprotect.idprotect = configprotectreference.idprotect""", (idpackage,)) 2910 protect = self.cursor.fetchone() 2911 if not protect: protect = '' 2912 else: protect = protect[0] 2913 return protect

2914

2915 - def retrieveProtectMask(self, idpackage):

2916 self.cursor.execute(""" 2917 SELECT protect FROM configprotectmask,configprotectreference 2918 WHERE idpackage = (?) AND 2919 configprotectmask.idprotect = configprotectreference.idprotect""", (idpackage,)) 2920 protect = self.cursor.fetchone() 2921 if not protect: protect = '' 2922 else: protect = protect[0] 2923 return protect

2924

2925 - def retrieveSources(self, idpackage, extended = False):

2926 self.cursor.execute(""" 2927 SELECT sourcesreference.source FROM sources,sourcesreference 2928 WHERE idpackage = (?) AND 2929 sources.idsource = sourcesreference.idsource""", (idpackage,)) 2930 sources = self.fetchall2set(self.cursor.fetchall()) 2931 if not extended: 2932 return sources 2933 2934 source_data = {} 2935 mirror_str = "mirror://" 2936 for source in sources: 2937 source_data[source] = set() 2938 if source.startswith(mirror_str): 2939 mirrorname = source.split("/")[2] 2940 mirror_url = source.split("/", 3)[3:][0] 2941 source_data[source] |= set([os.path.join(url, mirror_url) for url in self.retrieveMirrorInfo(mirrorname)]) 2942 else: 2943 source_data[source].add(source) 2944 2945 return source_data

2946

2947 - def retrieveAutomergefiles(self, idpackage, get_dict = False):

2948 if not self.doesTableExist('automergefiles'): 2949 self.createAutomergefilesTable() 2950 # like portage does 2951 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 2952 self.cursor.execute('SELECT configfile, md5 FROM automergefiles WHERE idpackage = (?)', (idpackage,)) 2953 data = self.cursor.fetchall() 2954 if get_dict: 2955 data = dict(((x, y,) for x, y in data)) 2956 return data

2957

2958 - def retrieveContent(self, idpackage, extended = False, contentType = None, 2959 formatted = False, insert_formatted = False, order_by = ''):

2960 2961 extstring = '' 2962 if extended: 2963 extstring = ",type" 2964 extstring_idpackage = '' 2965 if insert_formatted: 2966 extstring_idpackage = 'idpackage,' 2967 2968 searchkeywords = [idpackage] 2969 contentstring = '' 2970 if contentType: 2971 searchkeywords.append(contentType) 2972 contentstring = ' and type = (?)' 2973 2974 order_by_string = '' 2975 if order_by: 2976 order_by_string = ' order by %s' % (order_by,) 2977 2978 did_try = False 2979 while 1: 2980 try: 2981 2982 self.cursor.execute('SELECT %s file%s FROM content WHERE idpackage = (?) %s%s' % ( 2983 extstring_idpackage, extstring, contentstring, order_by_string,), 2984 searchkeywords) 2985 2986 if extended and insert_formatted: 2987 fl = self.cursor.fetchall() 2988 elif extended and formatted: 2989 fl = {} 2990 items = self.cursor.fetchone() 2991 while items: 2992 fl[items[0]] = items[1] 2993 items = self.cursor.fetchone() 2994 elif extended: 2995 fl = self.cursor.fetchall() 2996 else: 2997 if order_by: 2998 fl = self.fetchall2list(self.cursor.fetchall()) 2999 else: 3000 fl = self.fetchall2set(self.cursor.fetchall()) 3001 break 3002 except (self.dbapi2.OperationalError,): 3003 if did_try: 3004 raise 3005 did_try = True 3006 # XXX support for old entropy db entries, which were 3007 # not inserted in utf-8 3008 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3009 continue 3010 return fl

3011

3012 - def retrieveChangelog(self, idpackage):

3013 if not self.doesTableExist('packagechangelogs'): 3014 return None 3015 self.cursor.execute(""" 3016 SELECT packagechangelogs.changelog FROM packagechangelogs,baseinfo,categories 3017 WHERE baseinfo.idpackage = (?) AND 3018 baseinfo.idcategory = categories.idcategory AND 3019 packagechangelogs.name = baseinfo.name AND 3020 packagechangelogs.category = categories.category""", (idpackage,)) 3021 changelog = self.cursor.fetchone() 3022 if changelog: 3023 changelog = changelog[0] 3024 try: 3025 return unicode(changelog, 'raw_unicode_escape') 3026 except UnicodeDecodeError: 3027 return unicode(changelog, 'utf-8')

3028

3029 - def retrieveChangelogByKey(self, category, name):

3030 if not self.doesTableExist('packagechangelogs'): 3031 return None 3032 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3033 self.cursor.execute('SELECT changelog FROM packagechangelogs WHERE category = (?) AND name = (?)', (category, name,)) 3034 changelog = self.cursor.fetchone() 3035 if changelog: return unicode(changelog[0], 'raw_unicode_escape')

3036

3037 - def retrieveSlot(self, idpackage):

3038 self.cursor.execute('SELECT slot FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3039 slot = self.cursor.fetchone() 3040 if slot: return slot[0]

3041

3042 - def retrieveVersionTag(self, idpackage):

3043 self.cursor.execute('SELECT versiontag FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3044 vtag = self.cursor.fetchone() 3045 if vtag: return vtag[0]

3046

3047 - def retrieveMirrorInfo(self, mirrorname):

3048 self.cursor.execute('SELECT mirrorlink FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) 3049 mirrorlist = self.fetchall2set(self.cursor.fetchall()) 3050 return mirrorlist

3051

3052 - def retrieveCategory(self, idpackage):

3053 self.cursor.execute(""" 3054 SELECT category FROM baseinfo,categories 3055 WHERE baseinfo.idpackage = (?) AND 3056 baseinfo.idcategory = categories.idcategory""", (idpackage,)) 3057 cat = self.cursor.fetchone() 3058 if cat: return cat[0]

3059

3060 - def retrieveCategoryDescription(self, category):

3061 data = {} 3062 self.cursor.execute('SELECT description,locale FROM categoriesdescription WHERE category = (?)', (category,)) 3063 description_data = self.cursor.fetchall() 3064 for description, locale in description_data: 3065 data[locale] = description 3066 return data

3067

3068 - def retrieveLicensedata(self, idpackage):

3069 3070 # insert license information 3071 licenses = self.retrieveLicense(idpackage) 3072 if licenses == None: 3073 return {} 3074 licenses = licenses.split() 3075 licdata = {} 3076 for licname in licenses: 3077 licname = licname.strip() 3078 if not self.entropyTools.is_valid_string(licname): 3079 continue 3080 3081 self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,)) 3082 lictext = self.cursor.fetchone() 3083 if lictext != None: 3084 lictext = lictext[0] 3085 try: 3086 licdata[licname] = unicode(lictext, 'raw_unicode_escape') 3087 except UnicodeDecodeError: 3088 licdata[licname] = unicode(lictext, 'utf-8') 3089 3090 return licdata

3091

3092 - def retrieveLicensedataKeys(self, idpackage):

3093 3094 licenses = self.retrieveLicense(idpackage) 3095 if licenses == None: 3096 return set() 3097 licenses = licenses.split() 3098 licdata = set() 3099 for licname in licenses: 3100 licname = licname.strip() 3101 if not self.entropyTools.is_valid_string(licname): 3102 continue 3103 self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,)) 3104 licidentifier = self.cursor.fetchone() 3105 if licidentifier: 3106 licdata.add(licidentifier[0]) 3107 3108 return licdata

3109

3110 - def retrieveLicenseText(self, license_name):

3111 3112 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3113 3114 self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (license_name,)) 3115 text = self.cursor.fetchone() 3116 if not text: 3117 return None 3118 return str(text[0])

3119

3120 - def retrieveLicense(self, idpackage):

3121 self.cursor.execute(""" 3122 SELECT license FROM baseinfo,licenses 3123 WHERE baseinfo.idpackage = (?) AND 3124 baseinfo.idlicense = licenses.idlicense""", (idpackage,)) 3125 licname = self.cursor.fetchone() 3126 if licname: return licname[0]

3127

3128 - def retrieveCompileFlags(self, idpackage):

3129 self.cursor.execute(""" 3130 SELECT chost,cflags,cxxflags FROM flags,extrainfo 3131 WHERE extrainfo.idpackage = (?) AND 3132 extrainfo.idflags = flags.idflags""", (idpackage,)) 3133 flags = self.cursor.fetchone() 3134 if not flags: 3135 flags = ("N/A", "N/A", "N/A",) 3136 return flags

3137

3138 - def retrieveDepends(self, idpackage, atoms = False, key_slot = False, 3139 exclude_deptypes = None):

3140 3141 # WARNING: never remove this, otherwise equo.db 3142 # (client database) dependstable will be always broken (trust me) 3143 # sanity check on the table 3144 if not self.isDependsTableSane(): # is empty, need generation 3145 self.regenerateDependsTable(output = False) 3146 3147 excluded_deptypes_query = "" 3148 if exclude_deptypes != None: 3149 for dep_type in exclude_deptypes: 3150 excluded_deptypes_query += " AND dependencies.type != %s" % ( 3151 dep_type,) 3152 3153 if atoms: 3154 self.cursor.execute(""" 3155 SELECT baseinfo.atom FROM dependstable,dependencies,baseinfo 3156 WHERE dependstable.idpackage = (?) AND 3157 dependstable.iddependency = dependencies.iddependency AND 3158 baseinfo.idpackage = dependencies.idpackage %s""" % ( 3159 excluded_deptypes_query,), (idpackage,)) 3160 result = self.fetchall2set(self.cursor.fetchall()) 3161 elif key_slot: 3162 self.cursor.execute(""" 3163 SELECT categories.category || "/" || baseinfo.name,baseinfo.slot 3164 FROM baseinfo,categories,dependstable,dependencies 3165 WHERE dependstable.idpackage = (?) AND 3166 dependstable.iddependency = dependencies.iddependency AND 3167 baseinfo.idpackage = dependencies.idpackage AND 3168 categories.idcategory = baseinfo.idcategory %s""" % ( 3169 excluded_deptypes_query,), (idpackage,)) 3170 result = self.cursor.fetchall() 3171 else: 3172 self.cursor.execute(""" 3173 SELECT dependencies.idpackage FROM dependstable,dependencies 3174 WHERE dependstable.idpackage = (?) AND 3175 dependstable.iddependency = dependencies.iddependency %s""" % ( 3176 excluded_deptypes_query,), (idpackage,)) 3177 result = self.fetchall2set(self.cursor.fetchall()) 3178 3179 return result

3180

3181 - def retrieveUnusedIdpackages(self):

3182 3183 # WARNING: never remove this, otherwise equo.db (client database) 3184 # dependstable will be always broken (trust me) 3185 # sanity check on the table 3186 if not self.isDependsTableSane(): # is empty, need generation 3187 self.regenerateDependsTable(output = False) 3188 3189 self.cursor.execute(""" 3190 SELECT idpackage FROM baseinfo 3191 WHERE idpackage NOT IN (SELECT idpackage FROM dependstable) ORDER BY atom 3192 """) 3193 return self.fetchall2list(self.cursor.fetchall())

3194

3195 - def isPackageAvailable(self, pkgatom):

3196 # You must provide the full atom to this function 3197 # WARNING: this function does not support branches 3198 pkgatom = self.entropyTools.remove_package_operators(pkgatom) 3199 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)', (pkgatom,)) 3200 result = self.cursor.fetchone() 3201 if result: return result[0] 3202 return -1

3203

3204 - def isIDPackageAvailable(self, idpackage):

3205 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = (?)', (idpackage,)) 3206 result = self.cursor.fetchone() 3207 if not result: 3208 return False 3209 return True

3210

3211 - def areIDPackagesAvailable(self, idpackages):

3212 sql = 'SELECT count(idpackage) FROM baseinfo WHERE idpackage IN (%s)' % (','.join([str(x) for x in set(idpackages)]),) 3213 self.cursor.execute(sql) 3214 count = self.cursor.fetchone()[0] 3215 if count != len(idpackages): 3216 return False 3217 return True

3218

3219 - def isCategoryAvailable(self, category):

3220 self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,)) 3221 result = self.cursor.fetchone() 3222 if result: return result[0] 3223 return -1

3224

3225 - def isProtectAvailable(self, protect):

3226 self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,)) 3227 result = self.cursor.fetchone() 3228 if result: return result[0] 3229 return -1

3230

3231 - def isFileAvailable(self, myfile, get_id = False):

3232 self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (myfile,)) 3233 result = self.cursor.fetchall() 3234 if get_id: 3235 return self.fetchall2set(result) 3236 elif result: 3237 return True 3238 return False

3239

3240 - def resolveNeeded(self, needed, elfclass = -1, extended = False):

3241 3242 args = [needed] 3243 elfclass_txt = '' 3244 3245 if extended: 3246 if elfclass != -1: 3247 elfclass_txt = ' AND neededlibraryidpackages.elfclass = (?)' 3248 args.append(elfclass) 3249 self.cursor.execute(""" 3250 SELECT neededlibraryidpackages.idpackage, 3251 neededlibrarypaths.path 3252 FROM neededlibraryidpackages, neededlibrarypaths 3253 WHERE neededlibraryidpackages.library = (?) AND 3254 neededlibraryidpackages.library = neededlibrarypaths.library AND 3255 neededlibraryidpackages.elfclass = neededlibrarypaths.elfclass 3256 """ + elfclass_txt, args) 3257 return self.cursor.fetchall() 3258 else: 3259 if elfclass != -1: 3260 elfclass_txt = ' AND elfclass = (?)' 3261 args.append(elfclass) 3262 self.cursor.execute(""" 3263 SELECT idpackage FROM neededlibraryidpackages 3264 WHERE library = (?) 3265 """ + elfclass_txt, args) 3266 return self.fetchall2set(self.cursor.fetchall())

3267

3268 - def isSourceAvailable(self, source):

3269 self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = (?)', (source,)) 3270 result = self.cursor.fetchone() 3271 if result: return result[0] 3272 return -1

3273

3274 - def isDependencyAvailable(self, dependency):

3275 self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = (?)', (dependency,)) 3276 result = self.cursor.fetchone() 3277 if result: return result[0] 3278 return -1

3279

3280 - def isKeywordAvailable(self, keyword):

3281 self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = (?)', (keyword,)) 3282 result = self.cursor.fetchone() 3283 if result: return result[0] 3284 return -1

3285

3286 - def isUseflagAvailable(self, useflag):

3287 self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = (?)', (useflag,)) 3288 result = self.cursor.fetchone() 3289 if result: return result[0] 3290 return -1

3291

3292 - def isEclassAvailable(self, eclass):

3293 self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,)) 3294 result = self.cursor.fetchone() 3295 if result: return result[0] 3296 return -1

3297

3298 - def isNeededAvailable(self, needed):

3299 self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,)) 3300 result = self.cursor.fetchone() 3301 if result: return result[0] 3302 return -1

3303

3304 - def isCounterAvailable(self, counter, branch = None, branch_operator = "="):

3305 params = [counter] 3306 branch_string = '' 3307 if branch: 3308 branch_string = ' and branch '+branch_operator+' (?)' 3309 params = [counter, branch] 3310 3311 self.cursor.execute('SELECT counter FROM counters WHERE counter = (?)'+branch_string, params) 3312 result = self.cursor.fetchone() 3313 if result: return True 3314 return False

3315

3316 - def isCounterTrashed(self, counter):

3317 self.cursor.execute('SELECT counter FROM trashedcounters WHERE counter = (?)', (counter,)) 3318 result = self.cursor.fetchone() 3319 if result: return True 3320 return False

3321

3322 - def isLicensedataKeyAvailable(self, license_name):

3323 self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,)) 3324 result = self.cursor.fetchone() 3325 if not result: 3326 return False 3327 return True

3328

3329 - def isLicenseAccepted(self, license_name):

3330 self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,)) 3331 result = self.cursor.fetchone() 3332 if not result: 3333 return False 3334 return True

3335

3336 - def acceptLicense(self, license_name):

3337 if self.readOnly or (not self.entropyTools.is_user_in_entropy_group()): 3338 return 3339 if self.isLicenseAccepted(license_name): 3340 return 3341 with self.__write_mutex: 3342 self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,)) 3343 self.commitChanges()

3344

3345 - def isLicenseAvailable(self, pkglicense):

3346 if not self.entropyTools.is_valid_string(pkglicense): 3347 pkglicense = ' ' 3348 self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,)) 3349 result = self.cursor.fetchone() 3350 if result: return result[0] 3351 return -1

3352

3353 - def isSystemPackage(self, idpackage):

3354 self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) 3355 result = self.cursor.fetchone() 3356 if result: 3357 return True 3358 return False

3359

3360 - def isInjected(self, idpackage):

3361 self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,)) 3362 result = self.cursor.fetchone() 3363 if result: 3364 return True 3365 return False

3366

3367 - def areCompileFlagsAvailable(self, chost, cflags, cxxflags):

3368 3369 self.cursor.execute('SELECT idflags FROM flags WHERE chost = (?) AND cflags = (?) AND cxxflags = (?)', 3370 (chost, cflags, cxxflags,) 3371 ) 3372 result = self.cursor.fetchone() 3373 if result: return result[0] 3374 return -1

3375

3376 - def searchBelongs(self, file, like = False, branch = None, branch_operator = "="):

3377 3378 branchstring = '' 3379 searchkeywords = [file] 3380 if branch: 3381 searchkeywords.append(branch) 3382 branchstring = ' and baseinfo.branch '+branch_operator+' (?)' 3383 3384 if like: 3385 self.cursor.execute(""" 3386 SELECT content.idpackage FROM content,baseinfo 3387 WHERE file LIKE (?) AND 3388 content.idpackage = baseinfo.idpackage %s""" % (branchstring,), searchkeywords) 3389 else: 3390 self.cursor.execute("""SELECT content.idpackage FROM content,baseinfo 3391 WHERE file = (?) AND 3392 content.idpackage = baseinfo.idpackage %s""" % (branchstring,), searchkeywords) 3393 3394 return self.fetchall2set(self.cursor.fetchall())

3395 3396 ''' search packages that uses the eclass provided '''

3397 - def searchEclassedPackages(self, eclass, atoms = False): # atoms = return atoms directly

3398 if atoms: 3399 self.cursor.execute(""" 3400 SELECT baseinfo.atom,eclasses.idpackage FROM baseinfo,eclasses,eclassesreference 3401 WHERE eclassesreference.classname = (?) AND 3402 eclassesreference.idclass = eclasses.idclass AND 3403 eclasses.idpackage = baseinfo.idpackage""", (eclass,)) 3404 return self.cursor.fetchall() 3405 else: 3406 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (eclass,)) 3407 return self.fetchall2set(self.cursor.fetchall()) 3408 3409 ''' search packages whose versiontag matches the one provided '''

3410 - def searchTaggedPackages(self, tag, atoms = False): # atoms = return atoms directly

3411 if atoms: 3412 self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) 3413 return self.cursor.fetchall() 3414 else: 3415 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (tag,)) 3416 return self.fetchall2set(self.cursor.fetchall()) 3417

3418 - def searchLicenses(self, mylicense, caseSensitive = False, atoms = False):

3419 3420 if not self.entropyTools.is_valid_string(mylicense): 3421 return [] 3422 3423 request = "baseinfo.idpackage" 3424 if atoms: 3425 request = "baseinfo.atom,baseinfo.idpackage" 3426 3427 if caseSensitive: 3428 self.cursor.execute(""" 3429 SELECT %s FROM baseinfo,licenses 3430 WHERE licenses.license LIKE (?) AND 3431 licenses.idlicense = baseinfo.idlicense""" % (request,), ("%"+mylicense+"%",)) 3432 else: 3433 self.cursor.execute(""" 3434 SELECT %s FROM baseinfo,licenses 3435 WHERE LOWER(licenses.license) LIKE (?) AND 3436 licenses.idlicense = baseinfo.idlicense""" % (request,), ("%"+mylicense+"%".lower(),)) 3437 if atoms: 3438 return self.cursor.fetchall() 3439 return self.fetchall2set(self.cursor.fetchall())

3440 3441 ''' search packages whose slot matches the one provided '''

3442 - def searchSlottedPackages(self, slot, atoms = False): # atoms = return atoms directly

3443 if atoms: 3444 self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE slot = (?)', (slot,)) 3445 return self.cursor.fetchall() 3446 else: 3447 self.cursor.execute('SELECT idpackage FROM baseinfo WHERE slot = (?)', (slot,)) 3448 return self.fetchall2set(self.cursor.fetchall()) 3449

3450 - def searchKeySlot(self, key, slot, branch = None):

3451 3452 branchstring = '' 3453 cat, name = key.split("/") 3454 params = [cat, name, slot] 3455 if branch: 3456 params.append(branch) 3457 branchstring = 'and baseinfo.branch = (?)' 3458 3459 self.cursor.execute(""" 3460 SELECT idpackage FROM baseinfo,categories 3461 WHERE baseinfo.idcategory = categories.idcategory AND 3462 categories.category = (?) AND 3463 baseinfo.name = (?) AND 3464 baseinfo.slot = (?) %s""" % (branchstring,), params) 3465 return self.cursor.fetchall()

3466 3467 ''' search packages that need the specified library (in neededreference table) specified by keyword '''

3468 - def searchNeeded(self, keyword, like = False):

3469 if like: 3470 self.cursor.execute(""" 3471 SELECT needed.idpackage FROM needed,neededreference 3472 WHERE library LIKE (?) AND 3473 needed.idneeded = neededreference.idneeded""", (keyword,)) 3474 else: 3475 self.cursor.execute(""" 3476 SELECT needed.idpackage FROM needed,neededreference 3477 WHERE library = (?) AND 3478 needed.idneeded = neededreference.idneeded""", (keyword,)) 3479 return self.fetchall2set(self.cursor.fetchall())

3480 3481 ''' search dependency string inside dependenciesreference table and retrieve iddependency '''

3482 - def searchDependency(self, dep, like = False, multi = False, strings = False):

3483 sign = "=" 3484 if like: 3485 sign = "LIKE" 3486 dep = "%"+dep+"%" 3487 item = 'iddependency' 3488 if strings: 3489 item = 'dependency' 3490 self.cursor.execute('SELECT %s FROM dependenciesreference WHERE dependency %s (?)' % (item, sign,), (dep,)) 3491 if multi: 3492 return self.fetchall2set(self.cursor.fetchall()) 3493 else: 3494 iddep = self.cursor.fetchone() 3495 if iddep: 3496 iddep = iddep[0] 3497 else: 3498 iddep = -1 3499 return iddep

3500 3501 ''' search iddependency inside dependencies table and retrieve idpackages '''

3502 - def searchIdpackageFromIddependency(self, iddep):

3503 self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,)) 3504 return self.fetchall2set(self.cursor.fetchall())

3505

3506 - def searchSets(self, keyword):

3507 self.cursor.execute('SELECT DISTINCT(setname) FROM packagesets WHERE setname LIKE (?)', ("%"+keyword+"%",)) 3508 return self.fetchall2set(self.cursor.fetchall())

3509

3510 - def searchSimilarPackages(self, mystring, atom = False):

3511 s_item = 'name' 3512 if atom: s_item = 'atom' 3513 self.cursor.execute(""" 3514 SELECT idpackage FROM baseinfo 3515 WHERE soundex(%s) = soundex((?)) ORDER BY %s""" % (s_item, s_item,), (mystring,)) 3516 return self.fetchall2list(self.cursor.fetchall())

3517

3518 - def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None, order_by = 'atom', just_id = False):

3519 3520 searchkeywords = ["%"+keyword+"%"] 3521 slotstring = '' 3522 if slot: 3523 searchkeywords.append(slot) 3524 slotstring = ' and slot = (?)' 3525 tagstring = '' 3526 if tag: 3527 searchkeywords.append(tag) 3528 tagstring = ' and versiontag = (?)' 3529 branchstring = '' 3530 if branch: 3531 searchkeywords.append(branch) 3532 branchstring = ' and branch = (?)' 3533 order_by_string = '' 3534 if order_by in ("atom", "idpackage", "branch",): 3535 order_by_string = ' order by %s' % (order_by,) 3536 3537 search_elements = 'atom,idpackage,branch' 3538 if just_id: search_elements = 'idpackage' 3539 3540 if sensitive: 3541 self.cursor.execute(""" 3542 SELECT %s FROM baseinfo WHERE atom LIKE (?) %s %s %s %s""" % ( 3543 search_elements,slotstring,tagstring,branchstring,order_by_string,), 3544 searchkeywords 3545 ) 3546 else: 3547 self.cursor.execute(""" 3548 SELECT %s FROM baseinfo WHERE 3549 LOWER(atom) LIKE (?) %s %s %s %s""" % ( 3550 search_elements,slotstring,tagstring,branchstring,order_by_string,), 3551 searchkeywords 3552 ) 3553 if just_id: 3554 return self.fetchall2list(self.cursor.fetchall()) 3555 return self.cursor.fetchall()

3556

3557 - def searchProvide(self, keyword, slot = None, tag = None, branch = None, justid = False):

3558 3559 slotstring = '' 3560 searchkeywords = [keyword] 3561 if slot: 3562 searchkeywords.append(slot) 3563 slotstring = ' and baseinfo.slot = (?)' 3564 tagstring = '' 3565 if tag: 3566 searchkeywords.append(tag) 3567 tagstring = ' and baseinfo.versiontag = (?)' 3568 branchstring = '' 3569 if branch: 3570 searchkeywords.append(branch) 3571 branchstring = ' and baseinfo.branch = (?)' 3572 atomstring = '' 3573 if not justid: 3574 atomstring = 'baseinfo.atom,' 3575 3576 self.cursor.execute(""" 3577 SELECT %s baseinfo.idpackage FROM baseinfo,provide 3578 WHERE provide.atom = (?) AND 3579 provide.idpackage = baseinfo.idpackage %s %s %s""" % ( 3580 atomstring,slotstring,tagstring,branchstring,), 3581 searchkeywords 3582 ) 3583 3584 if justid: 3585 results = self.fetchall2list(self.cursor.fetchall()) 3586 else: 3587 results = self.cursor.fetchall() 3588 return results

3589

3590 - def searchPackagesByDescription(self, keyword):

3591 self.cursor.execute(""" 3592 SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo 3593 WHERE LOWER(extrainfo.description) LIKE (?) AND 3594 baseinfo.idpackage = extrainfo.idpackage""", ("%"+keyword.lower()+"%",)) 3595 return self.cursor.fetchall()

3596

3597 - def searchPackagesByName(self, keyword, sensitive = False, branch = None, justid = False):

3598 3599 if sensitive: 3600 searchkeywords = [keyword] 3601 else: 3602 searchkeywords = [keyword.lower()] 3603 branchstring = '' 3604 atomstring = '' 3605 if not justid: 3606 atomstring = 'atom,' 3607 if branch: 3608 searchkeywords.append(branch) 3609 branchstring = ' and branch = (?)' 3610 3611 if sensitive: 3612 self.cursor.execute(""" 3613 SELECT %s idpackage FROM baseinfo 3614 WHERE name = (?) %s""" % (atomstring, branchstring,), searchkeywords) 3615 else: 3616 self.cursor.execute(""" 3617 SELECT %s idpackage FROM baseinfo 3618 WHERE LOWER(name) = (?) %s""" % (atomstring, branchstring,), searchkeywords) 3619 3620 if justid: 3621 results = self.fetchall2list(self.cursor.fetchall()) 3622 else: 3623 results = self.cursor.fetchall() 3624 return results

3625 3626

3627 - def searchPackagesByCategory(self, keyword, like = False, branch = None):

3628 3629 searchkeywords = [keyword] 3630 branchstring = '' 3631 if branch: 3632 searchkeywords.append(branch) 3633 branchstring = 'and branch = (?)' 3634 3635 if like: 3636 self.cursor.execute(""" 3637 SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories 3638 WHERE categories.category LIKE (?) AND 3639 baseinfo.idcategory = categories.idcategory %s""" % (branchstring,), searchkeywords) 3640 else: 3641 self.cursor.execute(""" 3642 SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories 3643 WHERE categories.category = (?) AND 3644 baseinfo.idcategory = categories.idcategory %s""" % (branchstring,), searchkeywords) 3645 3646 return self.cursor.fetchall()

3647

3648 - def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None, justid = False):

3649 3650 myname = name 3651 mycat = category 3652 if not sensitive: 3653 myname = name.lower() 3654 mycat = category.lower() 3655 3656 searchkeywords = [myname, mycat] 3657 branchstring = '' 3658 if branch: 3659 searchkeywords.append(branch) 3660 branchstring = ' and branch = (?)' 3661 atomstring = '' 3662 if not justid: 3663 atomstring = 'atom,' 3664 3665 if sensitive: 3666 self.cursor.execute(""" 3667 SELECT %s idpackage FROM baseinfo 3668 WHERE name = (?) AND 3669 idcategory IN ( 3670 SELECT idcategory FROM categories 3671 WHERE category = (?) 3672 ) %s""" % (atomstring, branchstring,), searchkeywords) 3673 else: 3674 self.cursor.execute(""" 3675 SELECT %s idpackage FROM baseinfo 3676 WHERE LOWER(name) = (?) AND 3677 idcategory IN ( 3678 SELECT idcategory FROM categories 3679 WHERE LOWER(category) = (?) 3680 ) %s""" % (atomstring, branchstring,), searchkeywords) 3681 3682 if justid: 3683 results = self.fetchall2list(self.cursor.fetchall()) 3684 else: 3685 results = self.cursor.fetchall() 3686 return results

3687

3688 - def isPackageScopeAvailable(self, atom, slot, revision):

3689 searchdata = (atom, slot, revision,) 3690 self.cursor.execute('SELECT idpackage FROM baseinfo where atom = (?) and slot = (?) and revision = (?)', searchdata) 3691 rslt = self.cursor.fetchone() 3692 idreason = 0 3693 idpackage = -1 3694 if rslt: 3695 # check if it's masked 3696 idpackage, idreason = self.idpackageValidator(rslt[0]) 3697 return idpackage, idreason

3698

3699 - def listAllPackages(self, get_scope = False, order_by = None, branch = None, branch_operator = "="):

3700 3701 branchstring = '' 3702 searchkeywords = [] 3703 if branch: 3704 searchkeywords = [branch] 3705 branchstring = ' where branch %s (?)' % (branch_operator,) 3706 3707 order_txt = '' 3708 if order_by: 3709 order_txt = ' order by %s' % (order_by,) 3710 if get_scope: 3711 self.cursor.execute('SELECT idpackage,atom,slot,revision FROM baseinfo'+order_txt+branchstring, searchkeywords) 3712 else: 3713 self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo'+order_txt+branchstring, searchkeywords) 3714 return self.cursor.fetchall()

3715

3716 - def listAllInjectedPackages(self, justFiles = False):

3717 self.cursor.execute('SELECT idpackage FROM injected') 3718 injecteds = self.fetchall2set(self.cursor.fetchall()) 3719 results = set() 3720 # get download 3721 for injected in injecteds: 3722 download = self.retrieveDownloadURL(injected) 3723 if justFiles: 3724 results.add(download) 3725 else: 3726 results.add((download, injected)) 3727 return results

3728

3729 - def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="):

3730 3731 branchstring = '' 3732 if branch: 3733 branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"' 3734 if onlycounters: 3735 self.cursor.execute('SELECT counter FROM counters'+branchstring) 3736 return self.fetchall2set(self.cursor.fetchall()) 3737 else: 3738 self.cursor.execute('SELECT counter,idpackage FROM counters'+branchstring) 3739 return self.cursor.fetchall()

3740

3741 - def listAllIdpackages(self, branch = None, branch_operator = "=", order_by = None):

3742 3743 branchstring = '' 3744 orderbystring = '' 3745 searchkeywords = [] 3746 if branch: 3747 searchkeywords.append(branch) 3748 branchstring = ' where branch %s (?)' % (str(branch_operator),) 3749 if order_by: 3750 orderbystring = ' order by '+order_by 3751 3752 self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring+orderbystring, searchkeywords) 3753 3754 try: 3755 if order_by: 3756 results = self.fetchall2list(self.cursor.fetchall()) 3757 else: 3758 results = self.fetchall2set(self.cursor.fetchall()) 3759 return results 3760 except self.dbapi2.OperationalError: 3761 if order_by: 3762 return [] 3763 return set()

3764

3765 - def listAllDependencies(self, only_deps = False):

3766 if only_deps: 3767 self.cursor.execute('SELECT dependency FROM dependenciesreference') 3768 return self.fetchall2set(self.cursor.fetchall()) 3769 else: 3770 self.cursor.execute('SELECT * FROM dependenciesreference') 3771 return self.cursor.fetchall()

3772

3773 - def listAllBranches(self):

3774 3775 cache = self.live_cache.get('listAllBranches') 3776 if cache != None: 3777 return cache 3778 3779 self.cursor.execute('SELECT distinct branch FROM baseinfo') 3780 results = self.fetchall2set(self.cursor.fetchall()) 3781 3782 self.live_cache['listAllBranches'] = results.copy() 3783 return results

3784

3785 - def listIdPackagesInIdcategory(self, idcategory, order_by = 'atom'):

3786 order_by_string = '' 3787 if order_by in ("atom", "name", "version",): 3788 order_by_string = ' ORDER BY %s' % (order_by,) 3789 self.cursor.execute('SELECT idpackage FROM baseinfo where idcategory = (?)'+order_by_string, (idcategory,)) 3790 return self.fetchall2set(self.cursor.fetchall())

3791

3792 - def listIdpackageDependencies(self, idpackage):

3793 self.cursor.execute(""" 3794 SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies 3795 WHERE dependencies.idpackage = (?) AND 3796 dependenciesreference.iddependency = dependencies.iddependency""", (idpackage,)) 3797 return set(self.cursor.fetchall())

3798

3799 - def listAllDownloads(self, do_sort = True, full_path = False):

3800 3801 order_string = '' 3802 if do_sort: 3803 order_string = 'ORDER BY extrainfo.download' 3804 self.cursor.execute(""" 3805 SELECT extrainfo.download FROM baseinfo,extrainfo 3806 WHERE baseinfo.idpackage = extrainfo.idpackage %s""" % (order_string,)) 3807 3808 if do_sort: 3809 results = self.fetchall2list(self.cursor.fetchall()) 3810 else: 3811 results = self.fetchall2set(self.cursor.fetchall()) 3812 3813 if not full_path: 3814 results = [os.path.basename(x) for x in results] 3815 3816 return results

3817

3818 - def listAllFiles(self, clean = False, count = False):

3819 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 3820 if count: 3821 self.cursor.execute('SELECT count(file) FROM content') 3822 else: 3823 self.cursor.execute('SELECT file FROM content') 3824 if count: 3825 return self.cursor.fetchone()[0] 3826 else: 3827 if clean: 3828 return self.fetchall2set(self.cursor.fetchall()) 3829 else: 3830 return self.fetchall2list(self.cursor.fetchall())

3831

3832 - def listAllCategories(self, order_by = ''):

3833 order_by_string = '' 3834 if order_by: order_by_string = ' order by %s' % (order_by,) 3835 self.cursor.execute('SELECT idcategory,category FROM categories %s' % ( 3836 order_by_string,)) 3837 return self.cursor.fetchall()

3838

3839 - def listConfigProtectDirectories(self, mask = False):

3840 mask_t = '' 3841 if mask: mask_t = 'mask' 3842 self.cursor.execute(""" 3843 SELECT DISTINCT(protect) FROM configprotectreference 3844 WHERE idprotect >= 1 AND 3845 idprotect <= (SELECT max(idprotect) FROM configprotect%s) 3846 ORDER BY protect""" % (mask_t,)) 3847 results = self.fetchall2set(self.cursor.fetchall()) 3848 dirs = set() 3849 for mystr in results: 3850 dirs |= set(map(unicode, mystr.split())) 3851 return sorted(list(dirs))

3852

3853 - def switchBranch(self, idpackage, tobranch):

3854 with self.__write_mutex: 3855 self.cursor.execute(""" 3856 UPDATE baseinfo SET branch = (?) 3857 WHERE idpackage = (?)""", (tobranch, idpackage,)) 3858 self.commitChanges() 3859 self.clearCache() 3860

3861 - def databaseStructureUpdates(self):

3862 3863 old_readonly = self.readOnly 3864 self.readOnly = False 3865 3866 if not self.doesTableExist("licenses_accepted"): 3867 self.createLicensesAcceptedTable() 3868 3869 if not self.doesTableExist("installedtable"): 3870 self.createInstalledTable() 3871 3872 if not self.doesColumnInTableExist("installedtable", "source"): 3873 self.createInstalledTableSource() 3874 3875 if not self.doesTableExist('packagesets'): 3876 self.createPackagesetsTable() 3877 3878 if not self.doesTableExist('packagechangelogs'): 3879 self.createPackagechangelogsTable() 3880 3881 if not self.doesTableExist('automergefiles'): 3882 self.createAutomergefilesTable() 3883 3884 if not self.doesTableExist('packagesignatures'): 3885 self.createPackagesignaturesTable() 3886 3887 if not self.doesTableExist('packagespmphases'): 3888 self.createPackagespmphases() 3889 3890 if not self.doesTableExist('neededlibrarypaths'): 3891 self.createNeededlibrarypathsTable() 3892 if not self.doesColumnInTableExist("neededlibrarypaths", "elfclass"): 3893 self.createNeededlibrarypathsTable() 3894 3895 if not self.doesTableExist('neededlibraryidpackages'): 3896 self.createNeededlibraryidpackagesTable() 3897 elif not self.doesColumnInTableExist("neededlibraryidpackages", "elfclass"): 3898 self.createNeededlibraryidpackagesTable() 3899 3900 if not self.doesTableExist('dependstable'): 3901 self.createDependsTable() 3902 3903 self.readOnly = old_readonly 3904 self.connection.commit()

3905

3906 - def validateDatabase(self):

3907 self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table", "baseinfo")) 3908 rslt = self.cursor.fetchone() 3909 if rslt == None: 3910 mytxt = _("baseinfo table not found. Either does not exist or corrupted.") 3911 raise SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,)) 3912 self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table", "extrainfo")) 3913 rslt = self.cursor.fetchone() 3914 if rslt == None: 3915 mytxt = _("extrainfo table not found. Either does not exist or corrupted.") 3916 raise SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,))

3917

3918 - def getIdpackagesDifferences(self, foreign_idpackages):

3919 myids = self.listAllIdpackages() 3920 if isinstance(foreign_idpackages, (list, tuple,)): 3921 outids = set(foreign_idpackages) 3922 else: 3923 outids = foreign_idpackages 3924 added_ids = outids - myids 3925 removed_ids = myids - outids 3926 return added_ids, removed_ids

3927

3928 - def uniformBranch(self, branch):

3929 with self.__write_mutex: 3930 self.cursor.execute('UPDATE baseinfo SET branch = (?)', (branch,)) 3931 self.commitChanges() 3932 self.clearCache()

3933

3934 - def alignDatabases(self, dbconn, force = False, output_header = " ", align_limit = 300):

3935 3936 added_ids, removed_ids = self.getIdpackagesDifferences(dbconn.listAllIdpackages()) 3937 3938 if not force: 3939 if len(added_ids) > align_limit: # too much hassle 3940 return 0 3941 if len(removed_ids) > align_limit: # too much hassle 3942 return 0 3943 3944 if not added_ids and not removed_ids: 3945 return -1 3946 3947 mytxt = red("%s, %s ...") % (_("Syncing current database"), _("please wait"),) 3948 self.updateProgress( 3949 mytxt, 3950 importance = 1, 3951 type = "info", 3952 header = output_header, 3953 back = True 3954 ) 3955 maxcount = len(removed_ids) 3956 mycount = 0 3957 for idpackage in removed_ids: 3958 mycount += 1 3959 mytxt = "%s: %s" % (red(_("Removing entry")), blue(str(self.retrieveAtom(idpackage))),) 3960 self.updateProgress( 3961 mytxt, 3962 importance = 0, 3963 type = "info", 3964 header = output_header, 3965 back = True, 3966 count = (mycount, maxcount) 3967 ) 3968 self.removePackage(idpackage, do_cleanup = False, do_commit = False) 3969 3970 maxcount = len(added_ids) 3971 mycount = 0 3972 for idpackage in added_ids: 3973 mycount += 1 3974 mytxt = "%s: %s" % (red(_("Adding entry")), blue(str(dbconn.retrieveAtom(idpackage))),) 3975 self.updateProgress( 3976 mytxt, 3977 importance = 0, 3978 type = "info", 3979 header = output_header, 3980 back = True, 3981 count = (mycount, maxcount) 3982 ) 3983 mydata = dbconn.getPackageData(idpackage, get_content = True, content_insert_formatted = True) 3984 self.addPackage( 3985 mydata, 3986 revision = mydata['revision'], 3987 idpackage = idpackage, 3988 do_remove = False, 3989 do_commit = False, 3990 formatted_content = True 3991 ) 3992 3993 # do some cleanups 3994 self.doCleanups() 3995 # clear caches 3996 self.clearCache() 3997 self.commitChanges() 3998 self.regenerateDependsTable(output = False) 3999 dbconn.clearCache() 4000 4001 # verify both checksums, if they don't match, bomb out 4002 mycheck = self.database_checksum(do_order = True, strict = False) 4003 outcheck = dbconn.database_checksum(do_order = True, strict = False) 4004 if mycheck == outcheck: 4005 return 1 4006 return 0

4007

4008 - def checkDatabaseApi(self):

4009 4010 dbapi = self.getApi() 4011 if int(dbapi) > int(etpConst['etpapi']): 4012 self.updateProgress( 4013 red(_("Repository EAPI > Entropy EAPI. Please update Equo/Entropy as soon as possible !")), 4014 importance = 1, 4015 type = "warning", 4016 header = " * ! * ! * ! * " 4017 )

4018

4019 - def doDatabaseImport(self, dumpfile, dbfile):

4020 import subprocess 4021 sqlite3_exec = "/usr/bin/sqlite3 %s < %s" % (dbfile, dumpfile,) 4022 retcode = subprocess.call(sqlite3_exec, shell = True) 4023 return retcode

4024

4025 - def doDatabaseExport(self, dumpfile, gentle_with_tables = True, 4026 exclude_tables = None):

4027 4028 if not exclude_tables: 4029 exclude_tables = [] 4030 4031 dumpfile.write("BEGIN TRANSACTION;\n") 4032 self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'") 4033 for name, x, sql in self.cursor.fetchall(): 4034 4035 self.updateProgress( 4036 red("%s " % (_("Exporting database table"),) )+"["+blue(str(name))+"]", 4037 importance = 0, 4038 type = "info", 4039 back = True, 4040 header = " " 4041 ) 4042 4043 if name == "sqlite_sequence": 4044 dumpfile.write("DELETE FROM sqlite_sequence;\n") 4045 elif name == "sqlite_stat1": 4046 dumpfile.write("ANALYZE sqlite_master;\n") 4047 elif name.startswith("sqlite_"): 4048 continue 4049 else: 4050 t_cmd = "CREATE TABLE" 4051 if sql.startswith(t_cmd) and gentle_with_tables: 4052 sql = "CREATE TABLE IF NOT EXISTS"+sql[len(t_cmd):] 4053 dumpfile.write("%s;\n" % sql) 4054 4055 if name in exclude_tables: 4056 continue 4057 4058 self.cursor.execute("PRAGMA table_info('%s')" % name) 4059 cols = [str(r[1]) for r in self.cursor.fetchall()] 4060 q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES(" 4061 q += ", ".join(["'||quote(" + x + ")||'" for x in cols]) 4062 q += ")' FROM '%(tbl_name)s'" 4063 self.cursor.execute(q % {'tbl_name': name}) 4064 self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") 4065 for row in self.cursor: 4066 dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape'))) 4067 4068 self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'") 4069 for name, x, sql in self.cursor.fetchall(): 4070 dumpfile.write("%s;\n" % sql) 4071 4072 dumpfile.write("COMMIT;\n") 4073 try: 4074 dumpfile.flush() 4075 except: 4076 pass 4077 self.updateProgress( 4078 red(_("Database Export completed.")), 4079 importance = 0, 4080 type = "info", 4081 header = " " 4082 )

4083 # remember to close the file 4084

4085 - def listAllTables(self):

4086 self.cursor.execute(""" 4087 SELECT name FROM SQLITE_MASTER WHERE type = "table" 4088 """) 4089 return self.fetchall2list(self.cursor.fetchall())

4090

4091 - def doesTableExist(self, table):

4092 self.cursor.execute('select name from SQLITE_MASTER where type = "table" and name = (?)', (table,)) 4093 rslt = self.cursor.fetchone() 4094 if rslt == None: 4095 return False 4096 return True

4097

4098 - def doesColumnInTableExist(self, table, column):

4099 self.cursor.execute('PRAGMA table_info( %s )' % (table,)) 4100 rslt = (x[1] for x in self.cursor.fetchall()) 4101 if column in rslt: 4102 return True 4103 return False

4104

4105 - def database_checksum(self, do_order = False, strict = True, strings = False):

4106 4107 c_tup = ("database_checksum", do_order, strict, strings,) 4108 cache = self.live_cache.get(c_tup) 4109 if cache != None: return cache 4110 4111 idpackage_order = '' 4112 category_order = '' 4113 license_order = '' 4114 flags_order = '' 4115 if do_order: 4116 idpackage_order = 'order by idpackage' 4117 category_order = 'order by category' 4118 license_order = 'order by license' 4119 flags_order = 'order by chost' 4120 4121 def do_update_md5(m, cursor): 4122 mydata = cursor.fetchall() 4123 for record in mydata: 4124 for item in record: 4125 m.update(str(item))

4126 4127 if strings: 4128 import hashlib 4129 m = hashlib.md5() 4130 4131 self.cursor.execute(""" 4132 SELECT idpackage,atom,name,version,versiontag, 4133 revision,branch,slot,etpapi,trigger FROM 4134 baseinfo %s""" % (idpackage_order,)) 4135 if strings: 4136 do_update_md5(m, self.cursor) 4137 else: 4138 a_hash = hash(tuple(self.cursor.fetchall())) 4139 self.cursor.execute(""" 4140 SELECT idpackage,description,homepage, 4141 download,size,digest,datecreation FROM 4142 extrainfo %s""" % (idpackage_order,)) 4143 if strings: 4144 do_update_md5(m, self.cursor) 4145 else: 4146 b_hash = hash(tuple(self.cursor.fetchall())) 4147 self.cursor.execute('select category from categories %s' % (category_order,)) 4148 if strings: 4149 do_update_md5(m, self.cursor) 4150 else: 4151 c_hash = hash(tuple(self.cursor.fetchall())) 4152 d_hash = '0' 4153 e_hash = '0' 4154 if strict: 4155 self.cursor.execute('select * from licenses %s' % (license_order,)) 4156 if strings: 4157 do_update_md5(m, self.cursor) 4158 else: 4159 d_hash = hash(tuple(self.cursor.fetchall())) 4160 self.cursor.execute('select * from flags %s' % (flags_order,)) 4161 if strings: 4162 do_update_md5(m, self.cursor) 4163 else: 4164 e_hash = hash(tuple(self.cursor.fetchall())) 4165 4166 if strings: 4167 result = m.hexdigest() 4168 else: 4169 result = "%s:%s:%s:%s:%s" % (a_hash, b_hash, c_hash, d_hash, e_hash,) 4170 4171 self.live_cache[c_tup] = result[:] 4172 return result 4173 4174 4175 ######################################################## 4176 #### 4177 ## Client Database API / but also used by server part 4178 # 4179

4180 - def updateInstalledTableSource(self, idpackage, source):

4181 with self.__write_mutex: 4182 self.cursor.execute(""" 4183 UPDATE installedtable SET source = (?) WHERE idpackage = (?) 4184 """, (source, idpackage,))

4185

4186 - def addPackageToInstalledTable(self, idpackage, repoid, source = 0):

4187 with self.__write_mutex: 4188 self.cursor.execute('INSERT into installedtable VALUES (?,?,?)', 4189 (idpackage, repoid, source,))

4190 # self.commitChanges() 4191

4192 - def retrievePackageFromInstalledTable(self, idpackage):

4193 with self.__write_mutex: 4194 try: 4195 self.cursor.execute(""" 4196 SELECT repositoryname FROM installedtable 4197 WHERE idpackage = (?)""", (idpackage,)) 4198 return self.cursor.fetchone()[0] 4199 except (self.dbapi2.OperationalError,TypeError,): 4200 return 'Not available'

4201

4202 - def removePackageFromInstalledTable(self, idpackage):

4203 with self.__write_mutex: 4204 self.cursor.execute(""" 4205 DELETE FROM installedtable 4206 WHERE idpackage = (?)""", (idpackage,))

4207

4208 - def removePackageFromDependsTable(self, idpackage):

4209 with self.__write_mutex: 4210 try: 4211 self.cursor.execute('DELETE FROM dependstable WHERE idpackage = (?)', (idpackage,)) 4212 return 0 4213 except (self.dbapi2.OperationalError,): 4214 return 1 # need reinit

4215

4216 - def createDependsTable(self):

4217 with self.__write_mutex: 4218 self.cursor.executescript(""" 4219 CREATE TABLE IF NOT EXISTS dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER ); 4220 INSERT INTO dependstable VALUES (-1,-1); 4221 """) 4222 if self.indexing: 4223 self.cursor.execute('CREATE INDEX IF NOT EXISTS dependsindex_idpackage ON dependstable ( idpackage )') 4224 self.commitChanges()

4225

4226 - def sanitizeDependsTable(self):

4227 with self.__write_mutex: 4228 self.cursor.execute('DELETE FROM dependstable where iddependency = -1') 4229 self.commitChanges()

4230

4231 - def isDependsTableSane(self):

4232 try: 4233 self.cursor.execute('SELECT iddependency FROM dependstable WHERE iddependency = -1') 4234 except (self.dbapi2.OperationalError,): 4235 return False # table does not exist, please regenerate and re-run 4236 status = self.cursor.fetchone() 4237 if status: return False 4238 4239 self.cursor.execute('select count(*) from dependstable') 4240 dependstable_count = self.cursor.fetchone() 4241 if dependstable_count < 2: 4242 return False 4243 return True

4244

4245 - def createXpakTable(self):

4246 with self.__write_mutex: 4247 self.cursor.execute('CREATE TABLE xpakdata ( idpackage INTEGER PRIMARY KEY, data BLOB );') 4248 self.commitChanges()

4249

4250 - def storeXpakMetadata(self, idpackage, blob):

4251 with self.__write_mutex: 4252 self.cursor.execute('INSERT into xpakdata VALUES (?,?)', (int(idpackage), buffer(blob),)) 4253 self.commitChanges()

4254

4255 - def retrieveXpakMetadata(self, idpackage):

4256 try: 4257 self.cursor.execute('SELECT data from xpakdata where idpackage = (?)', (idpackage,)) 4258 mydata = self.cursor.fetchone() 4259 if not mydata: 4260 return "" 4261 return mydata[0] 4262 except: 4263 return ""

4264

4265 - def dropContent(self):

4266 with self.__write_mutex: 4267 self.cursor.execute('DELETE FROM content')

4268

4269 - def dropAllIndexes(self):

4270 self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"') 4271 indexes = self.fetchall2set(self.cursor.fetchall()) 4272 with self.__write_mutex: 4273 for index in indexes: 4274 if not index.startswith("sqlite"): 4275 self.cursor.execute('DROP INDEX IF EXISTS %s' % (index,))

4276

4277 - def listAllIndexes(self, only_entropy = True):

4278 self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"') 4279 indexes = self.fetchall2set(self.cursor.fetchall()) 4280 if not only_entropy: 4281 return indexes 4282 myindexes = set() 4283 for index in indexes: 4284 if index.startswith("sqlite"): 4285 continue 4286 myindexes.add(index) 4287 return myindexes

4288 4289

4290 - def createAllIndexes(self):

4291 self.createContentIndex() 4292 self.createBaseinfoIndex() 4293 self.createKeywordsIndex() 4294 self.createDependenciesIndex() 4295 self.createProvideIndex() 4296 self.createConflictsIndex() 4297 self.createExtrainfoIndex() 4298 self.createNeededIndex() 4299 self.createUseflagsIndex() 4300 self.createLicensedataIndex() 4301 self.createLicensesIndex() 4302 self.createConfigProtectReferenceIndex() 4303 self.createMessagesIndex() 4304 self.createSourcesIndex() 4305 self.createCountersIndex() 4306 self.createEclassesIndex() 4307 self.createCategoriesIndex() 4308 self.createCompileFlagsIndex() 4309 self.createPackagesetsIndex() 4310 self.createAutomergefilesIndex() 4311 self.createNeededlibrarypathsIndex() 4312 self.createNeededlibraryidpackagesIndex()

4313

4314 - def createPackagesetsIndex(self):

4315 if self.indexing: 4316 with self.__write_mutex: 4317 try: 4318 self.cursor.execute('CREATE INDEX IF NOT EXISTS packagesetsindex ON packagesets ( setname )') 4319 self.commitChanges() 4320 except self.dbapi2.OperationalError: 4321 pass

4322

4323 - def createNeededlibraryidpackagesIndex(self):

4324 if self.indexing: 4325 with self.__write_mutex: 4326 try: 4327 self.cursor.executescript(""" 4328 CREATE INDEX IF NOT EXISTS neededlibidpackages_library 4329 ON neededlibraryidpackages ( library ); 4330 CREATE INDEX IF NOT EXISTS neededlibidpackages_idpackage 4331 ON neededlibraryidpackages ( idpackage ); 4332 CREATE INDEX IF NOT EXISTS neededlibidpackages_lib_elf 4333 ON neededlibraryidpackages ( library, elfclass ); 4334 """) 4335 except self.dbapi2.OperationalError: 4336 pass

4337

4338 - def createNeededlibrarypathsIndex(self):

4339 if self.indexing: 4340 with self.__write_mutex: 4341 try: 4342 self.cursor.executescript(""" 4343 CREATE INDEX IF NOT EXISTS neededlibpaths_library 4344 ON neededlibrarypaths ( library ); 4345 CREATE INDEX IF NOT EXISTS neededlibpaths_elf 4346 ON neededlibrarypaths ( elfclass ); 4347 CREATE INDEX IF NOT EXISTS neededlibpaths_path 4348 ON neededlibrarypaths ( path ); 4349 CREATE INDEX IF NOT EXISTS neededlibpaths_library_elf 4350 ON neededlibrarypaths ( library, elfclass ); 4351 """) 4352 except self.dbapi2.OperationalError: 4353 pass

4354

4355 - def createAutomergefilesIndex(self):

4356 if self.indexing: 4357 with self.__write_mutex: 4358 try: 4359 self.cursor.executescript(""" 4360 CREATE INDEX IF NOT EXISTS automergefiles_idpackage 4361 ON automergefiles ( idpackage ); 4362 CREATE INDEX IF NOT EXISTS automergefiles_file_md5 4363 ON automergefiles ( configfile, md5 ); 4364 """) 4365 except self.dbapi2.OperationalError: 4366 pass

4367

4368 - def createNeededIndex(self):

4369 if self.indexing: 4370 with self.__write_mutex: 4371 self.cursor.executescript(""" 4372 CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library ); 4373 CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded ); 4374 CREATE INDEX IF NOT EXISTS neededindex_idpackage ON needed ( idpackage ); 4375 CREATE INDEX IF NOT EXISTS neededindex_elfclass ON needed ( elfclass ); 4376 """)

4377

4378 - def createMessagesIndex(self):

4379 if self.indexing: 4380 with self.__write_mutex: 4381 self.cursor.execute('CREATE INDEX IF NOT EXISTS messagesindex ON messages ( idpackage )')

4382

4383 - def createCompileFlagsIndex(self):

4384 if self.indexing: 4385 with self.__write_mutex: 4386 self.cursor.execute('CREATE INDEX IF NOT EXISTS flagsindex ON flags ( chost,cflags,cxxflags )')

4387

4388 - def createUseflagsIndex(self):

4389 if self.indexing: 4390 with self.__write_mutex: 4391 self.cursor.executescript(""" 4392 CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idpackage ON useflags ( idpackage ); 4393 CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idflag ON useflags ( idflag ); 4394 CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( flagname ); 4395 """)

4396

4397 - def dropContentIndex(self, only_file = False):

4398 with self.__write_mutex: 4399 self.cursor.execute("DROP INDEX IF EXISTS contentindex_file") 4400 if not only_file: 4401 self.cursor.executescript("DROP INDEX IF EXISTS contentindex_couple;")

4402

4403 - def createContentIndex(self):

4404 if self.indexing: 4405 with self.__write_mutex: 4406 if self.doesTableExist("content"): 4407 self.cursor.executescript(""" 4408 CREATE INDEX IF NOT EXISTS contentindex_couple ON content ( idpackage ); 4409 CREATE INDEX IF NOT EXISTS contentindex_file ON content ( file ); 4410 """)

4411

4412 - def createConfigProtectReferenceIndex(self):

4413 if self.indexing: 4414 with self.__write_mutex: 4415 self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )')

4416

4417 - def createBaseinfoIndex(self):

4418 if self.indexing: 4419 with self.__write_mutex: 4420 self.cursor.executescript(""" 4421 CREATE INDEX IF NOT EXISTS baseindex_atom ON baseinfo ( atom ); 4422 CREATE INDEX IF NOT EXISTS baseindex_branch_name ON baseinfo ( name,branch ); 4423 CREATE INDEX IF NOT EXISTS baseindex_branch_name_idcategory ON baseinfo ( name,idcategory,branch ); 4424 CREATE INDEX IF NOT EXISTS baseindex_idcategory ON baseinfo ( idcategory ); 4425 """)

4426

4427 - def createLicensedataIndex(self):

4428 if self.indexing: 4429 with self.__write_mutex: 4430 self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )')

4431

4432 - def createLicensesIndex(self):

4433 if self.indexing: 4434 with self.__write_mutex: 4435 self.cursor.execute('CREATE INDEX IF NOT EXISTS licensesindex ON licenses ( license )')

4436

4437 - def createCategoriesIndex(self):

4438 if self.indexing: 4439 with self.__write_mutex: 4440 self.cursor.execute('CREATE INDEX IF NOT EXISTS categoriesindex_category ON categories ( category )')

4441

4442 - def createKeywordsIndex(self):

4443 if self.indexing: 4444 with self.__write_mutex: 4445 self.cursor.executescript(""" 4446 CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname ); 4447 CREATE INDEX IF NOT EXISTS keywordsindex_idpackage ON keywords ( idpackage ); 4448 CREATE INDEX IF NOT EXISTS keywordsindex_idkeyword ON keywords ( idkeyword ); 4449 """)

4450

4451 - def createDependenciesIndex(self):

4452 if self.indexing: 4453 with self.__write_mutex: 4454 self.cursor.executescript(""" 4455 CREATE INDEX IF NOT EXISTS dependenciesindex_idpackage ON dependencies ( idpackage ); 4456 CREATE INDEX IF NOT EXISTS dependenciesindex_iddependency ON dependencies ( iddependency ); 4457 CREATE INDEX IF NOT EXISTS dependenciesreferenceindex_dependency ON dependenciesreference ( dependency ); 4458 """)

4459

4460 - def createCountersIndex(self):

4461 if self.indexing: 4462 with self.__write_mutex: 4463 self.cursor.executescript(""" 4464 CREATE INDEX IF NOT EXISTS countersindex_idpackage ON counters ( idpackage ); 4465 CREATE INDEX IF NOT EXISTS countersindex_counter ON counters ( counter ); 4466 """)

4467

4468 - def createSourcesIndex(self):

4469 if self.indexing: 4470 with self.__write_mutex: 4471 self.cursor.executescript(""" 4472 CREATE INDEX IF NOT EXISTS sourcesindex_idpackage ON sources ( idpackage ); 4473 CREATE INDEX IF NOT EXISTS sourcesindex_idsource ON sources ( idsource ); 4474 CREATE INDEX IF NOT EXISTS sourcesreferenceindex_source ON sourcesreference ( source ); 4475 """)

4476

4477 - def createProvideIndex(self):

4478 if self.indexing: 4479 with self.__write_mutex: 4480 self.cursor.executescript(""" 4481 CREATE INDEX IF NOT EXISTS provideindex_idpackage ON provide ( idpackage ); 4482 CREATE INDEX IF NOT EXISTS provideindex_atom ON provide ( atom ); 4483 """)

4484

4485 - def createConflictsIndex(self):

4486 if self.indexing: 4487 with self.__write_mutex: 4488 self.cursor.executescript(""" 4489 CREATE INDEX IF NOT EXISTS conflictsindex_idpackage ON conflicts ( idpackage ); 4490 CREATE INDEX IF NOT EXISTS conflictsindex_atom ON conflicts ( conflict ); 4491 """)

4492

4493 - def createExtrainfoIndex(self):

4494 if self.indexing: 4495 with self.__write_mutex: 4496 self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( description )') 4497 self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex_pkgindex ON extrainfo ( idpackage )')

4498

4499 - def createEclassesIndex(self):

4500 if self.indexing: 4501 with self.__write_mutex: 4502 self.cursor.executescript(""" 4503 CREATE INDEX IF NOT EXISTS eclassesindex_idpackage ON eclasses ( idpackage ); 4504 CREATE INDEX IF NOT EXISTS eclassesindex_idclass ON eclasses ( idclass ); 4505 CREATE INDEX IF NOT EXISTS eclassesreferenceindex_classname ON eclassesreference ( classname ); 4506 """)

4507

4508 - def regenerateCountersTable(self, vdb_path, output = False):

4509 4510 # this is necessary now, counters table should be empty 4511 self.cursor.execute("DELETE FROM counters;") 4512 # assign a counter to an idpackage 4513 myids = self.listAllIdpackages() 4514 counter_path = etpConst['spm']['xpak_entries']['counter'] 4515 for myid in myids: 4516 # get atom 4517 myatom = self.retrieveAtom(myid) 4518 mybranch = self.retrieveBranch(myid) 4519 myatom = self.entropyTools.remove_tag(myatom) 4520 myatomcounterpath = "%s%s/%s" % (vdb_path, myatom, counter_path,) 4521 if os.path.isfile(myatomcounterpath): 4522 try: 4523 with open(myatomcounterpath, "r") as f: 4524 counter = int(f.readline().strip()) 4525 except: 4526 if output: 4527 mytxt = "%s: %s: %s" % ( 4528 bold(_("ATTENTION")), 4529 red(_("cannot open Spm counter file for")), 4530 bold(myatom), 4531 ) 4532 self.updateProgress( 4533 mytxt, 4534 importance = 1, 4535 type = "warning" 4536 ) 4537 continue 4538 # insert id+counter 4539 with self.__write_mutex: 4540 try: 4541 self.cursor.execute( 4542 'INSERT into counters VALUES ' 4543 '(?,?,?)', ( counter, myid, mybranch ) 4544 ) 4545 except self.dbapi2.IntegrityError: 4546 if output: 4547 mytxt = "%s: %s: %s" % ( 4548 bold(_("ATTENTION")), 4549 red(_("counter for atom is duplicated, ignoring")), 4550 bold(myatom), 4551 ) 4552 self.updateProgress( 4553 mytxt, 4554 importance = 1, 4555 type = "warning" 4556 ) 4557 continue 4558 # don't trust counters, they might not be unique 4559 4560 self.commitChanges()

4561

4562 - def clearTreeupdatesEntries(self, repository):

4563 if not self.doesTableExist("treeupdates"): 4564 self.createTreeupdatesTable() 4565 # treeupdates 4566 with self.__write_mutex: 4567 self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,)) 4568 self.commitChanges()

4569

4570 - def resetTreeupdatesDigests(self):

4571 with self.__write_mutex: 4572 self.cursor.execute('UPDATE treeupdates SET digest = "-1"') 4573 self.commitChanges()

4574

4575 - def migrateCountersTable(self):

4576 with self.__write_mutex: 4577 self._migrateCountersTable()

4578

4579 - def _migrateCountersTable(self):

4580 self.cursor.executescript(""" 4581 DROP TABLE IF EXISTS counterstemp; 4582 CREATE TABLE counterstemp ( 4583 counter INTEGER, idpackage INTEGER, branch VARCHAR, 4584 PRIMARY KEY(idpackage,branch) 4585 ); 4586 INSERT INTO counterstemp (counter, idpackage, branch) 4587 SELECT counter, idpackage, branch FROM counters; 4588 DROP TABLE counters; 4589 ALTER TABLE counterstemp RENAME TO counters; 4590 """) 4591 self.commitChanges()

4592

4593 - def createNeededlibrarypathsTable(self):

4594 with self.__write_mutex: 4595 self.cursor.executescript(""" 4596 DROP TABLE IF EXISTS neededlibrarypaths; 4597 CREATE TABLE neededlibrarypaths ( 4598 library VARCHAR, 4599 path VARCHAR, 4600 elfclass INTEGER, 4601 PRIMARY KEY(library, path, elfclass) 4602 ); 4603 """)

4604

4605 - def createNeededlibraryidpackagesTable(self):

4606 with self.__write_mutex: 4607 self.cursor.executescript(""" 4608 DROP TABLE IF EXISTS neededlibraryidpackages; 4609 CREATE TABLE neededlibraryidpackages ( 4610 idpackage INTEGER, 4611 library VARCHAR, 4612 elfclass INTEGER 4613 ); 4614 """)

4615

4616 - def createInstalledTableSource(self):

4617 with self.__write_mutex: 4618 self.cursor.execute('ALTER TABLE installedtable ADD source INTEGER;') 4619 self.cursor.execute(""" 4620 UPDATE installedtable SET source = (?) 4621 """, (etpConst['install_sources']['unknown'],))

4622

4623 - def createPackagechangelogsTable(self):

4624 with self.__write_mutex: 4625 self.cursor.execute('CREATE TABLE packagechangelogs ( category VARCHAR, name VARCHAR, changelog BLOB, PRIMARY KEY (category, name));')

4626

4627 - def createAutomergefilesTable(self):

4628 with self.__write_mutex: 4629 self.cursor.execute('CREATE TABLE automergefiles ( idpackage INTEGER, configfile VARCHAR, md5 VARCHAR );')

4630

4631 - def createPackagesignaturesTable(self):

4632 with self.__write_mutex: 4633 self.cursor.execute('CREATE TABLE packagesignatures ( idpackage INTEGER PRIMARY KEY, sha1 VARCHAR, sha256 VARCHAR, sha512 VARCHAR );')

4634

4635 - def createPackagespmphases(self):

4636 with self.__write_mutex: 4637 self.cursor.execute(""" 4638 CREATE TABLE packagespmphases ( 4639 idpackage INTEGER PRIMARY KEY, 4640 phases VARCHAR 4641 ); 4642 """)

4643

4644 - def createPackagesetsTable(self):

4645 with self.__write_mutex: 4646 self.cursor.execute('CREATE TABLE packagesets ( setname VARCHAR, dependency VARCHAR );')

4647

4648 - def createCategoriesdescriptionTable(self):

4649 with self.__write_mutex: 4650 self.cursor.execute('CREATE TABLE categoriesdescription ( category VARCHAR, locale VARCHAR, description VARCHAR );')

4651

4652 - def createTreeupdatesTable(self):

4653 with self.__write_mutex: 4654 self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );')

4655

4656 - def createLicensedataTable(self):

4657 with self.__write_mutex: 4658 self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );')

4659

4660 - def createLicensesAcceptedTable(self):

4661 with self.__write_mutex: 4662 self.cursor.execute('CREATE TABLE licenses_accepted ( licensename VARCHAR UNIQUE );')

4663

4664 - def createInstalledTable(self):

4665 with self.__write_mutex: 4666 self.cursor.execute('DROP TABLE IF EXISTS installedtable;') 4667 self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR, source INTEGER );')

4668

4669 - def addDependsRelationToDependsTable(self, iterable):

4670 with self.__write_mutex: 4671 self.cursor.executemany('INSERT into dependstable VALUES (?,?)', iterable) 4672 if (self.entropyTools.is_user_in_entropy_group()) and \ 4673 (self.dbname.startswith(etpConst['serverdbid'])): 4674 # force commit even if readonly, this will allow to automagically fix dependstable server side 4675 self.connection.commit() # we don't care much about syncing the database since it's quite trivial

4676

4677 - def clearDependsTable(self):

4678 if not self.doesTableExist("dependstable"): 4679 return 4680 self.cursor.executescript(""" 4681 DELETE FROM dependstable; 4682 INSERT INTO dependstable VALUES (-1,-1); 4683 """)

4684

4685 - def regenerateDependsTable(self, output = True):

4686 4687 depends = self.listAllDependencies() 4688 count = 0 4689 total = len(depends) 4690 mydata = set() 4691 am = self.atomMatch 4692 up = self.updateProgress 4693 self.clearDependsTable() 4694 for iddep, atom in depends: 4695 count += 1 4696 4697 if output and ((count == 0) or (count % 150 == 0) or \ 4698 (count == total)): 4699 up( red("Resolving %s") % (atom,), importance = 0, 4700 type = "info", back = True, count = (count, total) 4701 ) 4702 4703 idpackage, rc = am(atom) 4704 if idpackage == -1: 4705 continue 4706 mydata.add((iddep, idpackage)) 4707 4708 if mydata: 4709 self.addDependsRelationToDependsTable(mydata) 4710 4711 # now validate dependstable 4712 self.sanitizeDependsTable()

4713

4714 - def regenerateLibrarypathsidpackageTable(self, output = True):

4715 4716 if output: 4717 self.updateProgress( 4718 "%s ..." % ( 4719 purple(_("Resolving libraries, please wait")), 4720 ), 4721 importance = 0, type = "info", back = True 4722 ) 4723 self.cursor.executescript(""" 4724 DELETE FROM neededlibraryidpackages; 4725 INSERT INTO neededlibraryidpackages (idpackage, library, elfclass) 4726 SELECT 4727 baseinfo.idpackage as idpackage, 4728 neededreference.library as library, 4729 neededlibrarypaths.elfclass as elfclass 4730 FROM 4731 baseinfo, neededlibrarypaths, needed, neededreference, content 4732 WHERE 4733 neededreference.idneeded = needed.idneeded AND 4734 needed.idpackage = content.idpackage AND 4735 baseinfo.idpackage = needed.idpackage AND 4736 neededlibrarypaths.library = neededreference.library AND 4737 neededlibrarypaths.elfclass = needed.elfclass AND 4738 content.file = neededlibrarypaths.path 4739 GROUP BY idpackage, library; 4740 4741 """) 4742 if output: 4743 self.updateProgress( 4744 "%s" % ( 4745 purple(_("Libraries solved, all fine")), 4746 ), 4747 importance = 0, type = "info" 4748 )

4749

4750 - def moveCountersToBranch(self, to_branch, from_branch = None):

4751 with self.__write_mutex: 4752 if from_branch is not None: 4753 self.cursor.execute('UPDATE counters SET branch = (?) WHERE branch = (?)', (to_branch, from_branch,)) 4754 else: 4755 self.cursor.execute('UPDATE counters SET branch = (?)', (to_branch,)) 4756 self.commitChanges() 4757 self.clearCache()

4758

4759 - def atomMatchFetchCache(self, *args):

4760 if self.xcache: 4761 cached = self.dumpTools.loadobj("%s/%s/%s" % (self.dbMatchCacheKey, self.dbname, hash(tuple(args)),)) 4762 if cached != None: return cached

4763

4764 - def atomMatchStoreCache(self, *args, **kwargs):

4765 if self.xcache: 4766 self.Cacher.push("%s/%s/%s" % ( 4767 self.dbMatchCacheKey,self.dbname,hash(tuple(args)),), 4768 kwargs.get('result') 4769 )

4770

4771 - def atomMatchValidateCache(self, cached_obj, multiMatch, extendedResults):

4772 4773 # time wasted for a reason 4774 data, rc = cached_obj 4775 if rc != 0: return cached_obj 4776 4777 if (not extendedResults) and (not multiMatch): 4778 if not self.isIDPackageAvailable(data): return None 4779 elif extendedResults and (not multiMatch): 4780 # ((idpackage,0,version,versiontag,revision,),0) 4781 if not self.isIDPackageAvailable(data[0]): return None 4782 elif extendedResults and multiMatch: 4783 # (set([(idpackage,0,version,version_tag,revision) for x in dbpkginfo]),0) 4784 idpackages = set([x[0] for x in data]) 4785 if not self.areIDPackagesAvailable(idpackages): return None 4786 elif (not extendedResults) and multiMatch: 4787 # (set([x[0] for x in dbpkginfo]),0) 4788 idpackages = set(data) 4789 if not self.areIDPackagesAvailable(idpackages): return None 4790 4791 return cached_obj

4792

4793 - def _idpackageValidator_live(self, idpackage, reponame):

4794 if (idpackage, reponame) in self.SystemSettings['live_packagemasking']['mask_matches']: 4795 # do not cache this 4796 return -1, self.SystemSettings['pkg_masking_reference']['user_live_mask'] 4797 elif (idpackage, reponame) in self.SystemSettings['live_packagemasking']['unmask_matches']: 4798 return idpackage, self.SystemSettings['pkg_masking_reference']['user_live_unmask']

4799

4800 - def _idpackageValidator_user_package_mask(self, idpackage, reponame, live):

4801 # check if user package.mask needs it masked 4802 4803 mykw = "%smask_ids" % (reponame,) 4804 user_package_mask_ids = self.SystemSettings.get(mykw) 4805 if not isinstance(user_package_mask_ids, (list, set,)): 4806 user_package_mask_ids = set() 4807 for atom in self.SystemSettings['mask']: 4808 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4809 if r != 0: 4810 continue 4811 user_package_mask_ids |= set(matches) 4812 self.SystemSettings[mykw] = user_package_mask_ids 4813 if idpackage in user_package_mask_ids: 4814 # sorry, masked 4815 myr = self.SystemSettings['pkg_masking_reference']['user_package_mask'] 4816 try: 4817 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4818 validator_cache[(idpackage, reponame, live)] = -1, myr 4819 except KeyError: # system settings client plugin not found 4820 pass 4821 return -1, myr

4822

4823 - def _idpackageValidator_user_package_unmask(self, idpackage, reponame, live):

4824 # see if we can unmask by just lookin into user package.unmask stuff -> self.SystemSettings['unmask'] 4825 mykw = "%sunmask_ids" % (reponame,) 4826 user_package_unmask_ids = self.SystemSettings.get(mykw) 4827 if not isinstance(user_package_unmask_ids, (list, set,)): 4828 user_package_unmask_ids = set() 4829 for atom in self.SystemSettings['unmask']: 4830 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4831 if r != 0: 4832 continue 4833 user_package_unmask_ids |= set(matches) 4834 self.SystemSettings[mykw] = user_package_unmask_ids 4835 if idpackage in user_package_unmask_ids: 4836 myr = self.SystemSettings['pkg_masking_reference']['user_package_unmask'] 4837 try: 4838 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4839 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4840 except KeyError: # system settings client plugin not found 4841 pass 4842 return idpackage, myr

4843

4844 - def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live):

4845 # check if repository packages.db.mask needs it masked 4846 repos_mask = {} 4847 client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin'] 4848 client_settings = self.SystemSettings.get(client_plg_id, {}) 4849 if client_settings: 4850 repos_mask = client_settings['repositories']['mask'] 4851 repomask = repos_mask.get(reponame) 4852 if isinstance(repomask, (list, set,)): 4853 # first, seek into generic masking, all branches 4854 mask_repo_id = "%s_ids@@:of:%s" % (reponame, reponame,) # avoid issues with repository names 4855 repomask_ids = repos_mask.get(mask_repo_id) 4856 if not isinstance(repomask_ids, set): 4857 repomask_ids = set() 4858 for atom in repomask: 4859 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4860 if r != 0: 4861 continue 4862 repomask_ids |= set(matches) 4863 repos_mask[mask_repo_id] = repomask_ids 4864 if idpackage in repomask_ids: 4865 myr = self.SystemSettings['pkg_masking_reference']['repository_packages_db_mask'] 4866 try: 4867 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4868 validator_cache[(idpackage, reponame, live)] = -1, myr 4869 except KeyError: # system settings client plugin not found 4870 pass 4871 return -1, myr

4872

4873 - def _idpackageValidator_package_license_mask(self, idpackage, reponame, live):

4874 if self.SystemSettings['license_mask']: 4875 mylicenses = self.retrieveLicense(idpackage) 4876 mylicenses = mylicenses.strip().split() 4877 lic_mask = self.SystemSettings['license_mask'] 4878 for mylicense in mylicenses: 4879 if mylicense not in lic_mask: continue 4880 myr = self.SystemSettings['pkg_masking_reference']['user_license_mask'] 4881 try: 4882 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4883 validator_cache[(idpackage, reponame, live)] = -1, myr 4884 except KeyError: # system settings client plugin not found 4885 pass 4886 return -1, myr

4887

4888 - def _idpackageValidator_keyword_mask(self, idpackage, reponame, live):

4889 4890 mykeywords = self.retrieveKeywords(idpackage) 4891 # WORKAROUND for buggy entries 4892 if not mykeywords: mykeywords = [''] # ** is fine then 4893 # firstly, check if package keywords are in etpConst['keywords'] 4894 # (universal keywords have been merged from package.mask) 4895 for key in etpConst['keywords']: 4896 if key not in mykeywords: continue 4897 myr = self.SystemSettings['pkg_masking_reference']['system_keyword'] 4898 try: 4899 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4900 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4901 except KeyError: # system settings client plugin not found 4902 pass 4903 return idpackage, myr 4904 4905 # if we get here, it means we didn't find mykeywords in etpConst['keywords'] 4906 # we need to seek self.SystemSettings['keywords'] 4907 # seek in repository first 4908 if reponame in self.SystemSettings['keywords']['repositories']: 4909 for keyword in self.SystemSettings['keywords']['repositories'][reponame]: 4910 if keyword not in mykeywords: continue 4911 keyword_data = self.SystemSettings['keywords']['repositories'][reponame].get(keyword) 4912 if not keyword_data: continue 4913 if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok 4914 myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords_all'] 4915 try: 4916 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4917 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4918 except KeyError: # system settings client plugin not found 4919 pass 4920 return idpackage, myr 4921 kwd_key = "%s_ids" % (keyword,) 4922 keyword_data_ids = self.SystemSettings['keywords']['repositories'][reponame].get(kwd_key) 4923 if not isinstance(keyword_data_ids, set): 4924 keyword_data_ids = set() 4925 for atom in keyword_data: 4926 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4927 if r != 0: 4928 continue 4929 keyword_data_ids |= matches 4930 self.SystemSettings['keywords']['repositories'][reponame][kwd_key] = keyword_data_ids 4931 if idpackage in keyword_data_ids: 4932 myr = self.SystemSettings['pkg_masking_reference']['user_repo_package_keywords'] 4933 try: 4934 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4935 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4936 except KeyError: # system settings client plugin not found 4937 pass 4938 return idpackage, myr 4939 4940 # if we get here, it means we didn't find a match in repositories 4941 # so we scan packages, last chance 4942 for keyword in self.SystemSettings['keywords']['packages']: 4943 # first of all check if keyword is in mykeywords 4944 if keyword not in mykeywords: continue 4945 keyword_data = self.SystemSettings['keywords']['packages'].get(keyword) 4946 if not keyword_data: continue 4947 kwd_key = "%s_ids" % (keyword,) 4948 keyword_data_ids = self.SystemSettings['keywords']['packages'].get(reponame+kwd_key) 4949 if not isinstance(keyword_data_ids, (list, set,)): 4950 keyword_data_ids = set() 4951 for atom in keyword_data: 4952 # match atom 4953 matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False) 4954 if r != 0: 4955 continue 4956 keyword_data_ids |= matches 4957 self.SystemSettings['keywords']['packages'][reponame+kwd_key] = keyword_data_ids 4958 if idpackage in keyword_data_ids: 4959 # valid! 4960 myr = self.SystemSettings['pkg_masking_reference']['user_package_keywords'] 4961 try: 4962 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4963 validator_cache[(idpackage, reponame, live)] = idpackage, myr 4964 except KeyError: # system settings client plugin not found 4965 pass 4966 return idpackage, myr

4967 4968 4969 4970 # function that validate one atom by reading keywords settings 4971 # validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache']

4972 - def idpackageValidator(self, idpackage, live = True):

4973 4974 if self.dbname == etpConst['clientdbid']: 4975 return idpackage, 0 4976 elif self.dbname.startswith(etpConst['serverdbid']): 4977 return idpackage, 0 4978 4979 reponame = self.dbname[len(etpConst['dbnamerepoprefix']):] 4980 try: 4981 validator_cache = self.SystemSettings[self.client_settings_plugin_id]['masking_validation']['cache'] 4982 cached = validator_cache.get((idpackage, reponame, live)) 4983 if cached != None: 4984 return cached 4985 # avoid memleaks 4986 if len(validator_cache) > 10000: 4987 validator_cache.clear() 4988 except KeyError: # plugin does not exist 4989 pass 4990 4991 if live: 4992 data = self._idpackageValidator_live(idpackage, reponame) 4993 if data: return data 4994 4995 data = self._idpackageValidator_user_package_mask(idpackage, reponame, live) 4996 if data: return data 4997 4998 data = self._idpackageValidator_user_package_unmask(idpackage, reponame, live) 4999 if data: return data 5000 5001 data = self._idpackageValidator_packages_db_mask(idpackage, reponame, live) 5002 if data: return data 5003 5004 data = self._idpackageValidator_package_license_mask(idpackage, reponame, live) 5005 if data: return data 5006 5007 data = self._idpackageValidator_keyword_mask(idpackage, reponame, live) 5008 if data: return data 5009 5010 # holy crap, can't validate 5011 myr = self.SystemSettings['pkg_masking_reference']['completely_masked'] 5012 validator_cache[(idpackage, reponame, live)] = -1, myr 5013 return -1, myr

5014 5015 # packages filter used by atomMatch, input must me foundIDs, a list like this: 5016 # [608,1867]

5017 - def packagesFilter(self, results):

5018 # keywordsFilter ONLY FILTERS results if 5019 # self.dbname.startswith(etpConst['dbnamerepoprefix']) => repository database is open 5020 if not self.dbname.startswith(etpConst['dbnamerepoprefix']): 5021 return results 5022 5023 newresults = set() 5024 for idpackage in results: 5025 idpackage, reason = self.idpackageValidator(idpackage) 5026 if idpackage == -1: continue 5027 newresults.add(idpackage) 5028 return newresults

5029

5030 - def __filterSlot(self, idpackage, slot):

5031 if slot == None: 5032 return idpackage 5033 dbslot = self.retrieveSlot(idpackage) 5034 if str(dbslot) == str(slot): 5035 return idpackage

5036

5037 - def __filterTag(self, idpackage, tag, operators):

5038 if tag == None: 5039 return idpackage 5040 dbtag = self.retrieveVersionTag(idpackage) 5041 compare = cmp(tag, dbtag) 5042 if not operators or operators == "=": 5043 if compare == 0: 5044 return idpackage 5045 else: 5046 return self.__do_operator_compare(idpackage, operators, compare)

5047

5048 - def __filterUse(self, idpackage, use):

5049 if not use: 5050 return idpackage 5051 pkguse = self.retrieveUseflags(idpackage) 5052 disabled = set([x[1:] for x in use if x.startswith("-")]) 5053 enabled = set([x for x in use if not x.startswith("-")]) 5054 enabled_not_satisfied = enabled - pkguse 5055 # check enabled 5056 if enabled_not_satisfied: 5057 return None 5058 # check disabled 5059 disabled_not_satisfied = disabled - pkguse 5060 if len(disabled_not_satisfied) != len(disabled): 5061 return None 5062 return idpackage

5063

5064 - def __do_operator_compare(self, token, operators, compare):

5065 if operators == ">" and compare == -1: 5066 return token 5067 elif operators == ">=" and compare < 1: 5068 return token 5069 elif operators == "<" and compare == 1: 5070 return token 5071 elif operators == "<=" and compare > -1: 5072 return token

5073

5074 - def __filterSlotTagUse(self, foundIDs, slot, tag, use, operators):

5075 5076 def myfilter(idpackage): 5077 5078 idpackage = self.__filterSlot(idpackage, slot) 5079 if not idpackage: 5080 return False 5081 5082 idpackage = self.__filterUse(idpackage, use) 5083 if not idpackage: 5084 return False 5085 5086 idpackage = self.__filterTag(idpackage, tag, operators) 5087 if not idpackage: 5088 return False 5089 5090 return True

5091 5092 return set(filter(myfilter, foundIDs)) 5093

5094 - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, 5095 matchBranches = (), matchTag = None, matchUse = (), packagesFilter = True, 5096 matchRevision = None, extendedResults = False, useCache = True ):

5097 5098 """ 5099 5100 @description: matches the user chosen package name+ver, if possibile, in a single repository 5101 @input atom: string, atom to match 5102 @input caseSensitive: bool, should the atom be parsed case sensitive? 5103 @input matchSlot: string, match atoms with the provided slot 5104 @input multiMatch: bool, return all the available atoms 5105 @input matchBranches: tuple or list, match packages only in the specified branches 5106 @input matchTag: match packages only for the specified tag 5107 @input matchUse: match packages only if it owns the specified use flags 5108 @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter 5109 @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error 5110 5111 """ 5112 5113 if not atom: 5114 return -1, 1 5115 5116 if useCache: 5117 cached = self.atomMatchFetchCache( 5118 atom, caseSensitive, matchSlot, 5119 multiMatch, matchBranches, matchTag, 5120 matchUse, packagesFilter, matchRevision, 5121 extendedResults 5122 ) 5123 if isinstance(cached, tuple): 5124 try: 5125 cached = self.atomMatchValidateCache(cached, multiMatch, extendedResults) 5126 except (TypeError, ValueError, IndexError, KeyError,): 5127 cached = None 5128 if isinstance(cached, tuple): 5129 return cached 5130 5131 atomTag = self.entropyTools.dep_gettag(atom) 5132 try: 5133 atomUse = self.entropyTools.dep_getusedeps(atom) 5134 except InvalidAtom: 5135 atomUse = () 5136 atomSlot = self.entropyTools.dep_getslot(atom) 5137 atomRev = self.entropyTools.dep_get_entropy_revision(atom) 5138 if isinstance(atomRev, (int, long,)): 5139 if atomRev < 0: atomRev = None 5140 5141 # use match 5142 scan_atom = self.entropyTools.remove_usedeps(atom) 5143 if (not matchUse) and (atomUse): 5144 matchUse = atomUse 5145 5146 # tag match 5147 scan_atom = self.entropyTools.remove_tag(scan_atom) 5148 if (matchTag == None) and (atomTag != None): 5149 matchTag = atomTag 5150 5151 # slot match 5152 scan_atom = self.entropyTools.remove_slot(scan_atom) 5153 if (matchSlot == None) and (atomSlot != None): 5154 matchSlot = atomSlot 5155 5156 # revision match 5157 scan_atom = self.entropyTools.remove_entropy_revision(scan_atom) 5158 if (matchRevision == None) and (atomRev != None): 5159 matchRevision = atomRev 5160 5161 branch_list = () 5162 direction = '' 5163 justname = True 5164 pkgkey = '' 5165 strippedAtom = '' 5166 foundIDs = [] 5167 dbpkginfo = set() 5168 5169 if scan_atom: 5170 5171 while 1: 5172 pkgversion = '' 5173 # check for direction 5174 strippedAtom = self.entropyTools.dep_getcpv(scan_atom) 5175 if scan_atom[-1] == "*": 5176 strippedAtom += "*" 5177 direction = scan_atom[0:len(scan_atom)-len(strippedAtom)] 5178 5179 justname = self.entropyTools.isjustname(strippedAtom) 5180 pkgkey = strippedAtom 5181 if justname == 0: 5182 # get version 5183 data = self.entropyTools.catpkgsplit(strippedAtom) 5184 if data == None: break # badly formatted 5185 pkgversion = data[2]+"-"+data[3] 5186 pkgkey = self.entropyTools.dep_getkey(strippedAtom) 5187 5188 splitkey = pkgkey.split("/") 5189 if (len(splitkey) == 2): 5190 pkgname = splitkey[1] 5191 pkgcat = splitkey[0] 5192 else: 5193 pkgname = splitkey[0] 5194 pkgcat = "null" 5195 5196 branch_list = (self.db_branch,) 5197 if matchBranches: 5198 # force to tuple for security 5199 branch_list = tuple(matchBranches) 5200 break 5201 5202 5203 if branch_list: 5204 # IDs found in the database that match our search 5205 foundIDs = self.__generate_found_ids_match(branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch) 5206 5207 ### FILTERING 5208 # filter slot and tag 5209 if foundIDs: 5210 foundIDs = self.__filterSlotTagUse(foundIDs, matchSlot, matchTag, matchUse, direction) 5211 if packagesFilter: 5212 foundIDs = self.packagesFilter(foundIDs) 5213 ### END FILTERING 5214 5215 if foundIDs: 5216 dbpkginfo = self.__handle_found_ids_match(foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, pkgversion) 5217 5218 if not dbpkginfo: 5219 if extendedResults: 5220 if multiMatch: 5221 x = set() 5222 else: 5223 x = (-1, 1, None, None, None,) 5224 self.atomMatchStoreCache( 5225 atom, caseSensitive, matchSlot, 5226 multiMatch, matchBranches, matchTag, 5227 matchUse, packagesFilter, matchRevision, 5228 extendedResults, result = (x, 1) 5229 ) 5230 return x, 1 5231 else: 5232 if multiMatch: 5233 x = set() 5234 else: 5235 x = -1 5236 self.atomMatchStoreCache( 5237 atom, caseSensitive, matchSlot, 5238 multiMatch, matchBranches, matchTag, 5239 matchUse, packagesFilter, matchRevision, 5240 extendedResults, result = (x, 1) 5241 ) 5242 return x, 1 5243 5244 if multiMatch: 5245 if extendedResults: 5246 x = set([(x[0], 0, x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) for x in dbpkginfo]) 5247 self.atomMatchStoreCache( 5248 atom, caseSensitive, matchSlot, 5249 multiMatch, matchBranches, matchTag, 5250 matchUse, packagesFilter, matchRevision, 5251 extendedResults, result = (x, 0) 5252 ) 5253 return x, 0 5254 else: 5255 x = set([x[0] for x in dbpkginfo]) 5256 self.atomMatchStoreCache( 5257 atom, caseSensitive, matchSlot, 5258 multiMatch, matchBranches, matchTag, 5259 matchUse, packagesFilter, matchRevision, 5260 extendedResults, result = (x, 0) 5261 ) 5262 return x, 0 5263 5264 if len(dbpkginfo) == 1: 5265 x = dbpkginfo.pop() 5266 if extendedResults: 5267 x = (x[0], 0, x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) 5268 self.atomMatchStoreCache( 5269 atom, caseSensitive, matchSlot, 5270 multiMatch, matchBranches, matchTag, 5271 matchUse, packagesFilter, matchRevision, 5272 extendedResults, result = (x, 0) 5273 ) 5274 return x, 0 5275 else: 5276 self.atomMatchStoreCache( 5277 atom, caseSensitive, matchSlot, 5278 multiMatch, matchBranches, matchTag, 5279 matchUse, packagesFilter, matchRevision, 5280 extendedResults, result = (x[0], 0) 5281 ) 5282 return x[0], 0 5283 5284 dbpkginfo = list(dbpkginfo) 5285 pkgdata = {} 5286 versions = set() 5287 for x in dbpkginfo: 5288 info_tuple = (x[1], self.retrieveVersionTag(x[0]), self.retrieveRevision(x[0])) 5289 versions.add(info_tuple) 5290 pkgdata[info_tuple] = x[0] 5291 newer = self.entropyTools.get_entropy_newer_version(list(versions))[0] 5292 x = pkgdata[newer] 5293 if extendedResults: 5294 x = (x, 0, newer[0], newer[1], newer[2]) 5295 self.atomMatchStoreCache( 5296 atom, caseSensitive, matchSlot, 5297 multiMatch, matchBranches, matchTag, 5298 matchUse, packagesFilter, matchRevision, 5299 extendedResults, result = (x, 0) 5300 ) 5301 return x, 0 5302 else: 5303 self.atomMatchStoreCache( 5304 atom, caseSensitive, matchSlot, 5305 multiMatch, matchBranches, matchTag, 5306 matchUse, packagesFilter, matchRevision, 5307 extendedResults, result = (x, 0) 5308 ) 5309 return x, 0

5310

5311 - def __generate_found_ids_match(self, branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch):

5312 foundIDs = set() 5313 for idx in branch_list: 5314 5315 if pkgcat == "null": 5316 results = self.searchPackagesByName( 5317 pkgname, sensitive = caseSensitive, 5318 branch = idx, justid = True 5319 ) 5320 else: 5321 results = self.searchPackagesByNameAndCategory( 5322 name = pkgname, category = pkgcat, branch = idx, 5323 sensitive = caseSensitive, justid = True 5324 ) 5325 5326 mypkgcat = pkgcat 5327 mypkgname = pkgname 5328 virtual = False 5329 # if it's a PROVIDE, search with searchProvide 5330 # there's no package with that name 5331 if (not results) and (mypkgcat == "virtual"): 5332 virtuals = self.searchProvide(pkgkey, branch = idx, justid = True) 5333 if virtuals: 5334 virtual = True 5335 mypkgname = self.retrieveName(virtuals[0]) 5336 mypkgcat = self.retrieveCategory(virtuals[0]) 5337 results = virtuals 5338 5339 # now validate 5340 if not results: 5341 continue # search into a stabler branch 5342 5343 elif (len(results) > 1): 5344 5345 # if it's because category differs, it's a problem 5346 foundCat = None 5347 cats = set() 5348 for idpackage in results: 5349 cat = self.retrieveCategory(idpackage) 5350 cats.add(cat) 5351 if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): 5352 # in case of virtual packages only (that they're not stored as provide) 5353 foundCat = cat 5354 5355 # if we found something at least... 5356 if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual", "null")): 5357 foundCat = sorted(cats)[0] 5358 5359 if not foundCat: 5360 # got the issue 5361 continue 5362 5363 # we can use foundCat 5364 mypkgcat = foundCat 5365 5366 # we need to search using the category 5367 if (not multiMatch) and (pkgcat == "null" or virtual): 5368 # we searched by name, we need to search using category 5369 results = self.searchPackagesByNameAndCategory( 5370 name = mypkgname, category = mypkgcat, 5371 branch = idx, sensitive = caseSensitive, justid = True 5372 ) 5373 5374 # validate again 5375 if not results: 5376 continue # search into another branch 5377 5378 # if we get here, we have found the needed IDs 5379 foundIDs |= set(results) 5380 break 5381 5382 else: 5383 5384 idpackage = results[0] 5385 # if mypkgcat is virtual, we can force 5386 if (mypkgcat == "virtual") and (not virtual): 5387 # in case of virtual packages only (that they're not stored as provide) 5388 mypkgcat = self.retrieveCategory(idpackage) 5389 5390 # check if category matches 5391 if mypkgcat != "null": 5392 foundCat = self.retrieveCategory(idpackage) 5393 if mypkgcat == foundCat: 5394 foundIDs.add(idpackage) 5395 continue 5396 foundIDs.add(idpackage) 5397 break 5398 5399 return foundIDs

5400 5401

5402 - def __handle_found_ids_match(self, foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, pkgversion):

5403 5404 dbpkginfo = set() 5405 # now we have to handle direction 5406 if ((direction) or ((not direction) and (not justname)) or ((not direction) and (not justname) and strippedAtom.endswith("*"))) and foundIDs: 5407 5408 if (not justname) and \ 5409 ((direction == "~") or (direction == "=") or \ 5410 (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*"))): 5411 # any revision within the version specified OR the specified version 5412 5413 if (direction == '' and not justname): 5414 direction = "=" 5415 5416 # remove gentoo revision (-r0 if none) 5417 if (direction == "="): 5418 if (pkgversion.split("-")[-1] == "r0"): 5419 pkgversion = self.entropyTools.remove_revision(pkgversion) 5420 if (direction == "~"): 5421 pkgrevision = self.entropyTools.dep_get_portage_revision(pkgversion) 5422 pkgversion = self.entropyTools.remove_revision(pkgversion) 5423 5424 for idpackage in foundIDs: 5425 5426 dbver = self.retrieveVersion(idpackage) 5427 if (direction == "~"): 5428 myrev = self.entropyTools.dep_get_portage_revision(dbver) 5429 myver = self.entropyTools.remove_revision(dbver) 5430 if myver == pkgversion and pkgrevision <= myrev: 5431 # found 5432 dbpkginfo.add((idpackage, dbver)) 5433 else: 5434 # media-libs/test-1.2* support 5435 if pkgversion[-1] == "*": 5436 if dbver.startswith(pkgversion[:-1]): 5437 dbpkginfo.add((idpackage, dbver)) 5438 elif (matchRevision != None) and (pkgversion == dbver): 5439 dbrev = self.retrieveRevision(idpackage) 5440 if dbrev == matchRevision: 5441 dbpkginfo.add((idpackage, dbver)) 5442 elif (pkgversion == dbver) and (matchRevision == None): 5443 dbpkginfo.add((idpackage, dbver)) 5444 5445 elif (direction.find(">") != -1) or (direction.find("<") != -1): 5446 5447 if not justname: 5448 5449 # remove revision (-r0 if none) 5450 if pkgversion.endswith("r0"): 5451 # remove 5452 self.entropyTools.remove_revision(pkgversion) 5453 5454 for idpackage in foundIDs: 5455 5456 revcmp = 0 5457 tagcmp = 0 5458 if matchRevision != None: 5459 dbrev = self.retrieveRevision(idpackage) 5460 revcmp = cmp(matchRevision, dbrev) 5461 if matchTag != None: 5462 dbtag = self.retrieveVersionTag(idpackage) 5463 tagcmp = cmp(matchTag, dbtag) 5464 dbver = self.retrieveVersion(idpackage) 5465 pkgcmp = self.entropyTools.compare_versions(pkgversion, dbver) 5466 if pkgcmp == None: 5467 import warnings 5468 warnings.warn("WARNING, invalid version string stored in %s: %s <-> %s" % (self.dbname, pkgversion, dbver,)) 5469 continue 5470 if direction == ">": 5471 if pkgcmp < 0: 5472 dbpkginfo.add((idpackage, dbver)) 5473 elif (matchRevision != None) and pkgcmp <= 0 and revcmp < 0: 5474 dbpkginfo.add((idpackage, dbver)) 5475 elif (matchTag != None) and tagcmp < 0: 5476 dbpkginfo.add((idpackage, dbver)) 5477 elif direction == "<": 5478 if pkgcmp > 0: 5479 dbpkginfo.add((idpackage, dbver)) 5480 elif (matchRevision != None) and pkgcmp >= 0 and revcmp > 0: 5481 dbpkginfo.add((idpackage, dbver)) 5482 elif (matchTag != None) and tagcmp > 0: 5483 dbpkginfo.add((idpackage, dbver)) 5484 elif direction == ">=": 5485 if (matchRevision != None) and pkgcmp <= 0: 5486 if pkgcmp == 0: 5487 if revcmp <= 0: 5488 dbpkginfo.add((idpackage, dbver)) 5489 else: 5490 dbpkginfo.add((idpackage, dbver)) 5491 elif pkgcmp <= 0 and matchRevision == None: 5492 dbpkginfo.add((idpackage, dbver)) 5493 elif (matchTag != None) and tagcmp <= 0: 5494 dbpkginfo.add((idpackage, dbver)) 5495 elif direction == "<=": 5496 if (matchRevision != None) and pkgcmp >= 0: 5497 if pkgcmp == 0: 5498 if revcmp >= 0: 5499 dbpkginfo.add((idpackage, dbver)) 5500 else: 5501 dbpkginfo.add((idpackage, dbver)) 5502 elif pkgcmp >= 0 and matchRevision == None: 5503 dbpkginfo.add((idpackage, dbver)) 5504 elif (matchTag != None) and tagcmp >= 0: 5505 dbpkginfo.add((idpackage, dbver)) 5506 5507 else: # just the key 5508 5509 dbpkginfo = set([(x, self.retrieveVersion(x),) for x in foundIDs]) 5510 5511 return dbpkginfo

5512