From f3ff9aa8d65158dde84b13f8f7dc2771cc9dc97e Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Tue, 4 Mar 2008 10:10:24 +0000 Subject: [PATCH] improved database query speed by moving all the structural updates into one function ran once git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1383 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- libraries/databaseTools.py | 221 +++++++++++-------------------------- libraries/entropy.py | 12 +- 2 files changed, 72 insertions(+), 161 deletions(-) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 07dfcfc1d..244055c77 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -155,6 +155,9 @@ class etpDatabase: # lock mirror remotely and ensure to have latest database revision self.doServerDatabaseSyncLock(self.noUpload) + if (etpConst['uid'] == 0) and os.access(self.dbFile,os.W_OK): + self.databaseStructureUpdates() + def doServerDatabaseSyncLock(self, noUpload): from entropy import FtpInterface @@ -764,8 +767,6 @@ class etpDatabase: licid = self.addLicense(etpData['license']) # insert license information - if not self.doesTableExist("licensedata"): - self.createLicensedataTable() mylicenses = etpData['licensedata'].keys() for mylicense in mylicenses: found = self.isLicensedataKeyAvailable(mylicense) @@ -790,8 +791,6 @@ class etpDatabase: # baseinfo pkgatom = etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag - if not self.doesColumnInTableExist("baseinfo","trigger"): - self.createTriggerColumn() self.cursor.execute( 'INSERT into baseinfo VALUES ' '(NULL,?,?,?,?,?,?,?,?,?,?,?)' @@ -855,10 +854,6 @@ class etpDatabase: # content, a list self.insertContent(idpackage,etpData['content']) - if not self.doesTableExist("counters"): - self.createCountersTable() - elif not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() etpData['counter'] = int(etpData['counter']) # cast to integer if etpData['counter'] != -1 and not (etpData['injected']): @@ -901,8 +896,6 @@ class etpDatabase: raise # on disk size - if not self.doesTableExist("sizes"): - self.createSizesTable() self.cursor.execute( 'INSERT into sizes VALUES ' '(?,?)' @@ -911,8 +904,6 @@ class etpDatabase: ) ) - if not self.doesTableExist("triggers"): - self.createTriggerTable() # trigger blob self.cursor.execute( 'INSERT into triggers VALUES ' @@ -969,13 +960,9 @@ class etpDatabase: # injected? if etpData['injected']: - if not self.doesTableExist("injected"): - self.createInjectedTable() self.setInjected(idpackage) # compile messages - if not self.doesTableExist("messages"): - self.createMessagesTable() for message in etpData['messages']: self.cursor.execute( 'INSERT into messages VALUES ' @@ -985,8 +972,6 @@ class etpDatabase: ) ) - if not self.doesTableExist("systempackages"): - self.createSystemPackagesTable() # is it a system package? if etpData['systempackage']: self.cursor.execute( @@ -996,8 +981,6 @@ class etpDatabase: ) ) - if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): - self.createProtectTable() # create new protect if it doesn't exist idprotect = self.isProtectAvailable(etpData['config_protect']) if (idprotect == -1): @@ -1220,12 +1203,7 @@ class etpDatabase: # counter self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage) except: - if self.dbname == etpConst['clientdbid']: - if not self.doesTableExist("counters"): - self.createCountersTable() - else: - raise - elif self.dbname == etpConst['serverdbid']: + if self.dbname in [etpConst['clientdbid'],etpConst['serverdbid']]: raise try: # on disk sizes @@ -1445,8 +1423,6 @@ class etpDatabase: def removeLicensedata(self, license_name): if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() return self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,)) @@ -1477,8 +1453,6 @@ class etpDatabase: self.commitChanges() def insertContent(self, idpackage, content): - if not self.doesColumnInTableExist("content","type"): - self.createContentTypeColumn() for xfile in content: contenttype = content[xfile] if type(xfile) is unicode: @@ -1494,10 +1468,6 @@ class etpDatabase: def insertCounter(self, idpackage, counter, branch = None): self.checkReadOnly() - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() if not branch: branch = etpConst['branch'] self.cursor.execute('DELETE FROM counters WHERE counter = (?) OR idpackage = (?)', (counter,idpackage,)) @@ -1507,11 +1477,6 @@ class etpDatabase: def setCounter(self, idpackage, counter, branch = None): self.checkReadOnly() - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() - branchstring = '' insertdata = [counter,idpackage] if branch: @@ -1524,14 +1489,7 @@ class etpDatabase: self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata) except: if self.dbname == etpConst['clientdbid']: - if not self.doesTableExist("counters"): - self.createCountersTable() - else: - raise - self.cursor.execute( - 'INSERT into counters VALUES ' - '(?,?,?)', (counter,idpackage,branch) - ) + raise self.commitChanges() def contentDiff(self, idpackage, dbconn, dbconn_idpackage): @@ -1623,8 +1581,6 @@ class etpDatabase: def cleanupEclasses(self): self.checkReadOnly() - if not self.doesTableExist("eclasses"): - self.createEclassesTable() self.cursor.execute('SELECT idclass FROM eclassesreference') idclasses = self.fetchall2set(self.cursor.fetchall()) # now parse them into useflags table @@ -2001,8 +1957,6 @@ class etpDatabase: pass def retrieveRepositoryUpdatesDigest(self, repository): - if not self.doesTableExist("treeupdates"): - self.createTreeupdatesTable() self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,)) mydigest = self.cursor.fetchone() if mydigest: @@ -2011,30 +1965,17 @@ class etpDatabase: return -1 def listAllTreeUpdatesActions(self): - if not self.doesTableExist("treeupdatesactions"): - self.createTreeupdatesactionsTable() - elif not self.doesColumnInTableExist("treeupdatesactions","branch"): - self.createTreeupdatesactionsBranchColumn() self.cursor.execute('SELECT * FROM treeupdatesactions') return self.cursor.fetchall() def retrieveTreeUpdatesActions(self, repository, forbranch = etpConst['branch']): if not self.doesTableExist("treeupdatesactions"): - self.createTreeupdatesactionsTable() return set() - elif not self.doesColumnInTableExist("treeupdatesactions","branch"): - self.createTreeupdatesactionsBranchColumn() - self.cursor.execute('SELECT command FROM treeupdatesactions where repository = (?) and branch = (?)', (repository,forbranch)) return self.fetchall2set(self.cursor.fetchall()) # mainly used to restore a previous table, used by reagent in --initialize def addTreeUpdatesActions(self, updates): - if not self.doesTableExist("treeupdatesactions"): - self.createTreeupdatesactionsTable() - elif not self.doesColumnInTableExist("treeupdatesactions","branch"): - self.createTreeupdatesactionsBranchColumn() - for update in updates: idupdate = update[0] repository = update[1] @@ -2043,19 +1984,11 @@ class etpDatabase: self.cursor.execute('INSERT INTO treeupdatesactions VALUES (?,?,?,?)', (idupdate,repository,command,branch,)) def setRepositoryUpdatesDigest(self, repository, digest): - if not self.doesTableExist("treeupdates"): - self.createTreeupdatesTable() - self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository,digest,)) self.commitChanges() def addRepositoryUpdatesActions(self, repository, actions, forbranch = etpConst['branch']): - if not self.doesTableExist("treeupdatesactions"): - self.createTreeupdatesactionsTable() - elif not self.doesColumnInTableExist("treeupdatesactions","branch"): - self.createTreeupdatesactionsBranchColumn() - for command in actions: self.cursor.execute('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?)', (repository,command,forbranch,)) @@ -2083,9 +2016,6 @@ class etpDatabase: def retrieveTrigger(self, idpackage): - if not self.doesTableExist("triggers"): - self.createTriggerTable() - self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,)) trigger = self.cursor.fetchone() if trigger: @@ -2133,11 +2063,6 @@ class etpDatabase: #cache = self.fetchInfoCache(idpackage,'retrieveCounter') #if cache != None: return cache - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() - counter = -1 self.cursor.execute('SELECT counter FROM counters WHERE idpackage = (?)', (idpackage,)) mycounter = self.cursor.fetchone() @@ -2180,9 +2105,6 @@ class etpDatabase: #cache = self.fetchInfoCache(idpackage,'retrieveOnDiskSize') #if cache != None: return cache - if not self.doesTableExist("sizes"): - self.createSizesTable() - self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,)) size = self.cursor.fetchone() # do not use [0]! if not size: @@ -2288,9 +2210,6 @@ class etpDatabase: cache = self.fetchInfoCache(idpackage,'retrieveEclasses') if cache != None: return cache - if not self.doesTableExist("eclasses"): - self.createEclassesTable() - self.cursor.execute('SELECT classname FROM eclasses,eclassesreference WHERE eclasses.idpackage = (?) and eclasses.idclass = eclassesreference.idclass', (idpackage,)) classes = self.fetchall2set(self.cursor.fetchall()) @@ -2302,9 +2221,6 @@ class etpDatabase: cache = self.fetchInfoCache(idpackage,'retrieveNeeded') if cache != None: return cache - if not self.doesTableExist("needed"): - self.createNeededTable() - self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,)) needed = self.fetchall2set(self.cursor.fetchall()) @@ -2392,9 +2308,6 @@ class etpDatabase: cache = self.fetchInfoCache(idpackage,'retrieveProtect') if cache != None: return cache - if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): - self.createProtectTable() - self.cursor.execute('SELECT protect FROM configprotect,configprotectreference WHERE configprotect.idpackage = (?) and configprotect.idprotect = configprotectreference.idprotect', (idpackage,)) protect = self.cursor.fetchone() if not protect: @@ -2410,9 +2323,6 @@ class etpDatabase: cache = self.fetchInfoCache(idpackage,'retrieveProtectMask') if cache != None: return cache - if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): - self.createProtectTable() - self.cursor.execute('SELECT protect FROM configprotectmask,configprotectreference WHERE idpackage = (?) and configprotectmask.idprotect= configprotectreference.idprotect', (idpackage,)) protect = self.cursor.fetchone() if not protect: @@ -2456,9 +2366,6 @@ class etpDatabase: searchkeywords.append(contentType) contentstring = ' and type = (?)' - if not self.doesColumnInTableExist("content","type"): - self.createContentTypeColumn() - self.cursor.execute('SELECT file'+extstring+' FROM content WHERE idpackage = (?) '+contentstring, searchkeywords) if extended: @@ -2515,8 +2422,6 @@ class etpDatabase: # insert license information if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() return {} licenses = self.retrieveLicense(idpackage) licenses = licenses.split() @@ -2543,8 +2448,6 @@ class etpDatabase: if cache != None: return cache if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() return set() licenses = self.retrieveLicense(idpackage) licenses = licenses.split() @@ -2563,8 +2466,6 @@ class etpDatabase: def retrieveLicenseText(self, license_name): if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() return None self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") @@ -2653,8 +2554,6 @@ class etpDatabase: return result[0] def isProtectAvailable(self,protect): - if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): - self.createProtectTable() self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,)) result = self.cursor.fetchone() if not result: @@ -2705,8 +2604,6 @@ class etpDatabase: return result[0] def isEclassAvailable(self,eclass): - if not self.doesTableExist("eclasses"): - self.createEclassesTable() self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,)) result = self.cursor.fetchone() if not result: @@ -2714,8 +2611,6 @@ class etpDatabase: return result[0] def isNeededAvailable(self,needed): - if not self.doesTableExist("needed"): - self.createNeededTable() self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,)) result = self.cursor.fetchone() if not result: @@ -2723,12 +2618,6 @@ class etpDatabase: return result[0] def isCounterAvailable(self,counter, branch = None, branch_operator = "="): - - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() - result = False if not branch: branch = etpConst['branch'] @@ -2741,10 +2630,7 @@ class etpDatabase: def isLicensedataKeyAvailable(self, license_name): if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() - else: - return True + return True self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,)) result = self.cursor.fetchone() if not result: @@ -2752,8 +2638,6 @@ class etpDatabase: return True def isLicenseAccepted(self, license_name): - if not self.doesTableExist("licenses_accepted"): - self.createLicensesAcceptedTable() self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,)) result = self.cursor.fetchone() if not result: @@ -2763,8 +2647,6 @@ class etpDatabase: def acceptLicense(self, license_name): if self.readOnly or (etpConst['uid'] != 0): return - if not self.doesTableExist("licenses_accepted"): - self.createLicensesAcceptedTable() if self.isLicenseAccepted(license_name): return self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,)) @@ -2780,12 +2662,7 @@ class etpDatabase: return result[0] def isSystemPackage(self,idpackage): - - if not self.doesTableExist("systempackages"): - self.createSystemPackagesTable() - self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) - result = self.cursor.fetchone() if result: return True @@ -2796,9 +2673,6 @@ class etpDatabase: cache = self.fetchInfoCache(idpackage,'isInjected') if cache != None: return cache - if not self.doesTableExist("injected"): - self.createInjectedTable() - try: self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,)) except: @@ -3141,11 +3015,6 @@ class etpDatabase: def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="): - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() - branchstring = '' if branch: branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"' @@ -3259,8 +3128,6 @@ class etpDatabase: query = 'SELECT idprotect FROM configprotect' if mask: query += 'mask' - if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): - self.createProtectTable() self.cursor.execute(query) idprotects = self.fetchall2set(self.cursor.fetchall()) @@ -3317,6 +3184,61 @@ class etpDatabase: self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (newdownload,idpackage,)) self.commitChanges() + def databaseStructureUpdates(self): + + if not self.doesTableExist("licensedata"): + self.createLicensedataTable() + + if not self.doesTableExist("licenses_accepted"): + self.createLicensesAcceptedTable() + + if not self.doesColumnInTableExist("baseinfo","trigger"): + self.createTriggerColumn() + + if not self.doesTableExist("counters"): + self.createCountersTable() + elif not self.doesColumnInTableExist("counters","branch"): + self.createCountersBranchColumn() + + if not self.doesTableExist("sizes"): + self.createSizesTable() + + if not self.doesTableExist("triggers"): + self.createTriggerTable() + + if not self.doesTableExist("messages"): + self.createMessagesTable() + + if not self.doesTableExist("injected"): + self.createInjectedTable() + + if not self.doesTableExist("systempackages"): + self.createSystemPackagesTable() + + if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")): + self.createProtectTable() + + if not self.doesColumnInTableExist("content","type"): + self.createContentTypeColumn() + + if not self.doesTableExist("eclasses"): + self.createEclassesTable() + + if not self.doesTableExist("treeupdates"): + self.createTreeupdatesTable() + + if not self.doesTableExist("treeupdatesactions"): + self.createTreeupdatesactionsTable() + elif not self.doesColumnInTableExist("treeupdatesactions","branch"): + self.createTreeupdatesactionsBranchColumn() + + if not self.doesTableExist("needed"): + self.createNeededTable() + + if not self.doesTableExist("installedtable") and (self.dbname == etpConst['clientdbid']): + self.createInstalledTable() + + def validateDatabase(self): self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","baseinfo")) rslt = self.cursor.fetchone() @@ -3456,8 +3378,6 @@ class etpDatabase: return result def removePackageFromInstalledTable(self, idpackage): - if not self.doesTableExist("installedtable"): - self.createInstalledTable() self.cursor.execute('DELETE FROM installedtable WHERE idpackage = (?)', (idpackage,)) def removePackageFromDependsTable(self, idpackage): @@ -3575,10 +3495,7 @@ class etpDatabase: def createLicensedataIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: if not self.doesTableExist("licensedata"): - if (etpConst['uid'] == 0) and (not self.readOnly): - self.createLicensedataTable() - else: - return + return self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )') self.commitChanges() @@ -3642,27 +3559,17 @@ class etpDatabase: def clearTreeupdatesEntries(self, repository): self.checkReadOnly() # treeupdates - if not self.doesTableExist("treeupdates"): - self.createTreeupdatesTable() - else: - self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,)) + self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,)) def resetTreeupdatesDigests(self): self.checkReadOnly() - if not self.doesTableExist("treeupdates"): - self.createTreeupdatesTable() - else: - self.cursor.execute('UPDATE treeupdates SET digest = "-1"') + self.cursor.execute('UPDATE treeupdates SET digest = "-1"') # # FIXME: remove these when 1.0 will be out # def migrateCountersTable(self): - if not self.doesTableExist("counters"): - self.createCountersTable() - if not self.doesColumnInTableExist("counters","branch"): - self.createCountersBranchColumn() self.cursor.execute('DROP TABLE IF EXISTS counterstemp;') self.cursor.execute('CREATE TABLE counterstemp ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );') self.cursor.execute('select * from counters') diff --git a/libraries/entropy.py b/libraries/entropy.py index 111968692..21946af14 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -750,6 +750,12 @@ class EquoInterface(TextInterface): revision = -1 return revision + def update_repository_revision(self, reponame): + r = self.get_repository_revision(reponame) + etpRepositories[reponame]['dbrevision'] = "0" + if r != -1: + etpRepositories[reponame]['dbrevision'] = str(r) + # @returns -1 if the file does not exist # @returns int>0 if the file exists def get_repository_db_file_checksum(self, reponame): @@ -3856,10 +3862,6 @@ class RepoInterface: self.notAvailable = 0 self.reset_dbformat_eapi() - # check if I am root - if (not self.Entropy.entropyTools.isRoot()): - raise exceptionTools.PermissionDenied("PermissionDenied: not allowed as user.") - # check etpRepositories if not etpRepositories: raise exceptionTools.MissingParameter("MissingParameter: no repositories specified in %s" % (etpConst['repositoriesconf'],)) @@ -4334,6 +4336,8 @@ class RepoInterface: self.Entropy.clientDbconn.createAllIndexes() except: pass + # update revision in etpRepositories + self.Entropy.update_repository_revision(repo) self.Entropy.cycleDone()