- updated TODO

- improved databaseTools.listIdpackageDependencies() speed - improved databaseTools.listConfigProtectDirectories() speed - improved databaseTools.searchPackagesByDescription() speed git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1480 cd1c1023-2f26-0410-ae45-c471fc1f0318
2008-03-17 11:46:25 +00:00
parent 94040dc7f6
commit 35003d4625
4 changed files with 35 additions and 102 deletions
@@ -1,12 +1,10 @@
 TODO list:
-  - create a meta-packages list ... I hate Joost
  - during "equo remove" show disk size for each package
-  - NEEDED + ldconfig -p ?
  - make an ETP_DIR switching function
  - lilo.conf trigger?
  - equo: "you meant...?" feature
-  - add i18n support
-  - calculate extra deps from NEEDED, server side?
+  - calculate extra deps from NEEDED, server side? - NEEDED + ldconfig -p ?
+  - create a meta-packages list ... I hate Joost
  - split RDEPEND and PDEPEND ( + && and || )
  - log messages from portage doebuild() calls
  - remove conflicting packages right before the conflict
@@ -21,6 +19,7 @@ TODO list:
  - packages.sabayonlinux.org interactivity (comments + images upload + connection to phpbb user db)
  - Community repositories
  - write a ncurses interface to manage entropy database
+  - add i18n support
  - find a way to better handle real smartapps deps (need split PDEPEND?)

  Spritz:
@@ -2454,7 +2454,7 @@ class etpDatabase:
            lictext = self.cursor.fetchone()
            if lictext != None:
                #licdata[licname] = unicode(lictext[0],'raw_unicode_escape','replace')
-		licdata[licname] = str(lictext[0])
+                licdata[licname] = str(lictext[0])

        self.storeInfoCache(idpackage,'retrieveLicensedata',licdata)
        return licdata
@@ -2875,37 +2875,8 @@ class etpDatabase:
        return results

    def searchPackagesByDescription(self, keyword):
-	self.cursor.execute('SELECT idpackage FROM extrainfo WHERE LOWER(description) LIKE (?)', ("%"+keyword.lower()+"%",))
-	idpkgs = self.fetchall2set(self.cursor.fetchall())
-	if not idpkgs:
-	    return ()
-
-	result = set()
-	query = 'WHERE idpackage = '
-	counter = 0
-	run = False
-	for idpkg in idpkgs:
-	    run = True
-	    counter += 1
-	    query += str(idpkg)+' OR idpackage = '
-	    if counter > 25:
-		counter = 0
-		query = query[:-16]
-		self.cursor.execute('SELECT atom,idpackage FROM baseinfo '+query)
-		qry = self.cursor.fetchall()
-		for x in qry:
-		    result.add(x)
-		query = 'WHERE idpackage = '
-		run = False
-	
-	if (run):
-	    query = query[:-16]
-	    self.cursor.execute('SELECT atom,idpackage FROM baseinfo '+query)
-	    qry = self.cursor.fetchall()
-	    for x in qry:
-		result.add(x)
-	
-	return result
+        self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo WHERE LOWER(extrainfo.description) LIKE (?) and baseinfo.idpackage = extrainfo.idpackage', ("%"+keyword.lower()+"%",))
+        return self.cursor.fetchall()

    def searchPackagesByName(self, keyword, sensitive = False, branch = None):

@@ -3081,36 +3052,12 @@ class etpDatabase:

    def listIdpackageDependencies(self, idpackage):
        self.cursor.execute('SELECT iddependency FROM dependencies where idpackage = (?)', (idpackage,))
-        iddeps = self.fetchall2set(self.cursor.fetchall())
+        iddeps = tuple(self.fetchall2set(self.cursor.fetchall()))
        if not iddeps:
            return ()
-
        result = set()
-        query = 'WHERE iddependency = '
-        counter = 0
-        run = False
-        for iddep in iddeps:
-            run = True
-            counter += 1
-            query += str(iddep)+' OR iddependency = '
-            if counter > 25:
-                counter = 0
-                query = query[:-19]
-                self.cursor.execute('SELECT iddependency,dependency FROM dependenciesreference '+query)
-                qry = self.cursor.fetchall()
-                for x in qry:
-                    result.add(x)
-                query = 'WHERE iddependency = '
-                run = False
-
-        if (run):
-            query = query[:-19]
-            self.cursor.execute('SELECT iddependency,dependency FROM dependenciesreference '+query)
-            qry = self.cursor.fetchall()
-            for x in qry:
-                result.add(x)
-
-        return result
+        self.cursor.execute('SELECT iddependency,dependency FROM dependenciesreference WHERE iddependency IN %s' % (iddeps,))
+        return set(self.cursor.fetchall())

    def listBranchPackagesTbz2(self, branch):
        result = set()
@@ -3141,42 +3088,23 @@ class etpDatabase:
        return self.cursor.fetchall()

    def listConfigProtectDirectories(self, mask = False):
-        dirs = set()
-        query = 'SELECT idprotect FROM configprotect'
+        query = 'SELECT max(idprotect) FROM configprotect'
        if mask:
            query += 'mask'
        self.cursor.execute(query)
-        idprotects = self.fetchall2set(self.cursor.fetchall())
-
-        if not idprotects:
+        r = self.cursor.fetchone()
+        if not r:
            return []

-        results = set()
-        query = 'WHERE idprotect = '
-        counter = 0
-        run = False
-        for idprotect in idprotects:
-            run = True
-            counter += 1
-            query += str(idprotect)+' OR idprotect = '
-            if counter > 25:
-                counter = 0
-                query = query[:-16]
-                self.cursor.execute('SELECT protect FROM configprotectreference '+query)
-                results.update(self.fetchall2set(self.cursor.fetchall()))
-                query = 'WHERE idprotect = '
-                run = False
-
-        if (run):
-            query = query[:-16]
-            self.cursor.execute('SELECT protect FROM configprotectreference '+query)
-            results.update(self.fetchall2set(self.cursor.fetchall()))
-
-        for result in results:
-            for x in result.split():
-                dirs.add(x)
-        dirs = list(dirs)
-        dirs.sort()
+        mymax = r[0]
+        self.cursor.execute('SELECT protect FROM configprotectreference where idprotect >= (?) and idprotect <= (?) order by protect', (1,mymax,))
+        results = self.cursor.fetchall()
+        dirs = []
+        for row in results:
+            mydirs = row[0].split()
+            for x in mydirs:
+                if x not in dirs:
+                    dirs.append(x)
        return dirs

    def switchBranch(self, idpackage, tobranch):
@@ -3489,6 +3417,7 @@ class etpDatabase:
        self.createUseflagsIndex()
        self.createLicensedataIndex()
        self.createLicensesIndex()
+        self.createConfigProtectReferenceIndex()

    def createNeededIndex(self):
        if self.dbname != etpConst['serverdbid'] and self.indexing:
@@ -3505,6 +3434,11 @@ class etpDatabase:
            self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex ON content ( file )')
            self.commitChanges()

+    def createConfigProtectReferenceIndex(self):
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
+            self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )')
+            self.commitChanges()
+
    def createBaseinfoIndex(self):
        if self.dbname != etpConst['serverdbid'] and self.indexing:
            self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, versiontag, slot, branch, revision )')
@@ -23,8 +23,6 @@
 import shutil
 import commands
 import urllib2
-import socket
-import random
 import time
 from entropyConstants import *
 from outputTools import *
@@ -4429,11 +4427,13 @@ class FtpInterface:
        self.entropyTools = entropyTools
        import ftplib
        self.ftplib = ftplib
+        import socket
+        self.socket = socket

        self.oldprogress = 0.0

        # import FTP modules
-        socket.setdefaulttimeout(60)
+        self.socket.setdefaulttimeout(60)

        self.ftpuri = ftpuri
        self.ftphost = self.entropyTools.extractFTPHostFromUri(self.ftpuri)
@@ -4488,7 +4488,7 @@ class FtpInterface:
    # this can be used in case of exceptions
    def reconnectHost(self):
        # import FTP modules
-        socket.setdefaulttimeout(60)
+        self.socket.setdefaulttimeout(60)
        counter = 10
        while 1:
            counter -= 1
@@ -4753,7 +4753,9 @@ class urlFetcher:
        self.showSpeed = showSpeed
        self.initVars()
        import entropyTools
+        import socket
        self.entropyTools = entropyTools
+        self.socket = socket

        # resume support
        if os.path.isfile(self.pathToSave) and os.access(self.pathToSave,os.R_OK) and self.resume:
@@ -4804,7 +4806,7 @@ class urlFetcher:
        self.speedUpdater.start()

        # set timeout
-        socket.setdefaulttimeout(20)
+        self.socket.setdefaulttimeout(20)

        # get file size if available
        try:
@@ -4936,7 +4938,7 @@ class urlFetcher:
        except:
            pass
        self.speedUpdater.kill()
-        socket.setdefaulttimeout(2)
+        self.socket.setdefaulttimeout(2)

    def updateSpeedInfo(self):
        self.elapsed += self.transferpollingtime
@@ -22,7 +22,6 @@


 import random
-import gc
 import sys
 import os
 import stat
@@ -912,7 +911,6 @@ def initConfig_entropyConstants(rootdir):
                url += etpConst['product']+"/handlers/"
                etpRemoteSupport[servername] = url

-    gc.collect()
    initConfig_clientConstants()

 def initConfig_clientConstants():