- improved orphans search tool in terms of memory consumption and speed

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1216 cd1c1023-2f26-0410-ae45-c471fc1f0318
2008-02-15 09:57:27 +00:00
parent 15076a74b7
commit 9bf2a61c1a
4 changed files with 106 additions and 33 deletions
@@ -12,8 +12,9 @@ TODO list:
  - find a way to better handle real smartapps deps (need split PDEPEND?)

  Spritz:
-    - handle multi repos in "available" menu (*)
+    - handle multi repos in "available" menu (*) (un set keyslotcache)
    - add masking interface (*)
+    - add categories view? (*)
    - add branch switching menu (for upgrades) (*)
    - wrap root password request (*)
    - handle queue errors (*)
@@ -389,21 +389,34 @@ def searchOrphans():

    # start to list all files on the system:
    dirs = etpConst['filesystemdirs']
-    foundFiles = set()
+    filepath = Equo.entropyTools.getRandomTempFile()
+    tdbconn = Equo.openGenericDatabase(filepath)
+    tdbconn.initializeDatabase()
    for xdir in dirs:
        for currentdir,subdirs,files in os.walk(xdir):
+            foundFiles = {}
            for filename in files:
                # filter python compiled objects?
                if filename.endswith(".pyo") or filename.endswith(".pyc") or filename == '.keep':
                    continue
                filename = os.path.join(currentdir,filename)
+                if filename.endswith(".ph") and \
+                    (filename.startswith("/usr/lib/perl") or filename.startswith("/usr/lib64/perl")):
+                        continue
                mask = [x for x in etpConst['filesystemdirsmask'] if filename.startswith(x)]
                if not mask:
-                    if (not etpUi['quiet']):
+                    if not etpUi['quiet']:
                        print_info(red(" @@ ")+blue("Looking: ")+bold(filename[:50]+"..."), back = True)
-                    foundFiles.add(filename)
-    totalfiles = len(foundFiles)
-    if (not etpUi['quiet']):
+                    foundFiles[filename] = "obj"
+            if foundFiles:
+                tdbconn.insertContent(1,foundFiles)
+
+    tdbconn.commitChanges()
+    tdbconn.createContentIndex()
+    tdbconn.cursor.execute('select count(file) from content')
+    totalfiles = tdbconn.cursor.fetchone()[0]
+
+    if not etpUi['quiet']:
        print_info(red(" @@ ")+blue("Analyzed directories: ")+' '.join(etpConst['filesystemdirs']))
        print_info(red(" @@ ")+blue("Masked directories: ")+' '.join(etpConst['filesystemdirsmask']))
        print_info(red(" @@ ")+blue("Number of files collected on the filesystem: ")+bold(str(totalfiles)))
@@ -415,37 +428,65 @@ def searchOrphans():
    length = str(len(idpackages))
    count = 0
    for idpackage in idpackages:
-        if (not etpUi['quiet']):
+        if not etpUi['quiet']:
            count += 1
            atom = clientDbconn.retrieveAtom(idpackage)
            txt = "["+str(count)+"/"+length+"] "
            print_info(red(" @@ ")+blue("Intersecting content of package: ")+txt+bold(atom), back = True)
+        content = set()
        for x in clientDbconn.retrieveContent(idpackage):
            if x.startswith("/usr/lib64"):
                x = "/usr/lib"+x[len("/usr/lib64"):]
            content.add(x)
        # remove from foundFiles
-        foundFiles -= content
+        for item in content:
+            tdbconn.cursor.execute('delete from content where file = (?)', (item,))
+
+    tdbconn.commitChanges()
+    tdbconn.cursor.execute('select count(file) from content')
+    orpanedfiles = tdbconn.cursor.fetchone()[0]
+
    if (not etpUi['quiet']):
        print_info(red(" @@ ")+blue("Intersection completed. Showing statistics: "))
        print_info(red(" @@ ")+blue("Number of total files: ")+bold(str(totalfiles)))
-        print_info(red(" @@ ")+blue("Number of matching files: ")+bold(str(totalfiles - len(foundFiles))))
-        print_info(red(" @@ ")+blue("Number of orphaned files: ")+bold(str(len(foundFiles))))
+        print_info(red(" @@ ")+blue("Number of matching files: ")+bold(str(totalfiles - orpanedfiles)))
+        print_info(red(" @@ ")+blue("Number of orphaned files: ")+bold(str(orpanedfiles)))

-    # order
-    foundFiles = list(foundFiles)
-    foundFiles.sort()
+    tdbconn.cursor.execute('select file from content order by file desc')
    if not etpUi['quiet']:
-        print_info(red(" @@ ")+blue("Writing file to disk: ")+bold("/tmp/equo-orphans.txt"))
        f = open("/tmp/equo-orphans.txt","w")
-        for x in foundFiles:
-            f.write(x+"\n")
+        print_info(red(" @@ ")+blue("Writing file to disk: ")+bold("/tmp/equo-orphans.txt"))
+
+    tdbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
+    myfile = tdbconn.cursor.fetchone()
+
+    sizecount = 0
+    while myfile:
+        myfile = str(myfile[0].encode('raw_unicode_escape'))
+        try:
+            sizecount += os.stat(myfile)[6]
+        except OSError:
+            pass
+        if not etpUi['quiet']:
+            f.write(myfile+"\n")
+        else:
+            print myfile
+        myfile = tdbconn.cursor.fetchone()
+
+    humansize = Equo.entropyTools.bytesIntoHuman(sizecount)
+    if not etpUi['quiet']:
+        print_info(red(" @@ ")+blue("Total wasted space: ")+bold(humansize))
        f.flush()
        f.close()
-        return 0
    else:
-        for x in foundFiles:
-            print x
+        print humansize
+
+    tdbconn.closeDB()
+    try:
+        os.remove(filepath)
+    except OSError:
+        pass
+
    return 0


@@ -96,6 +96,9 @@ class EquoInterface(TextInterface):
        # masking parser
        self.MaskingParser = self.PackageMaskingParserInterfaceLoader()

+        self.validate_repositories_cache()
+
+        # now if we are on live, we should disable it
        # are we running on a livecd? (/proc/cmdline has "cdroot")
        if self.entropyTools.islive():
            self.xcache = False
@@ -108,6 +111,32 @@ class EquoInterface(TextInterface):
            except:
                pass

+    def validate_repositories_cache(self):
+        # is the list of repos changed?
+        cached = self.dumpTools.loadobj(etpCache['repolist'])
+        if cached == None:
+            # invalidate matching cache
+            try:
+                self.repository_move_clear_cache()
+            except IOError:
+                pass
+        elif type(cached) is tuple:
+            # compare
+            myrepolist = tuple(etpRepositoriesOrder)
+            if cached != myrepolist:
+                cached = set(cached)
+                myrepolist = set(myrepolist)
+                difflist = cached - myrepolist # before minus now
+                for repoid in difflist:
+                    try:
+                        self.repository_move_clear_cache(repoid)
+                    except IOError:
+                        pass
+        try:
+            self.dumpTools.dumpobj(etpCache['repolist'],tuple(etpRepositoriesOrder))
+        except IOError:
+            pass
+
    def switchChroot(self, chroot = ""):
        # clean caches
        self.purge_cache()
@@ -884,15 +913,16 @@ class EquoInterface(TextInterface):
        return dbpkginfo


-    def repository_move_clear_cache(self, repoid):
+    def repository_move_clear_cache(self, repoid = None):
        self.clear_dump_cache(etpCache['world_available'])
        self.clear_dump_cache(etpCache['world_update'])
        self.clear_dump_cache(etpCache['check_package_update'])
        self.clear_dump_cache(etpCache['filter_satisfied_deps'])
        self.clear_dump_cache(etpCache['atomMatch'])
-        self.clear_dump_cache(etpCache['dbMatch']+repoid+"/")
-        self.clear_dump_cache(etpCache['dbInfo']+"/"+repoid+"/")
-        self.clear_dump_cache(etpCache['dbSearch']+repoid+"/")
+        if repoid != None:
+            self.clear_dump_cache(etpCache['dbMatch']+"/"+repoid+"/")
+            self.clear_dump_cache(etpCache['dbInfo']+"/"+repoid+"/")
+            self.clear_dump_cache(etpCache['dbSearch']+"/"+repoid+"/")


    def addRepository(self, repodata):
@@ -6445,11 +6475,13 @@ class SecurityInterface:
        self.security_package_checksum = self.security_package+etpConst['packageshashfileext']


-        if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']):
-            os.remove(etpConst['securitydir'])
-        if not os.path.isdir(etpConst['securitydir']):
-            os.makedirs(etpConst['securitydir'])
-        elif os.path.isfile(self.old_download_package_checksum):
+        if etpConst['uid'] == 0:
+            if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']):
+                os.remove(etpConst['securitydir'])
+            if not os.path.isdir(etpConst['securitydir']):
+                os.makedirs(etpConst['securitydir'])
+
+        if os.path.isfile(self.old_download_package_checksum):
            f = open(self.old_download_package_checksum)
            try:
                self.previous_checksum = f.readline().strip().split()[0]
@@ -358,7 +358,8 @@ etpCache = {
    'dep_tree': 'deptree/dep_tree_',
    'depends_tree': 'depends/depends_tree_',
    'filter_satisfied_deps': 'depfilter/filter_satisfied_deps_',
-    'library_breakage': 'libs_break/library_breakage_'
+    'library_breakage': 'libs_break/library_breakage_',
+    'repolist': 'repos/repolist'
 }

 # ahahaha
@@ -575,11 +576,9 @@ def initConfig_entropyConstants(rootdir):
        'keywords': set([etpSys['arch'],"~"+etpSys['arch']]), # default allowed package keywords
        'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability)
        'edbcounter': edbCOUNTER,
-        'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem
+        'filesystemdirs': ['/bin','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem
        'filesystemdirsmask': [
-                                    '/var/cache','/var/db','/var/empty','/var/lib/portage','/var/lib/entropy','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage',
-                                    '/var/lib/scrollkeeper', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d',
-                                    '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage'
+                                    '/var/cache','/var/db','/var/empty','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage', '/var/spool', '/var/lib', '/usr/lib/locale','/lib64/splash/cache'
        ],
        'libtest_blacklist': [
            'www-client/mozilla-firefox-bin',