From 9bf2a61c1a6c1ebcf31bb30d32a2cf0bcfda7b0d Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Fri, 15 Feb 2008 09:57:27 +0000 Subject: [PATCH] - improved orphans search tool in terms of memory consumption and speed git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1216 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 3 +- client/text_query.py | 77 +++++++++++++++++++++++++++-------- libraries/entropy.py | 50 +++++++++++++++++++---- libraries/entropyConstants.py | 9 ++-- 4 files changed, 106 insertions(+), 33 deletions(-) diff --git a/TODO b/TODO index 27d8b3265..9c40e668e 100644 --- a/TODO +++ b/TODO @@ -12,8 +12,9 @@ TODO list: - find a way to better handle real smartapps deps (need split PDEPEND?) Spritz: - - handle multi repos in "available" menu (*) + - handle multi repos in "available" menu (*) (un set keyslotcache) - add masking interface (*) + - add categories view? (*) - add branch switching menu (for upgrades) (*) - wrap root password request (*) - handle queue errors (*) diff --git a/client/text_query.py b/client/text_query.py index 807c41664..27bb52c15 100644 --- a/client/text_query.py +++ b/client/text_query.py @@ -389,21 +389,34 @@ def searchOrphans(): # start to list all files on the system: dirs = etpConst['filesystemdirs'] - foundFiles = set() + filepath = Equo.entropyTools.getRandomTempFile() + tdbconn = Equo.openGenericDatabase(filepath) + tdbconn.initializeDatabase() for xdir in dirs: for currentdir,subdirs,files in os.walk(xdir): + foundFiles = {} for filename in files: # filter python compiled objects? if filename.endswith(".pyo") or filename.endswith(".pyc") or filename == '.keep': continue filename = os.path.join(currentdir,filename) + if filename.endswith(".ph") and \ + (filename.startswith("/usr/lib/perl") or filename.startswith("/usr/lib64/perl")): + continue mask = [x for x in etpConst['filesystemdirsmask'] if filename.startswith(x)] if not mask: - if (not etpUi['quiet']): + if not etpUi['quiet']: print_info(red(" @@ ")+blue("Looking: ")+bold(filename[:50]+"..."), back = True) - foundFiles.add(filename) - totalfiles = len(foundFiles) - if (not etpUi['quiet']): + foundFiles[filename] = "obj" + if foundFiles: + tdbconn.insertContent(1,foundFiles) + + tdbconn.commitChanges() + tdbconn.createContentIndex() + tdbconn.cursor.execute('select count(file) from content') + totalfiles = tdbconn.cursor.fetchone()[0] + + if not etpUi['quiet']: print_info(red(" @@ ")+blue("Analyzed directories: ")+' '.join(etpConst['filesystemdirs'])) print_info(red(" @@ ")+blue("Masked directories: ")+' '.join(etpConst['filesystemdirsmask'])) print_info(red(" @@ ")+blue("Number of files collected on the filesystem: ")+bold(str(totalfiles))) @@ -415,37 +428,65 @@ def searchOrphans(): length = str(len(idpackages)) count = 0 for idpackage in idpackages: - if (not etpUi['quiet']): + if not etpUi['quiet']: count += 1 atom = clientDbconn.retrieveAtom(idpackage) txt = "["+str(count)+"/"+length+"] " print_info(red(" @@ ")+blue("Intersecting content of package: ")+txt+bold(atom), back = True) + content = set() for x in clientDbconn.retrieveContent(idpackage): if x.startswith("/usr/lib64"): x = "/usr/lib"+x[len("/usr/lib64"):] content.add(x) # remove from foundFiles - foundFiles -= content + for item in content: + tdbconn.cursor.execute('delete from content where file = (?)', (item,)) + + tdbconn.commitChanges() + tdbconn.cursor.execute('select count(file) from content') + orpanedfiles = tdbconn.cursor.fetchone()[0] + if (not etpUi['quiet']): print_info(red(" @@ ")+blue("Intersection completed. Showing statistics: ")) print_info(red(" @@ ")+blue("Number of total files: ")+bold(str(totalfiles))) - print_info(red(" @@ ")+blue("Number of matching files: ")+bold(str(totalfiles - len(foundFiles)))) - print_info(red(" @@ ")+blue("Number of orphaned files: ")+bold(str(len(foundFiles)))) + print_info(red(" @@ ")+blue("Number of matching files: ")+bold(str(totalfiles - orpanedfiles))) + print_info(red(" @@ ")+blue("Number of orphaned files: ")+bold(str(orpanedfiles))) - # order - foundFiles = list(foundFiles) - foundFiles.sort() + tdbconn.cursor.execute('select file from content order by file desc') if not etpUi['quiet']: - print_info(red(" @@ ")+blue("Writing file to disk: ")+bold("/tmp/equo-orphans.txt")) f = open("/tmp/equo-orphans.txt","w") - for x in foundFiles: - f.write(x+"\n") + print_info(red(" @@ ")+blue("Writing file to disk: ")+bold("/tmp/equo-orphans.txt")) + + tdbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape") + myfile = tdbconn.cursor.fetchone() + + sizecount = 0 + while myfile: + myfile = str(myfile[0].encode('raw_unicode_escape')) + try: + sizecount += os.stat(myfile)[6] + except OSError: + pass + if not etpUi['quiet']: + f.write(myfile+"\n") + else: + print myfile + myfile = tdbconn.cursor.fetchone() + + humansize = Equo.entropyTools.bytesIntoHuman(sizecount) + if not etpUi['quiet']: + print_info(red(" @@ ")+blue("Total wasted space: ")+bold(humansize)) f.flush() f.close() - return 0 else: - for x in foundFiles: - print x + print humansize + + tdbconn.closeDB() + try: + os.remove(filepath) + except OSError: + pass + return 0 diff --git a/libraries/entropy.py b/libraries/entropy.py index b78c589ef..04e186771 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -96,6 +96,9 @@ class EquoInterface(TextInterface): # masking parser self.MaskingParser = self.PackageMaskingParserInterfaceLoader() + self.validate_repositories_cache() + + # now if we are on live, we should disable it # are we running on a livecd? (/proc/cmdline has "cdroot") if self.entropyTools.islive(): self.xcache = False @@ -108,6 +111,32 @@ class EquoInterface(TextInterface): except: pass + def validate_repositories_cache(self): + # is the list of repos changed? + cached = self.dumpTools.loadobj(etpCache['repolist']) + if cached == None: + # invalidate matching cache + try: + self.repository_move_clear_cache() + except IOError: + pass + elif type(cached) is tuple: + # compare + myrepolist = tuple(etpRepositoriesOrder) + if cached != myrepolist: + cached = set(cached) + myrepolist = set(myrepolist) + difflist = cached - myrepolist # before minus now + for repoid in difflist: + try: + self.repository_move_clear_cache(repoid) + except IOError: + pass + try: + self.dumpTools.dumpobj(etpCache['repolist'],tuple(etpRepositoriesOrder)) + except IOError: + pass + def switchChroot(self, chroot = ""): # clean caches self.purge_cache() @@ -884,15 +913,16 @@ class EquoInterface(TextInterface): return dbpkginfo - def repository_move_clear_cache(self, repoid): + def repository_move_clear_cache(self, repoid = None): self.clear_dump_cache(etpCache['world_available']) self.clear_dump_cache(etpCache['world_update']) self.clear_dump_cache(etpCache['check_package_update']) self.clear_dump_cache(etpCache['filter_satisfied_deps']) self.clear_dump_cache(etpCache['atomMatch']) - self.clear_dump_cache(etpCache['dbMatch']+repoid+"/") - self.clear_dump_cache(etpCache['dbInfo']+"/"+repoid+"/") - self.clear_dump_cache(etpCache['dbSearch']+repoid+"/") + if repoid != None: + self.clear_dump_cache(etpCache['dbMatch']+"/"+repoid+"/") + self.clear_dump_cache(etpCache['dbInfo']+"/"+repoid+"/") + self.clear_dump_cache(etpCache['dbSearch']+"/"+repoid+"/") def addRepository(self, repodata): @@ -6445,11 +6475,13 @@ class SecurityInterface: self.security_package_checksum = self.security_package+etpConst['packageshashfileext'] - if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']): - os.remove(etpConst['securitydir']) - if not os.path.isdir(etpConst['securitydir']): - os.makedirs(etpConst['securitydir']) - elif os.path.isfile(self.old_download_package_checksum): + if etpConst['uid'] == 0: + if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']): + os.remove(etpConst['securitydir']) + if not os.path.isdir(etpConst['securitydir']): + os.makedirs(etpConst['securitydir']) + + if os.path.isfile(self.old_download_package_checksum): f = open(self.old_download_package_checksum) try: self.previous_checksum = f.readline().strip().split()[0] diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 4cb87dc57..9c9d3dad2 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -358,7 +358,8 @@ etpCache = { 'dep_tree': 'deptree/dep_tree_', 'depends_tree': 'depends/depends_tree_', 'filter_satisfied_deps': 'depfilter/filter_satisfied_deps_', - 'library_breakage': 'libs_break/library_breakage_' + 'library_breakage': 'libs_break/library_breakage_', + 'repolist': 'repos/repolist' } # ahahaha @@ -575,11 +576,9 @@ def initConfig_entropyConstants(rootdir): 'keywords': set([etpSys['arch'],"~"+etpSys['arch']]), # default allowed package keywords 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) 'edbcounter': edbCOUNTER, - 'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem + 'filesystemdirs': ['/bin','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem 'filesystemdirsmask': [ - '/var/cache','/var/db','/var/empty','/var/lib/portage','/var/lib/entropy','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage', - '/var/lib/scrollkeeper', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', - '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' + '/var/cache','/var/db','/var/empty','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage', '/var/spool', '/var/lib', '/usr/lib/locale','/lib64/splash/cache' ], 'libtest_blacklist': [ 'www-client/mozilla-firefox-bin',