From 3a643f70d8a45db3eb63244002abddd78a975ed6 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Mon, 19 Nov 2007 04:07:28 +0000 Subject: [PATCH] - fixed a lot of memory leaks - updated some functions inside triggerTools - improved speed of content table by indexing the content - added --resume support for remove,install,world tools - removed memory hog contentCache (set()) - some misc fixes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@728 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 8 +- client/equo | 3 + client/equoTools.py | 131 +--- client/triggerTools.py | 6 +- client/uiTools.py | 1222 ++++++++++++++++++--------------- libraries/databaseTools.py | 27 +- libraries/dumpTools.py | 54 +- libraries/entropyConstants.py | 12 +- libraries/entropyTools.py | 70 +- 9 files changed, 830 insertions(+), 703 deletions(-) diff --git a/TODO b/TODO index ab9a76385..ec700d4ca 100644 --- a/TODO +++ b/TODO @@ -1,12 +1,14 @@ TODO list: CLIENT: - - give removal depends calculation a bit of love + - better implement --deep (for install) + - implement --skipfirst for world and install/remove + - keep outdated packages on the server for a week - find a way to better handle real smartapps deps - add support for database revisions log - add external trigger for splashutils - env-update source /etc/profile after install? - - implement --resume and --skipfirst for world and install - - keep outdated packages on the server for a week + + Project Status: diff --git a/client/equo b/client/equo index 6885a0d75..130675a92 100644 --- a/client/equo +++ b/client/equo @@ -51,6 +51,7 @@ def print_help(): print_info(" \t\t"+red("--verbose")+"\t\t show more details about what's going on") print_info(" \t\t"+red("--replay")+"\t\t reinstall all the packages and their dependencies") print_info(" \t\t"+red("--empty")+"\t\t\t same as --replay") + print_info(" \t\t"+red("--resume")+"\t\t resume previously interrupted operations") print_info(" \t\t"+red("--upgrade")+"\t\t upgrade "+etpConst['systemname']+" to the specified release (3.5,3.6...)") print_info(" \t"+blue("install")+brown("\t\t install one or more packages or .tbz2")) @@ -58,6 +59,7 @@ def print_help(): print_info(" \t\t"+red("--pretend")+"\t\t just show what would be done") print_info(" \t\t"+red("--fetch")+"\t\t\t just download packages without doing the install") print_info(" \t\t"+red("--nodeps")+"\t\t do not manage any dependency") + print_info(" \t\t"+red("--resume")+"\t\t resume previously interrupted operations") print_info(" \t\t"+red("--empty")+"\t\t\t set all dependencies as unsatisfied") print_info(" \t\t"+red("--deep")+"\t\t\t analyze dependencies deeply") print_info(" \t\t"+red("--verbose")+"\t\t show more details about what's going on") @@ -66,6 +68,7 @@ def print_help(): print_info(" \t"+blue("remove")+brown("\t\t remove one or more packages")) print_info(" \t\t"+red("--deep")+"\t\t\t also pull unused dependencies where depends list is empty") print_info(" \t\t"+red("--configfiles")+"\t\t also remove configuration files") + print_info(" \t\t"+red("--resume")+"\t\t resume previously interrupted operations") print_info(" \t"+blue("deptest")+brown("\t\t look for unsatisfied dependencies")) print_info(" \t\t"+red("--quiet")+"\t\t\t show less details (useful for scripting)") print_info(" \t\t"+red("--ask")+"\t\t\t ask before making any changes") diff --git a/client/equoTools.py b/client/equoTools.py index f7efbcdad..2c25d42e3 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -21,17 +21,13 @@ ''' from sys import path, getfilesystemencoding -path.append('../libraries') -path.append('../client') -path.append('/usr/lib/entropy/libraries') import os import shutil -import stat from entropyConstants import * from clientConstants import * from outputTools import * from remoteTools import downloadData -from entropyTools import compareMd5, bytesIntoHuman, askquestion, getRandomNumber, dep_getkey, entropyCompareVersions, filterDuplicatedEntries, extractDuplicatedEntries, uncompressTarBz2, extractXpak, applicationLockCheck, countdown, isRoot, spliturl, remove_tag, dep_striptag, md5sum, allocateMaskedFile, istextfile, isnumber, extractEdb, getNewerVersion, getNewerVersionTag, unpackXpak, lifobuffer, writeNewBranch +from entropyTools import compareMd5, bytesIntoHuman, askquestion, getRandomNumber, dep_getkey, entropyCompareVersions, filterDuplicatedEntries, extractDuplicatedEntries, uncompressTarBz2, extractXpak, applicationLockCheck, countdown, isRoot, spliturl, remove_tag, dep_striptag, md5sum, allocateMaskedFile, istextfile, isnumber, extractEdb, getNewerVersion, getNewerVersionTag, unpackXpak, lifobuffer from databaseTools import openRepositoryDatabase, openClientDatabase, openGenericDatabase, fetchRepositoryIfNotAvailable, listAllAvailableBranches import triggerTools import confTools @@ -896,6 +892,7 @@ def removePackage(infoDict): atom = infoDict['removeatom'] content = infoDict['removecontent'] removeidpackage = infoDict['removeidpackage'] + clientDbconn = openClientDatabase() equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: "+str(atom)) @@ -903,15 +900,6 @@ def removePackage(infoDict): generateDependsTreeCache.clear() dumpTools.dumpobj(etpCache['generateDependsTree'],generateDependsTreeCache) - # load content cache if found empty - if etpConst['collisionprotect'] > 0: - if (not contentCache): - clientDbconn = openClientDatabase() - xlist = clientDbconn.listAllFiles(clean = True) - for x in xlist: - contentCache[x] = 1 - clientDbconn.closeDB() - # remove from database if removeidpackage != -1: print_info(red(" ## ")+blue("Removing from database: ")+red(infoDict['removeatom'])) @@ -927,20 +915,17 @@ def removePackage(infoDict): protect = etpConst['dbconfigprotect'] mask = etpConst['dbconfigprotectmask'] + # remove files from system directories = set() for file in content: # collision check if etpConst['collisionprotect'] > 0: - if file in contentCache: + + if clientDbconn.isFileAvailable(file) and os.path.isfile(file): # in this way we filter out directories print_warning(darkred(" ## ")+red("Collision found during remove for ")+file+red(" - cannot overwrite")) equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Collision found during remove for "+file+" - cannot overwrite") - content.remove(file) continue - try: - del contentCache[file] - except: - pass protected = False if (not infoDict['removeconfig']) and (not infoDict['diffremoval']): @@ -965,7 +950,7 @@ def removePackage(infoDict): pass # some filenames are buggy encoded if (protected): - equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"[remove] Protecting config file: "+file) + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"[remove] Protecting config file: "+file) print_warning(darkred(" ## ")+red("[remove] Protecting config file: ")+file) else: try: @@ -1029,6 +1014,7 @@ def removePackage(infoDict): if not taint: break + clientDbconn.closeDB() return 0 @@ -1038,7 +1024,7 @@ def removePackage(infoDict): @output: 0 = all fine, >0 = error! ''' def installPackage(infoDict): - + clientDbconn = openClientDatabase() package = infoDict['download'] @@ -1048,13 +1034,6 @@ def installPackage(infoDict): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing package: "+str(infoDict['atom'])) - # load content cache if found empty - if etpConst['collisionprotect'] > 0: - if (not contentCache): - xlist = clientDbconn.listAllFiles(clean = True) - for x in xlist: - contentCache[x] = 1 - # unpack and install if infoDict['repository'].endswith(".tbz2"): pkgpath = etpRepositories[infoDict['repository']]['pkgpath'] @@ -1065,17 +1044,17 @@ def installPackage(infoDict): shutil.rmtree(unpackDir) imageDir = unpackDir+"/image" os.makedirs(imageDir) - + rc = uncompressTarBz2(pkgpath,imageDir) if (rc != 0): return rc if not os.path.isdir(imageDir): return 2 - + # load CONFIG_PROTECT and its mask protect = etpRepositories[infoDict['repository']]['configprotect'] mask = etpRepositories[infoDict['repository']]['configprotectmask'] - + packageContent = [] # setup imageDir properly imageDir = imageDir.encode(getfilesystemencoding()) @@ -1121,9 +1100,9 @@ def installPackage(infoDict): tofile = fromfile[len(imageDir):] if etpConst['collisionprotect'] > 1: - if tofile in contentCache: - print_warning(darkred(" ## ")+red("Collision found during install for ")+file.encode(getfilesystemencoding())+" - cannot overwrite") - equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! Collision found during install for "+file.encode(getfilesystemencoding())+" - cannot overwrite") + if clientDbconn.isFileAvailable(tofile): + print_warning(darkred(" ## ")+red("Collision found during install for ")+tofile+" - cannot overwrite") + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! Collision found during install for "+tofile+" - cannot overwrite") equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[collision] Protecting config file: "+tofile) print_warning(darkred(" ## ")+red("[collision] Protecting config file: ")+tofile) continue @@ -1142,15 +1121,7 @@ def installPackage(infoDict): protected = False break - if os.access(tofile,os.F_OK): - try: - if not protected: os.remove(tofile) - except: - if not protected: - rc = os.system("rm -f "+tofile) - if (rc != 0): - return 3 - else: + if not os.path.lexists(tofile): protected = False # file doesn't exist # check if it's a text file @@ -1159,21 +1130,14 @@ def installPackage(infoDict): else: protected = False # it's not a file - # check md5 - if (protected) and os.path.isfile(tofile) and os.path.isfile(fromfile): - mymd5 = md5sum(fromfile) - sysmd5 = md5sum(tofile) - if mymd5 == sysmd5: - protected = False # files are the same - else: - protected = False # a broken symlink inside our image dir - # request new tofile then if (protected): - equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Protecting config file: "+tofile) - print_warning(darkred(" ## ")+red("Protecting config file: ")+tofile) - tofile = allocateMaskedFile(tofile) - + tofile, prot_status = allocateMaskedFile(tofile, fromfile) + if not prot_status: + protected = False + else: + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Protecting config file: "+tofile) + print_warning(darkred(" ## ")+red("Protecting config file: ")+tofile) # -- CONFIGURATION FILE PROTECTION -- @@ -1223,17 +1187,14 @@ def installPackage(infoDict): clientDbconn.closeDB() - + rc = 0 if (etpConst['gentoo-compat']): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing new Gentoo database entry: "+str(infoDict['atom'])) rc = installPackageIntoGentooDatabase(infoDict,pkgpath, newidpackage = newidpackage) - if (rc >= 0): - shutil.rmtree(unpackDir,True) - return rc # remove unpack dir shutil.rmtree(unpackDir,True) - return 0 + return rc ''' @description: remove package entry from Gentoo database @@ -1397,41 +1358,18 @@ def installPackageIntoDatabase(idpackage, repository): # fetch info dbconn = openRepositoryDatabase(repository) data = dbconn.getPackageData(idpackage) - # get current revision - rev = dbconn.retrieveRevision(idpackage) - branch = dbconn.retrieveBranch(idpackage) - newcontent = dbconn.retrieveContent(idpackage) - + dbconn.closeDB() + # open client db clientDbconn = openClientDatabase() - # load content cache - if etpConst['collisionprotect'] > 0: - if (not contentCache): - xlist = clientDbconn.listAllFiles(clean = True) - for x in xlist: - contentCache[x] = 1 - - # sync contentCache before install - for x in newcontent: - contentCache[x] = 1 - - dbconn.closeDB() - - idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = rev) + idpk, rev, x, status = clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision']) del x + if (not status): - clientDbconn.closeDB() print "DEBUG!!! THIS SHOULD NOT NEVER HAPPEN. Package "+str(idpk)+" has not been inserted, status: "+str(status) idpk = -1 # it hasn't been insterted ? why?? else: # all fine - # regenerate contentCache - if etpConst['collisionprotect'] > 0: - xlist = clientDbconn.listAllFiles(clean = True) - contentCache.clear() - for x in xlist: - contentCache[x] = 1 - # add idpk to the installedtable clientDbconn.removePackageFromInstalledTable(idpk) clientDbconn.addPackageToInstalledTable(idpk,repository) @@ -1461,22 +1399,7 @@ def installPackageIntoDatabase(idpackage, repository): def removePackageFromDatabase(idpackage): clientDbconn = openClientDatabase() - # load content cache - if etpConst['collisionprotect'] > 0: - if (not contentCache): - xlist = clientDbconn.listAllFiles(clean = True) - for x in xlist: - contentCache[x] = 1 - # sync contentCache before removal - content = clientDbconn.retrieveContent(idpackage) - for x in content: - try: - del contentCache[x] - except: - pass - clientDbconn.removePackage(idpackage) - clientDbconn.closeDB() return 0 diff --git a/client/triggerTools.py b/client/triggerTools.py index 168883610..ab22a5ea8 100644 --- a/client/triggerTools.py +++ b/client/triggerTools.py @@ -125,7 +125,7 @@ def postinstall(pkgdata): functions.add('createkernelsym') for path in ldpaths: if x.startswith(path) and (x.find(".so") != -1): - functions.add('ldconfig') + functions.add('run_ldconfig') #if x.startswith("/etc/init.d/"): do it externally # functions.add('initadd') @@ -213,7 +213,7 @@ def postremove(pkgdata): functions.add('cleanpy') for path in ldpaths: if x.startswith(path) and (x.find(".so") != -1): - functions.add('ldconfig') + functions.add('run_ldconfig') return functions @@ -531,7 +531,7 @@ def createkernelsym(pkgdata): os.symlink(todir,"/usr/src/linux") break -def ldconfig(pkgdata): +def run_ldconfig(pkgdata): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Running ldconfig") print_info(red(" ##")+brown(" Regenerating /etc/ld.so.cache")) os.system("ldconfig &> /dev/null") diff --git a/client/uiTools.py b/client/uiTools.py index 189436e93..b76aaf6cf 100644 --- a/client/uiTools.py +++ b/client/uiTools.py @@ -32,6 +32,7 @@ import equoTools import repositoriesTools from databaseTools import openRepositoryDatabase, openClientDatabase, openGenericDatabase, listAllAvailableBranches import entropyTools +import dumpTools import shutil import logTools @@ -56,6 +57,7 @@ def package(options): equoRequestConfigFiles = False equoRequestReplay = False equoRequestUpgrade = False + equoRequestResume = False equoRequestUpgradeTo = '' rc = 0 _myopts = [] @@ -83,6 +85,8 @@ def package(options): equoRequestReplay = True elif (opt == "--upgrade"): equoRequestUpgrade = True + elif (opt == "--resume"): + equoRequestResume = True else: if (equoRequestUpgrade): equoRequestUpgradeTo = opt @@ -97,21 +101,21 @@ def package(options): rc, garbage = dependenciesTest(quiet = equoRequestQuiet, ask = equoRequestAsk, pretend = equoRequestPretend) elif (options[0] == "install"): - if (myopts) or (mytbz2paths): + if (myopts) or (mytbz2paths) or (equoRequestResume): equoTools.loadCaches() - rc, status = installPackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, deps = equoRequestDeps, emptydeps = equoRequestEmptyDeps, onlyfetch = equoRequestOnlyFetch, deepdeps = equoRequestDeep, configFiles = equoRequestConfigFiles, tbz2 = mytbz2paths) + rc, status = installPackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, deps = equoRequestDeps, emptydeps = equoRequestEmptyDeps, onlyfetch = equoRequestOnlyFetch, deepdeps = equoRequestDeep, configFiles = equoRequestConfigFiles, tbz2 = mytbz2paths, resume = equoRequestResume) else: print_error(red(" Nothing to do.")) rc = 127 elif (options[0] == "world"): equoTools.loadCaches() - rc, status = worldUpdate(ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, onlyfetch = equoRequestOnlyFetch, replay = (equoRequestReplay or equoRequestEmptyDeps), upgradeTo = equoRequestUpgradeTo) + rc, status = worldUpdate(ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, onlyfetch = equoRequestOnlyFetch, replay = (equoRequestReplay or equoRequestEmptyDeps), upgradeTo = equoRequestUpgradeTo, resume = equoRequestResume) elif (options[0] == "remove"): - if len(myopts) > 0: + if myopts or equoRequestResume: equoTools.loadCaches() - rc, status = removePackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, deps = equoRequestDeps, deep = equoRequestDeep, configFiles = equoRequestConfigFiles) + rc, status = removePackages(myopts, ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, deps = equoRequestDeps, deep = equoRequestDeep, configFiles = equoRequestConfigFiles, resume = equoRequestResume) else: print_error(red(" Nothing to do.")) rc = 127 @@ -121,105 +125,140 @@ def package(options): return rc -def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False, replay = False, upgradeTo = ''): +def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False, replay = False, upgradeTo = '', resume = False): # check if I am root if (not entropyTools.isRoot()): print_warning(red("Running with ")+bold("--pretend")+red("...")) pretend = True - if not pretend: - repositoriesTools.syncRepositories() + if not resume: - # verify selected release (branch) - if (upgradeTo): - availbranches = listAllAvailableBranches() - if upgradeTo not in availbranches: - print_error(red("Selected release: ")+bold(upgradeTo)+red(" is not available.")) - return 1,-2 + if not pretend: + repositoriesTools.syncRepositories() + + # verify selected release (branch) + if (upgradeTo): + availbranches = listAllAvailableBranches() + if upgradeTo not in availbranches: + print_error(red("Selected release: ")+bold(upgradeTo)+red(" is not available.")) + return 1,-2 + else: + branches = (upgradeTo,) else: - branches = (upgradeTo,) - else: - branches = (etpConst['branch'],) - - if (not pretend) and (upgradeTo): - # update configuration - entropyTools.writeNewBranch(upgradeTo) - - updateList = set() - fineList = set() - removedList = set() - clientDbconn = openClientDatabase() - # get all the installed packages - packages = clientDbconn.listAllPackages() - - print_info(red(" @@ ")+blue("Calculating world packages...")) - for package in packages: - tainted = False - atom = package[0] - idpackage = package[1] - name = clientDbconn.retrieveName(idpackage) - category = clientDbconn.retrieveCategory(idpackage) - revision = clientDbconn.retrieveRevision(idpackage) - slot = clientDbconn.retrieveSlot(idpackage) - atomkey = category+"/"+name - # search in the packages - match = equoTools.atomMatch(atom) - if match[0] == -1: # atom has been changed, or removed? - tainted = True - else: # not changed, is the revision changed? - adbconn = openRepositoryDatabase(match[1]) - arevision = adbconn.retrieveRevision(match[0]) - # if revision is 9999, then any revision is fine - if revision == 9999: arevision = 9999 - adbconn.closeDB() - if revision != arevision: - tainted = True - elif (replay): - tainted = True - if (tainted): - # Alice! use the key! ... and the slot - matchresults = equoTools.atomMatch(atomkey, matchSlot = slot, matchBranches = branches) - if matchresults[0] != -1: - mdbconn = openRepositoryDatabase(matchresults[1]) - matchatom = mdbconn.retrieveAtom(matchresults[0]) - mdbconn.closeDB() - updateList.add((matchatom,matchresults)) - else: - removedList.add(idpackage) - # look for packages that would match key with any slot (for eg, gcc updates), slot changes handling - matchresults = equoTools.atomMatch(atomkey, matchBranches = branches) + branches = (etpConst['branch'],) + + if (not pretend) and (upgradeTo): + # update configuration + entropyTools.writeNewBranch(upgradeTo) + + updateList = set() + fineList = set() + removedList = set() + clientDbconn = openClientDatabase() + # get all the installed packages + packages = clientDbconn.listAllPackages() + + print_info(red(" @@ ")+blue("Calculating world packages...")) + for package in packages: + tainted = False + atom = package[0] + idpackage = package[1] + name = clientDbconn.retrieveName(idpackage) + category = clientDbconn.retrieveCategory(idpackage) + revision = clientDbconn.retrieveRevision(idpackage) + slot = clientDbconn.retrieveSlot(idpackage) + atomkey = category+"/"+name + # search in the packages + match = equoTools.atomMatch(atom) + if match[0] == -1: # atom has been changed, or removed? + tainted = True + else: # not changed, is the revision changed? + adbconn = openRepositoryDatabase(match[1]) + arevision = adbconn.retrieveRevision(match[0]) + # if revision is 9999, then any revision is fine + if revision == 9999: arevision = 9999 + adbconn.closeDB() + if revision != arevision: + tainted = True + elif (replay): + tainted = True + if (tainted): + # Alice! use the key! ... and the slot + matchresults = equoTools.atomMatch(atomkey, matchSlot = slot, matchBranches = branches) if matchresults[0] != -1: mdbconn = openRepositoryDatabase(matchresults[1]) matchatom = mdbconn.retrieveAtom(matchresults[0]) mdbconn.closeDB() - # compare versions - unsatisfied, satisfied = equoTools.filterSatisfiedDependencies((matchatom,)) - if unsatisfied: - updateList.add((matchatom,matchresults)) - else: - fineList.add(idpackage) + updateList.add((matchatom,matchresults)) + else: + removedList.add(idpackage) + # look for packages that would match key with any slot (for eg, gcc updates), slot changes handling + matchresults = equoTools.atomMatch(atomkey, matchBranches = branches) + if matchresults[0] != -1: + mdbconn = openRepositoryDatabase(matchresults[1]) + matchatom = mdbconn.retrieveAtom(matchresults[0]) + mdbconn.closeDB() + # compare versions + unsatisfied, satisfied = equoTools.filterSatisfiedDependencies((matchatom,)) + if unsatisfied: + updateList.add((matchatom,matchresults)) + else: + fineList.add(idpackage) + + if (verbose or pretend): + print_info(red(" @@ ")+darkgreen("Packages matching update:\t\t")+bold(str(len(updateList)))) + print_info(red(" @@ ")+darkred("Packages matching not available:\t\t")+bold(str(len(removedList)))) + print_info(red(" @@ ")+blue("Packages matching already up to date:\t")+bold(str(len(fineList)))) + + # clear old resume information + dumpTools.dumpobj(etpCache['world'],{}) + dumpTools.dumpobj(etpCache['install'],{}) + dumpTools.dumpobj(etpCache['remove'],[]) + if (not pretend): + # store resume information + resume_cache = {} + resume_cache['ask'] = ask + resume_cache['verbose'] = verbose + resume_cache['onlyfetch'] = onlyfetch + resume_cache['removedList'] = removedList + dumpTools.dumpobj(etpCache['world'],resume_cache) - if (verbose or pretend): - print_info(red(" @@ ")+darkgreen("Packages matching update:\t\t")+bold(str(len(updateList)))) - print_info(red(" @@ ")+darkred("Packages matching not available:\t\t")+bold(str(len(removedList)))) - print_info(red(" @@ ")+blue("Packages matching already up to date:\t")+bold(str(len(fineList)))) + else: # if resume, load cache if possible + + # check if there's something to resume + resume_cache = dumpTools.loadobj(etpCache['world']) + if not resume_cache: # None or {} + print_error(red("Nothing to resume.")) + return 128,-1 + else: + try: + updateList = [] + removedList = resume_cache['removedList'].copy() + ask = resume_cache['ask'] + verbose = resume_cache['verbose'] + onlyfetch = resume_cache['onlyfetch'] + # save removal stuff into etpCache['remove'] + dumpTools.dumpobj(etpCache['remove'],list(removedList)) + except: + print_error(red("Resume cache corrupted.")) + dumpTools.dumpobj(etpCache['world'],{}) + dumpTools.dumpobj(etpCache['install'],{}) + dumpTools.dumpobj(etpCache['remove'],[]) + return 128,-1 # disable collisions protection, better oldcollprotect = etpConst['collisionprotect'] etpConst['collisionprotect'] = 0 - if (updateList): - rc = installPackages(atomsdata = updateList, ask = ask, pretend = pretend, verbose = verbose, onlyfetch = onlyfetch) + if (updateList) or (resume): + rc = installPackages(atomsdata = updateList, ask = ask, pretend = pretend, verbose = verbose, onlyfetch = onlyfetch, resume = resume) if rc[0] != 0: return rc else: print_info(red(" @@ ")+blue("Nothing to update.")) - - etpConst['collisionprotect'] = oldcollprotect - if (removedList): removedList = list(removedList) removedList.sort() @@ -239,7 +278,7 @@ def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False time.sleep(5) # run removePackages with --nodeps - removePackages(atomsdata = removedList, ask = ask, verbose = verbose, deps = False, systemPackagesCheck = False, configFiles = True) + removePackages(atomsdata = removedList, ask = ask, verbose = verbose, deps = False, systemPackagesCheck = False, configFiles = True, resume = resume) else: print_info(red(" @@ ")+blue("Calculation complete.")) @@ -249,371 +288,398 @@ def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False clientDbconn.closeDB() return 0,0 -def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, verbose = False, deps = True, emptydeps = False, onlyfetch = False, deepdeps = False, configFiles = False, tbz2 = []): +def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, verbose = False, deps = True, emptydeps = False, onlyfetch = False, deepdeps = False, configFiles = False, tbz2 = [], resume = False): # check if I am root if (not entropyTools.isRoot()): print_warning(red("Running with ")+bold("--pretend")+red("...")) pretend = True - + dirsCleanup = set() def dirscleanup(): - for x in dirsCleanup: - try: - if os.path.isdir(x): shutil.rmtree(x) - except: - pass - if (atomsdata): - foundAtoms = atomsdata - else: - foundAtoms = [] - for package in packages: - foundAtoms.append([package,equoTools.atomMatch(package)]) - if tbz2: - for pkg in tbz2: - # create a repository for each database - basefile = os.path.basename(pkg) - if os.path.isdir(etpConst['entropyunpackdir']+"/"+basefile[:-5]): - shutil.rmtree(etpConst['entropyunpackdir']+"/"+basefile[:-5]) - os.makedirs(etpConst['entropyunpackdir']+"/"+basefile[:-5]) - dbfile = entropyTools.extractEdb(pkg,dbpath = etpConst['entropyunpackdir']+"/"+basefile[:-5]+"/packages.db") - if dbfile == None: - print_warning(red("## ATTENTION:")+bold(" "+basefile+" ")+red(" is not a valid Entropy package. Skipping...")) - continue - dirsCleanup.add(os.path.dirname(dbfile)) - # add dbfile - etpRepositories[basefile] = {} - etpRepositories[basefile]['description'] = "Dynamic database from "+basefile - etpRepositories[basefile]['packages'] = [] - etpRepositories[basefile]['dbpath'] = os.path.dirname(dbfile) - etpRepositories[basefile]['pkgpath'] = os.path.realpath(pkg) # extra info added - etpRepositories[basefile]['configprotect'] = set() - etpRepositories[basefile]['configprotectmask'] = set() - etpRepositories[basefile]['smartpackage'] = False # extra info added - # get max count - repoordercount = [x[0] for x in etpRepositoriesOrder] - repoordercount.sort() - etpRepositoriesOrder.add((repoordercount[-1]+1,basefile)) - mydbconn = openGenericDatabase(dbfile) - # read all idpackages - myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files - if len(myidpackages) > 1: - etpRepositories[basefile]['smartpackage'] = True - for myidpackage in myidpackages: - foundAtoms.append([pkg,(int(myidpackage),basefile)]) - mydbconn.closeDB() + for x in dirsCleanup: + try: + if os.path.isdir(x): shutil.rmtree(x) + except: + pass - # filter packages not found - _foundAtoms = [] - for result in foundAtoms: - exitcode = result[1][0] - if (exitcode != -1): - _foundAtoms.append(result[1]) - else: - print_warning(red("## ATTENTION: no match for ")+bold(" "+result[0]+" ")+red(" in database. If you omitted the category, try adding it.")) + if not resume: + + if (atomsdata): + foundAtoms = atomsdata + else: + foundAtoms = [] + for package in packages: + foundAtoms.append([package,equoTools.atomMatch(package)]) + if tbz2: + for pkg in tbz2: + # create a repository for each database + basefile = os.path.basename(pkg) + if os.path.isdir(etpConst['entropyunpackdir']+"/"+basefile[:-5]): + shutil.rmtree(etpConst['entropyunpackdir']+"/"+basefile[:-5]) + os.makedirs(etpConst['entropyunpackdir']+"/"+basefile[:-5]) + dbfile = entropyTools.extractEdb(pkg,dbpath = etpConst['entropyunpackdir']+"/"+basefile[:-5]+"/packages.db") + if dbfile == None: + print_warning(red("## ATTENTION:")+bold(" "+basefile+" ")+red(" is not a valid Entropy package. Skipping...")) + continue + dirsCleanup.add(os.path.dirname(dbfile)) + # add dbfile + etpRepositories[basefile] = {} + etpRepositories[basefile]['description'] = "Dynamic database from "+basefile + etpRepositories[basefile]['packages'] = [] + etpRepositories[basefile]['dbpath'] = os.path.dirname(dbfile) + etpRepositories[basefile]['pkgpath'] = os.path.realpath(pkg) # extra info added + etpRepositories[basefile]['configprotect'] = set() + etpRepositories[basefile]['configprotectmask'] = set() + etpRepositories[basefile]['smartpackage'] = False # extra info added + # get max count + repoordercount = [x[0] for x in etpRepositoriesOrder] + repoordercount.sort() + etpRepositoriesOrder.add((repoordercount[-1]+1,basefile)) + mydbconn = openGenericDatabase(dbfile) + # read all idpackages + myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files + if len(myidpackages) > 1: + etpRepositories[basefile]['smartpackage'] = True + for myidpackage in myidpackages: + foundAtoms.append([pkg,(int(myidpackage),basefile)]) + mydbconn.closeDB() - foundAtoms = _foundAtoms + # filter packages not found + _foundAtoms = [] + for result in foundAtoms: + exitcode = result[1][0] + if (exitcode != -1): + _foundAtoms.append(result[1]) + else: + print_warning(red("## ATTENTION: no match for ")+bold(" "+result[0]+" ")+red(" in database. If you omitted the category, try adding it.")) + + foundAtoms = _foundAtoms + + # are there packages in foundAtoms? + if (not foundAtoms): + print_error(red("No packages found")) + dirscleanup() + return 127,-1 + + if (ask or pretend or verbose): + # now print the selected packages + print_info(red(" @@ ")+blue("These are the chosen packages:")) + totalatoms = len(foundAtoms) + atomscounter = 0 + for atomInfo in foundAtoms: + atomscounter += 1 + idpackage = atomInfo[0] + reponame = atomInfo[1] + # open database + dbconn = openRepositoryDatabase(reponame) - # are packages in foundAtoms? - if (not foundAtoms): - print_error(red("No packages found")) - dirscleanup() - return 127,-1 - - if (ask or pretend or verbose): - # now print the selected packages - print_info(red(" @@ ")+blue("These are the chosen packages:")) - totalatoms = len(foundAtoms) - atomscounter = 0 - for atomInfo in foundAtoms: - atomscounter += 1 - idpackage = atomInfo[0] - reponame = atomInfo[1] - # open database - dbconn = openRepositoryDatabase(reponame) - - # get needed info - pkgatom = dbconn.retrieveAtom(idpackage) - pkgver = dbconn.retrieveVersion(idpackage) - pkgtag = dbconn.retrieveVersionTag(idpackage) - if not pkgtag: - pkgtag = "NoTag" - pkgrev = dbconn.retrieveRevision(idpackage) - pkgslot = dbconn.retrieveSlot(idpackage) - - # client info - installedVer = "Not installed" - installedTag = "NoTag" - installedRev = "NoRev" - clientDbconn = openClientDatabase() - if (clientDbconn != -1): - pkginstalled = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), matchSlot = pkgslot) - if (pkginstalled[1] == 0): - # found - idx = pkginstalled[0] - installedVer = clientDbconn.retrieveVersion(idx) - installedTag = clientDbconn.retrieveVersionTag(idx) - if not installedTag: - installedTag = "NoTag" - installedRev = clientDbconn.retrieveRevision(idx) - clientDbconn.closeDB() - - print_info(" # "+red("(")+bold(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+bold(pkgatom)+" >>> "+red(etpRepositories[reponame]['description'])) - print_info("\t"+red("Versioning:\t")+" "+blue(installedVer)+" / "+blue(installedTag)+" / "+blue(str(installedRev))+bold(" ===> ")+darkgreen(pkgver)+" / "+darkgreen(pkgtag)+" / "+darkgreen(str(pkgrev))) - # tell wether we should update it - if installedVer == "Not installed": - installedVer = "0" - if installedTag == "NoTag": - installedTag == '' - if installedRev == "NoRev": - installedRev == 0 - cmp = entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) - if (cmp == 0): - action = darkgreen("No update needed") - elif (cmp > 0): - if (installedVer == "0"): - action = darkgreen("Install") - else: - action = blue("Upgrade") - else: - action = red("Downgrade") - print_info("\t"+red("Action:\t\t")+" "+action) - - dbconn.closeDB() - - if (verbose or ask or pretend): - print_info(red(" @@ ")+blue("Number of packages: ")+str(totalatoms)) + # get needed info + pkgatom = dbconn.retrieveAtom(idpackage) + pkgver = dbconn.retrieveVersion(idpackage) + pkgtag = dbconn.retrieveVersionTag(idpackage) + if not pkgtag: + pkgtag = "NoTag" + pkgrev = dbconn.retrieveRevision(idpackage) + pkgslot = dbconn.retrieveSlot(idpackage) + + # client info + installedVer = "Not installed" + installedTag = "NoTag" + installedRev = "NoRev" + clientDbconn = openClientDatabase() + if (clientDbconn != -1): + pkginstalled = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), matchSlot = pkgslot) + if (pkginstalled[1] == 0): + # found + idx = pkginstalled[0] + installedVer = clientDbconn.retrieveVersion(idx) + installedTag = clientDbconn.retrieveVersionTag(idx) + if not installedTag: + installedTag = "NoTag" + installedRev = clientDbconn.retrieveRevision(idx) + clientDbconn.closeDB() + + print_info(" # "+red("(")+bold(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+bold(pkgatom)+" >>> "+red(etpRepositories[reponame]['description'])) + print_info("\t"+red("Versioning:\t")+" "+blue(installedVer)+" / "+blue(installedTag)+" / "+blue(str(installedRev))+bold(" ===> ")+darkgreen(pkgver)+" / "+darkgreen(pkgtag)+" / "+darkgreen(str(pkgrev))) + # tell wether we should update it + if installedVer == "Not installed": + installedVer = "0" + if installedTag == "NoTag": + installedTag == '' + if installedRev == "NoRev": + installedRev == 0 + cmp = entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) + if (cmp == 0): + action = darkgreen("No update needed") + elif (cmp > 0): + if (installedVer == "0"): + action = darkgreen("Install") + else: + action = blue("Upgrade") + else: + action = red("Downgrade") + print_info("\t"+red("Action:\t\t")+" "+action) + + dbconn.closeDB() + + if (verbose or ask or pretend): + print_info(red(" @@ ")+blue("Number of packages: ")+str(totalatoms)) + + if (deps): + if (ask): + rc = entropyTools.askquestion(" Would you like to continue with dependencies calculation ?") + if rc == "No": + dirscleanup() + return 0,0 + + runQueue = [] + removalQueue = [] # aka, conflicts + print_info(red(" @@ ")+blue("Calculating dependencies...")) if (deps): - if (ask): - rc = entropyTools.askquestion(" Would you like to continue with dependencies calculation ?") - if rc == "No": - dirscleanup() - return 0,0 - - runQueue = [] - removalQueue = [] # aka, conflicts + treepackages, result = equoTools.getRequiredPackages(foundAtoms, emptydeps, deepdeps, spinning = True) + # add dependencies, explode them + + if (result == -2): + print_error(red(" @@ ")+blue("Cannot find needed dependencies: ")+str(treepackages)) + for atom in treepackages: + for repo in etpRepositories: + rdbconn = openRepositoryDatabase(repo) + riddep = rdbconn.searchDependency(atom) + if riddep != -1: + ridpackages = rdbconn.searchIdpackageFromIddependency(riddep) + if ridpackages: + print_error(red(" @@ ")+blue("Dependency found and probably needed by:")) + for i in ridpackages: + iatom = rdbconn.retrieveAtom(i) + print_error(red(" # ")+" [from:"+repo+"] "+darkred(iatom)) + rdbconn.closeDB() + dirscleanup() + return 130, -1 + + elif (result == -1): # no database connection + print_error(red(" @@ ")+blue("Cannot find the Installed Packages Database. It's needed to accomplish dependency resolving. Try to run ")+bold("equo database generate")) + dirscleanup() + return 200, -1 + + for x in range(len(treepackages)): + if x == 0: + # conflicts + for a in treepackages[x]: + removalQueue.append(a) + else: + for a in treepackages[x]: + runQueue.append(a) + del treepackages + else: + for atomInfo in foundAtoms: + runQueue.append(atomInfo) - print_info(red(" @@ ")+blue("Calculating dependencies...")) - - if (deps): - treepackages, result = equoTools.getRequiredPackages(foundAtoms, emptydeps, deepdeps, spinning = True) - # add dependencies, explode them - - if (result == -2): - print_error(red(" @@ ")+blue("Cannot find needed dependencies: ")+str(treepackages)) - for atom in treepackages: - for repo in etpRepositories: - rdbconn = openRepositoryDatabase(repo) - riddep = rdbconn.searchDependency(atom) - if riddep != -1: - ridpackages = rdbconn.searchIdpackageFromIddependency(riddep) - if ridpackages: - print_error(red(" @@ ")+blue("Dependency found and probably needed by:")) - for i in ridpackages: - iatom = rdbconn.retrieveAtom(i) - print_error(red(" # ")+" [from:"+repo+"] "+darkred(iatom)) - rdbconn.closeDB() + downloadSize = 0 + onDiskUsedSize = 0 + onDiskFreedSize = 0 + pkgsToInstall = 0 + pkgsToUpdate = 0 + pkgsToReinstall = 0 + pkgsToDowngrade = 0 + pkgsToRemove = len(removalQueue) + actionQueue = {} + + if (not runQueue) and (not removalQueue): + print_error(red("Nothing to do.")) dirscleanup() - return 130, -1 - - elif (result == -1): # no database connection - print_error(red(" @@ ")+blue("Cannot find the Installed Packages Database. It's needed to accomplish dependency resolving. Try to run ")+bold("equo database generate")) - dirscleanup() - return 200, -1 - - for x in range(len(treepackages)): - if x == 0: - # conflicts - for a in treepackages[x]: - removalQueue.append(a) - else: - for a in treepackages[x]: - runQueue.append(a) - - else: - for atomInfo in foundAtoms: - runQueue.append(atomInfo) + return 127,-1 + + if (runQueue): + if (ask or pretend): + print_info(red(" @@ ")+blue("These are the packages that would be ")+bold("merged:")) + + if not (ask or pretend): count = 0 + atomlen = len(runQueue) + for packageInfo in runQueue: + if not (ask or pretend): count += 1; print_info(":: Collecting data: "+str(round((float(count)/atomlen)*100,1))+"% ::", back = True) + dbconn = openRepositoryDatabase(packageInfo[1]) + mydata = dbconn.getBaseData(packageInfo[0]) + pkgatom = mydata[0] + pkgver = mydata[2] + pkgtag = mydata[3] + pkgrev = mydata[18] + pkgslot = mydata[14] + pkgfile = mydata[12] + pkgcat = mydata[5] + pkgname = mydata[1] - downloadSize = 0 - onDiskUsedSize = 0 - onDiskFreedSize = 0 - pkgsToInstall = 0 - pkgsToUpdate = 0 - pkgsToReinstall = 0 - pkgsToDowngrade = 0 - pkgsToRemove = len(removalQueue) - actionQueue = {} - - if (not runQueue) and (not removalQueue): - print_error(red("Nothing to do.")) - dirscleanup() - return 127,-1 - - if (runQueue): - if (ask or pretend): - print_info(red(" @@ ")+blue("These are the packages that would be ")+bold("merged:")) - - if not (ask or pretend): count = 0 - atomlen = len(runQueue) - for packageInfo in runQueue: - if not (ask or pretend): count += 1; print_info(":: Collecting data: "+str(round((float(count)/atomlen)*100,1))+"% ::", back = True) - dbconn = openRepositoryDatabase(packageInfo[1]) - mydata = dbconn.getBaseData(packageInfo[0]) - pkgatom = mydata[0] - pkgver = mydata[2] - pkgtag = mydata[3] - pkgrev = mydata[18] - pkgslot = mydata[14] - pkgdigest = mydata[13] - pkgfile = mydata[12] - pkgcat = mydata[5] - pkgname = mydata[1] - pkgtrigger = dbconn.retrieveTrigger(packageInfo[0]) - pkgmessages = dbconn.retrieveMessages(packageInfo[0]) # still new - onDiskUsedSize += dbconn.retrieveOnDiskSize(packageInfo[0]) # still new - - # fill action queue - actionQueue[pkgatom] = {} - actionQueue[pkgatom]['repository'] = packageInfo[1] - actionQueue[pkgatom]['idpackage'] = packageInfo[0] - actionQueue[pkgatom]['slot'] = pkgslot - actionQueue[pkgatom]['atom'] = pkgatom - actionQueue[pkgatom]['version'] = pkgver - actionQueue[pkgatom]['category'] = pkgcat - actionQueue[pkgatom]['name'] = pkgname - actionQueue[pkgatom]['removeidpackage'] = -1 - actionQueue[pkgatom]['download'] = pkgfile - actionQueue[pkgatom]['checksum'] = pkgdigest - actionQueue[pkgatom]['messages'] = pkgmessages - actionQueue[pkgatom]['removeconfig'] = configFiles - dl = equoTools.checkNeededDownload(pkgfile, None) # we'll do a good check during installPackage - actionQueue[pkgatom]['fetch'] = dl - if dl < 0: - pkgsize = dbconn.retrieveSize(packageInfo[0]) - downloadSize += int(pkgsize) - - # get installed package data - installedVer = '0' - installedTag = '' - installedRev = 0 - clientDbconn = openClientDatabase() - if (clientDbconn != -1): - pkginstalled = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), matchSlot = pkgslot) - if (pkginstalled[1] == 0): - # found - idx = pkginstalled[0] - installedVer = clientDbconn.retrieveVersion(idx) - installedTag = clientDbconn.retrieveVersionTag(idx) - installedRev = clientDbconn.retrieveRevision(idx) - actionQueue[pkgatom]['removeidpackage'] = idx - onDiskFreedSize += clientDbconn.retrieveOnDiskSize(idx) - clientDbconn.closeDB() - - if not (ask or pretend or verbose): - continue - - action = 0 - flags = " [" - cmp = entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) - if (cmp == 0): - pkgsToReinstall += 1 - actionQueue[pkgatom]['removeidpackage'] = -1 # disable removal, not needed - flags += red("R") - action = 1 - elif (cmp > 0): - if (installedVer == "0"): - pkgsToInstall += 1 - actionQueue[pkgatom]['removeidpackage'] = -1 # disable removal, not needed - flags += darkgreen("N") - else: - pkgsToUpdate += 1 - flags += blue("U") - action = 2 - else: - pkgsToDowngrade += 1 - flags += darkblue("D") - action = -1 - flags += "] " - - # disable removal for packages already in removalQueue - if actionQueue[pkgatom]['removeidpackage'] in removalQueue: - actionQueue[pkgatom]['removeidpackage'] = -1 - - repoinfo = red("[")+bold(packageInfo[1])+red("] ") - oldinfo = '' - if action != 0: - oldinfo = " ["+blue(installedVer)+"/"+red(str(installedRev)) - oldtag = "]" - if installedTag: - oldtag = "/"+darkred(installedTag)+oldtag - oldinfo += oldtag - - print_info(darkred(" ##")+flags+repoinfo+enlightenatom(str(pkgatom))+"/"+str(pkgrev)+oldinfo) - dbconn.closeDB() - - if (removalQueue): - - # add depends to removalQueue that are not in runQueue - clientDbconn = openClientDatabase() - dependQueue = set() - for idpackage in removalQueue: - depends = clientDbconn.retrieveDepends(idpackage) - if depends == -2: - clientDbconn.regenerateDependsTable(output = False) + onDiskUsedSize += dbconn.retrieveOnDiskSize(packageInfo[0]) # still new + + # fill action queue + actionQueue[pkgatom] = {} + actionQueue[pkgatom]['repository'] = packageInfo[1] + actionQueue[pkgatom]['idpackage'] = packageInfo[0] + actionQueue[pkgatom]['slot'] = pkgslot + actionQueue[pkgatom]['atom'] = pkgatom + actionQueue[pkgatom]['version'] = pkgver + actionQueue[pkgatom]['category'] = pkgcat + actionQueue[pkgatom]['name'] = pkgname + actionQueue[pkgatom]['removeidpackage'] = -1 + actionQueue[pkgatom]['download'] = pkgfile + actionQueue[pkgatom]['removeconfig'] = configFiles + dl = equoTools.checkNeededDownload(pkgfile, None) # we'll do a good check during installPackage + actionQueue[pkgatom]['fetch'] = dl + if dl < 0: + pkgsize = dbconn.retrieveSize(packageInfo[0]) + downloadSize += int(pkgsize) + + # get installed package data + installedVer = '0' + installedTag = '' + installedRev = 0 + clientDbconn = openClientDatabase() + if (clientDbconn != -1): + pkginstalled = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), matchSlot = pkgslot) + if (pkginstalled[1] == 0): + # found + idx = pkginstalled[0] + installedVer = clientDbconn.retrieveVersion(idx) + installedTag = clientDbconn.retrieveVersionTag(idx) + installedRev = clientDbconn.retrieveRevision(idx) + actionQueue[pkgatom]['removeidpackage'] = idx + onDiskFreedSize += clientDbconn.retrieveOnDiskSize(idx) + clientDbconn.closeDB() + + if not (ask or pretend or verbose): + continue + + action = 0 + flags = " [" + cmp = entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) + if (cmp == 0): + pkgsToReinstall += 1 + actionQueue[pkgatom]['removeidpackage'] = -1 # disable removal, not needed + flags += red("R") + action = 1 + elif (cmp > 0): + if (installedVer == "0"): + pkgsToInstall += 1 + actionQueue[pkgatom]['removeidpackage'] = -1 # disable removal, not needed + flags += darkgreen("N") + else: + pkgsToUpdate += 1 + flags += blue("U") + action = 2 + else: + pkgsToDowngrade += 1 + flags += darkblue("D") + action = -1 + flags += "] " + + # disable removal for packages already in removalQueue + if actionQueue[pkgatom]['removeidpackage'] in removalQueue: + actionQueue[pkgatom]['removeidpackage'] = -1 + + repoinfo = red("[")+bold(packageInfo[1])+red("] ") + oldinfo = '' + if action != 0: + oldinfo = " ["+blue(installedVer)+"/"+red(str(installedRev)) + oldtag = "]" + if installedTag: + oldtag = "/"+darkred(installedTag)+oldtag + oldinfo += oldtag + + print_info(darkred(" ##")+flags+repoinfo+enlightenatom(str(pkgatom))+"/"+str(pkgrev)+oldinfo) + dbconn.closeDB() + + if (removalQueue): + + # add depends to removalQueue that are not in runQueue + clientDbconn = openClientDatabase() + dependQueue = set() + for idpackage in removalQueue: depends = clientDbconn.retrieveDepends(idpackage) - for depend in depends: - dependkey = clientDbconn.retrieveCategory(depend)+"/"+clientDbconn.retrieveName(depend) - dependslot = clientDbconn.retrieveSlot(depend) - # match in repositories - match = equoTools.atomMatch(dependkey, matchSlot = dependslot) - if match[0] != -1: - matchdbconn = openRepositoryDatabase(match[1]) - matchatom = matchdbconn.retrieveAtom(match[0]) - matchdbconn.closeDB() - if (matchatom not in actionQueue) and (depend not in removalQueue): # if the atom hasn't been pulled in, we need to remove depend - dependQueue.add(depend) - for depend in dependQueue: - removalQueue.append(depend) + if depends == -2: + clientDbconn.regenerateDependsTable(output = False) + depends = clientDbconn.retrieveDepends(idpackage) + for depend in depends: + dependkey = clientDbconn.retrieveCategory(depend)+"/"+clientDbconn.retrieveName(depend) + dependslot = clientDbconn.retrieveSlot(depend) + # match in repositories + match = equoTools.atomMatch(dependkey, matchSlot = dependslot) + if match[0] != -1: + matchdbconn = openRepositoryDatabase(match[1]) + matchatom = matchdbconn.retrieveAtom(match[0]) + matchdbconn.closeDB() + if (matchatom not in actionQueue) and (depend not in removalQueue): # if the atom hasn't been pulled in, we need to remove depend + dependQueue.add(depend) + for depend in dependQueue: + removalQueue.append(depend) + + if (ask or pretend or verbose): + print_info(red(" @@ ")+blue("These are the packages that would be ")+bold("removed")+blue(" (conflicting/substituted):")) + + for idpackage in removalQueue: + pkgatom = clientDbconn.retrieveAtom(idpackage) + onDiskFreedSize += clientDbconn.retrieveOnDiskSize(idpackage) + installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) + repoinfo = red("[")+brown("from: ")+bold(installedfrom)+red("] ") + print_info(red(" ## ")+"["+red("W")+"] "+repoinfo+enlightenatom(pkgatom)) + + clientDbconn.closeDB() - if (ask or pretend or verbose): - print_info(red(" @@ ")+blue("These are the packages that would be ")+bold("removed")+blue(" (conflicting/substituted):")) - - for idpackage in removalQueue: - pkgatom = clientDbconn.retrieveAtom(idpackage) - onDiskFreedSize += clientDbconn.retrieveOnDiskSize(idpackage) - installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) - repoinfo = red("[")+brown("from: ")+bold(installedfrom)+red("] ") - print_info(red(" ## ")+"["+red("W")+"] "+repoinfo+enlightenatom(pkgatom)) - - clientDbconn.closeDB() + if (runQueue) or (removalQueue): + # show download info + print_info(red(" @@ ")+blue("Packages needing install:\t")+red(str(len(runQueue)))) + print_info(red(" @@ ")+blue("Packages needing removal:\t")+red(str(pkgsToRemove))) + if (ask or verbose or pretend): + print_info(red(" @@ ")+darkgreen("Packages needing install:\t")+darkgreen(str(pkgsToInstall))) + print_info(red(" @@ ")+darkgreen("Packages needing reinstall:\t")+darkgreen(str(pkgsToReinstall))) + print_info(red(" @@ ")+blue("Packages needing update:\t\t")+blue(str(pkgsToUpdate))) + print_info(red(" @@ ")+red("Packages needing downgrade:\t")+red(str(pkgsToDowngrade))) + print_info(red(" @@ ")+blue("Download size:\t\t\t")+bold(str(entropyTools.bytesIntoHuman(downloadSize)))) + deltaSize = onDiskUsedSize - onDiskFreedSize + if (deltaSize > 0): + print_info(red(" @@ ")+blue("Used disk space:\t\t\t")+bold(str(entropyTools.bytesIntoHuman(deltaSize)))) + else: + print_info(red(" @@ ")+blue("Freed disk space:\t\t")+bold(str(entropyTools.bytesIntoHuman(abs(deltaSize))))) - if (runQueue) or (removalQueue): - # show download info - print_info(red(" @@ ")+blue("Packages needing install:\t")+red(str(len(runQueue)))) - print_info(red(" @@ ")+blue("Packages needing removal:\t")+red(str(pkgsToRemove))) - if (ask or verbose or pretend): - print_info(red(" @@ ")+darkgreen("Packages needing install:\t")+darkgreen(str(pkgsToInstall))) - print_info(red(" @@ ")+darkgreen("Packages needing reinstall:\t")+darkgreen(str(pkgsToReinstall))) - print_info(red(" @@ ")+blue("Packages needing update:\t\t")+blue(str(pkgsToUpdate))) - print_info(red(" @@ ")+red("Packages needing downgrade:\t")+red(str(pkgsToDowngrade))) - print_info(red(" @@ ")+blue("Download size:\t\t\t")+bold(str(entropyTools.bytesIntoHuman(downloadSize)))) - deltaSize = onDiskUsedSize - onDiskFreedSize - if (deltaSize > 0): - print_info(red(" @@ ")+blue("Used disk space:\t\t\t")+bold(str(entropyTools.bytesIntoHuman(deltaSize)))) - else: - print_info(red(" @@ ")+blue("Freed disk space:\t\t")+bold(str(entropyTools.bytesIntoHuman(abs(deltaSize))))) - - - if (ask): - rc = entropyTools.askquestion(" Would you like to run the queue ?") - if rc == "No": + if (ask): + rc = entropyTools.askquestion(" Would you like to run the queue ?") + if rc == "No": + dirscleanup() + return 0,0 + if (pretend): dirscleanup() - return 0,0 - if (pretend): - dirscleanup() - return 0,0 - + return 0,0 + + # clear old resume information + dumpTools.dumpobj(etpCache['install'],{}) + # store resume information + if not tbz2: # .tbz2 install resume not supported + resume_cache = {} + resume_cache['removalQueue'] = removalQueue[:] + resume_cache['runQueue'] = runQueue[:] + resume_cache['actionQueue'] = actionQueue.copy() + resume_cache['onlyfetch'] = onlyfetch + dumpTools.dumpobj(etpCache['install'],resume_cache) + + else: # if resume, load cache if possible + + # check if there's something to resume + resume_cache = dumpTools.loadobj(etpCache['install']) + if not resume_cache: # None or {} + print_error(red("Nothing to resume.")) + return 128,-1 + else: + try: + removalQueue = resume_cache['removalQueue'][:] + runQueue = resume_cache['runQueue'][:] + actionQueue = resume_cache['actionQueue'].copy() + onlyfetch = resume_cache['onlyfetch'] + print_warning(red("Resuming previous operations...")) + except: + print_error(red("Resume cache corrupted.")) + dumpTools.dumpobj(etpCache['install'],{}) + return 128,-1 + # running tasks totalqueue = str(len(runQueue)) totalremovalqueue = str(len(removalQueue)) @@ -642,6 +708,12 @@ def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, clientDbconn.closeDB() dirscleanup() return -1,rc + + # update resume cache + if not tbz2: # tbz2 caching not supported + resume_cache['removalQueue'].remove(idpackage) + dumpTools.dumpobj(etpCache['install'],resume_cache) + for packageInfo in runQueue: currentqueue += 1 @@ -650,6 +722,8 @@ def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, # get package atom dbconn = openRepositoryDatabase(repository) pkgatom = dbconn.retrieveAtom(idpackage) + actionQueue[pkgatom]['messages'] = dbconn.retrieveMessages(idpackage) + actionQueue[pkgatom]['checksum'] = dbconn.retrieveDigest(idpackage) steps = [] # download @@ -691,8 +765,6 @@ def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, steps.append("postinstall") steps.append("showmessages") - #print "steps for "+pkgatom+" -> "+str(steps) - print_info(red(" @@ ")+bold("(")+blue(str(currentqueue))+"/"+red(totalqueue)+bold(") ")+">>> "+darkgreen(pkgatom)) for step in steps: @@ -702,18 +774,25 @@ def installPackages(packages = [], atomsdata = [], ask = False, pretend = False, dirscleanup() return -1,rc + # update resume cache + if not tbz2: # tbz2 caching not supported + resume_cache['runQueue'].remove(packageInfo) + dumpTools.dumpobj(etpCache['install'],resume_cache) + if (onlyfetch): print_info(red(" @@ ")+blue("Fetch Complete.")) else: print_info(red(" @@ ")+blue("Install Complete.")) + # clear resume information + dumpTools.dumpobj(etpCache['install'],{}) + clientDbconn.closeDB() - dirscleanup() return 0,0 -def removePackages(packages = [], atomsdata = [], ask = False, pretend = False, verbose = False, deps = True, deep = False, systemPackagesCheck = True, configFiles = False): +def removePackages(packages = [], atomsdata = [], ask = False, pretend = False, verbose = False, deps = True, deep = False, systemPackagesCheck = True, configFiles = False, resume = False): # check if I am root if (not entropyTools.isRoot()): @@ -722,136 +801,169 @@ def removePackages(packages = [], atomsdata = [], ask = False, pretend = False, clientDbconn = openClientDatabase() - foundAtoms = [] - if (atomsdata): - for idpackage in atomsdata: - foundAtoms.append([clientDbconn.retrieveAtom(idpackage),(idpackage,0)]) - else: - for package in packages: - foundAtoms.append([package,clientDbconn.atomMatch(package)]) + if not resume: - # filter packages not found - _foundAtoms = [] - for result in foundAtoms: - exitcode = result[1][0] - if (exitcode != -1): - _foundAtoms.append(result[1]) - else: - print_warning(red("## ATTENTION -> package")+bold(" "+result[0]+" ")+red("is not installed.")) - - foundAtoms = _foundAtoms + foundAtoms = [] + if (atomsdata): + for idpackage in atomsdata: + foundAtoms.append([clientDbconn.retrieveAtom(idpackage),(idpackage,0)]) + else: + for package in packages: + foundAtoms.append([package,clientDbconn.atomMatch(package)]) - # are packages in foundAtoms? - if (len(foundAtoms) == 0): - print_error(red("No packages found")) - return 127,-1 - - plainRemovalQueue = [] + # filter packages not found + _foundAtoms = [] + for result in foundAtoms: + exitcode = result[1][0] + if (exitcode != -1): + _foundAtoms.append(result[1]) + else: + print_warning(red("## ATTENTION -> package")+bold(" "+result[0]+" ")+red("is not installed.")) - lookForOrphanedPackages = True - # now print the selected packages - print_info(red(" @@ ")+blue("These are the chosen packages:")) - totalatoms = len(foundAtoms) - atomscounter = 0 - for atomInfo in foundAtoms: - atomscounter += 1 - idpackage = atomInfo[0] - systemPackage = clientDbconn.isSystemPackage(idpackage) - - # get needed info - pkgatom = clientDbconn.retrieveAtom(idpackage) - installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) - - if (systemPackage) and (systemPackagesCheck): - # check if the package is slotted and exist more than one installed first - sysresults = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), multiMatch = True) - slots = set() - if sysresults[1] == 0: - for x in sysresults[0]: - slots.add(clientDbconn.retrieveSlot(x)) - if len(slots) < 2: - print_warning(darkred(" # !!! ")+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+red(" is a vital package. Removal forbidden.")) - continue - else: - print_warning(darkred(" # !!! ")+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+red(" is a vital package. Removal forbidden.")) - continue - plainRemovalQueue.append(idpackage) - - print_info(" # "+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+" | Installed from: "+red(installedfrom)) - - if (verbose or ask or pretend): - print_info(red(" @@ ")+blue("Number of packages: ")+str(totalatoms)) + foundAtoms = _foundAtoms + + # are packages in foundAtoms? + if (len(foundAtoms) == 0): + print_error(red("No packages found")) + return 127,-1 - if (deps): - question = " Would you like to look for packages that can be removed along with the selected above?" - else: - question = " Would you like to remove them now?" - lookForOrphanedPackages = False - - if (ask): - rc = entropyTools.askquestion(question) - if rc == "No": + plainRemovalQueue = [] + + lookForOrphanedPackages = True + # now print the selected packages + print_info(red(" @@ ")+blue("These are the chosen packages:")) + totalatoms = len(foundAtoms) + atomscounter = 0 + for atomInfo in foundAtoms: + atomscounter += 1 + idpackage = atomInfo[0] + systemPackage = clientDbconn.isSystemPackage(idpackage) + + # get needed info + pkgatom = clientDbconn.retrieveAtom(idpackage) + installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) + + if (systemPackage) and (systemPackagesCheck): + # check if the package is slotted and exist more than one installed first + sysresults = clientDbconn.atomMatch(entropyTools.dep_getkey(pkgatom), multiMatch = True) + slots = set() + if sysresults[1] == 0: + for x in sysresults[0]: + slots.add(clientDbconn.retrieveSlot(x)) + if len(slots) < 2: + print_warning(darkred(" # !!! ")+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+red(" is a vital package. Removal forbidden.")) + continue + else: + print_warning(darkred(" # !!! ")+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+red(" is a vital package. Removal forbidden.")) + continue + plainRemovalQueue.append(idpackage) + + print_info(" # "+red("(")+brown(str(atomscounter))+"/"+blue(str(totalatoms))+red(")")+" "+enlightenatom(pkgatom)+" | Installed from: "+red(installedfrom)) + + if (verbose or ask or pretend): + print_info(red(" @@ ")+blue("Number of packages: ")+str(totalatoms)) + + if (deps): + question = " Would you like to look for packages that can be removed along with the selected above?" + else: + question = " Would you like to remove them now?" lookForOrphanedPackages = False - if (not deps): - clientDbconn.closeDB() - return 0,0 - - if (not plainRemovalQueue): - print_error(red("Nothing to do.")) - return 127,-1 - - removalQueue = [] - if (lookForOrphanedPackages): - choosenRemovalQueue = [] - print_info(red(" @@ ")+blue("Calculating...")) - treeview = equoTools.generateDependsTree(plainRemovalQueue, deep = deep) - treelength = len(treeview[0]) - if treelength > 1: - treeview = treeview[0] - for x in range(treelength)[::-1]: - for y in treeview[x]: - choosenRemovalQueue.append(y) - - if (choosenRemovalQueue): - print_info(red(" @@ ")+blue("These are the packages that would be added to the removal queue:")) - totalatoms = str(len(choosenRemovalQueue)) - atomscounter = 0 - - for idpackage in choosenRemovalQueue: - atomscounter += 1 - rematom = clientDbconn.retrieveAtom(idpackage) - installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) - repositoryInfo = bold("[")+red("from: ")+brown(installedfrom)+bold("]") - stratomscounter = str(atomscounter) - while len(stratomscounter) < len(totalatoms): - stratomscounter = " "+stratomscounter - print_info(" # "+red("(")+bold(stratomscounter)+"/"+blue(str(totalatoms))+red(")")+repositoryInfo+" "+blue(rematom)) - - if (ask): - rc = entropyTools.askquestion(" Would you like to add these packages to the removal queue?") - if rc != "No": - print_info(red(" @@ ")+blue("Removal Queue updated.")) - for x in choosenRemovalQueue: - removalQueue.append(x) - else: - for x in choosenRemovalQueue: - removalQueue.append(x) - else: - writechar("\n") - if (ask): - if (deps): - rc = entropyTools.askquestion(" I am going to start the removal. Are you sure?") + if (ask): + rc = entropyTools.askquestion(question) if rc == "No": - clientDbconn.closeDB() - return 0,0 - else: - entropyTools.countdown(what = red(" @@ ")+blue("Starting removal in "),back = True) - - - for idpackage in plainRemovalQueue: - removalQueue.append(idpackage) + lookForOrphanedPackages = False + if (not deps): + clientDbconn.closeDB() + return 0,0 + if (not plainRemovalQueue): + print_error(red("Nothing to do.")) + return 127,-1 + + removalQueue = [] + + if (lookForOrphanedPackages): + choosenRemovalQueue = [] + print_info(red(" @@ ")+blue("Calculating...")) + treeview = equoTools.generateDependsTree(plainRemovalQueue, deep = deep) + treelength = len(treeview[0]) + if treelength > 1: + treeview = treeview[0] + for x in range(treelength)[::-1]: + for y in treeview[x]: + choosenRemovalQueue.append(y) + + if (choosenRemovalQueue): + print_info(red(" @@ ")+blue("These are the packages that would be added to the removal queue:")) + totalatoms = str(len(choosenRemovalQueue)) + atomscounter = 0 + + for idpackage in choosenRemovalQueue: + atomscounter += 1 + rematom = clientDbconn.retrieveAtom(idpackage) + installedfrom = clientDbconn.retrievePackageFromInstalledTable(idpackage) + repositoryInfo = bold("[")+red("from: ")+brown(installedfrom)+bold("]") + stratomscounter = str(atomscounter) + while len(stratomscounter) < len(totalatoms): + stratomscounter = " "+stratomscounter + print_info(" # "+red("(")+bold(stratomscounter)+"/"+blue(str(totalatoms))+red(")")+repositoryInfo+" "+blue(rematom)) + + if (ask): + rc = entropyTools.askquestion(" Would you like to add these packages to the removal queue?") + if rc != "No": + print_info(red(" @@ ")+blue("Removal Queue updated.")) + for x in choosenRemovalQueue: + removalQueue.append(x) + else: + for x in choosenRemovalQueue: + removalQueue.append(x) + else: + writechar("\n") + if (ask): + if (deps): + rc = entropyTools.askquestion(" I am going to start the removal. Are you sure?") + if rc == "No": + clientDbconn.closeDB() + return 0,0 + else: + if (deps): + entropyTools.countdown(what = red(" @@ ")+blue("Starting removal in "),back = True) + + for idpackage in plainRemovalQueue: + removalQueue.append(idpackage) + + # clear old resume information + dumpTools.dumpobj(etpCache['remove'],{}) + # store resume information + resume_cache = {} + resume_cache['removalQueue'] = removalQueue[:] + dumpTools.dumpobj(etpCache['remove'],resume_cache) + + else: # if resume, load cache if possible + + # check if there's something to resume + resume_cache = dumpTools.loadobj(etpCache['remove']) + if not resume_cache: # None or {} + print_error(red("Nothing to resume.")) + return 128,-1 + else: + try: + removalQueue = resume_cache['removalQueue'][:] + print_warning(red("Resuming previous operations...")) + except: + print_error(red("Resume cache corrupted.")) + dumpTools.dumpobj(etpCache['remove'],{}) + return 128,-1 + # validate removalQueue + invalid = [] + for idpackage in removalQueue: + try: + clientDbconn.retrieveAtom(idpackage) + except TypeError: + invalid.append(idpackage) + removalQueue = [x for x in removalQueue if x not in invalid] + currentqueue = 0 for idpackage in removalQueue: currentqueue += 1 @@ -873,6 +985,10 @@ def removePackages(packages = [], atomsdata = [], ask = False, pretend = False, if (rc != 0): clientDbconn.closeDB() return -1,rc + + # update resume cache + resume_cache['removalQueue'].remove(idpackage) + dumpTools.dumpobj(etpCache['remove'],resume_cache) print_info(red(" @@ ")+blue("All done.")) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 9ca097cbb..2cf5d901e 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -56,7 +56,7 @@ def openRepositoryDatabase(repositoryName, xcache = True): raise Exception, "openRepositoryDatabase: cannot sync repository "+repositoryName conn = etpDatabase(readOnly = True, dbFile = dbfile, clientDatabase = True, dbname = 'repo_'+repositoryName, xcache = xcache) # initialize CONFIG_PROTECT - if (not etpRepositories[repositoryName]['configprotect']) or (not etpRepositories[repositoryName]['configprotectmask']): + if (etpRepositories[repositoryName]['configprotect'] == None) or (etpRepositories[repositoryName]['configprotectmask'] == None): etpRepositories[repositoryName]['configprotect'] = conn.listConfigProtectDirectories() etpRepositories[repositoryName]['configprotectmask'] = conn.listConfigProtectDirectories(mask = True) etpRepositories[repositoryName]['configprotect'] += [x for x in etpConst['configprotect'] if x not in etpRepositories[repositoryName]['configprotect']] @@ -1564,11 +1564,11 @@ class etpDatabase: return data def fetchall2set(self, item): - content = set() + mycontent = set() for x in item: for y in x: - content.add(y) - return content + mycontent.add(y) + return mycontent def fetchall2list(self, item): content = [] @@ -2000,6 +2000,8 @@ class etpDatabase: def retrieveContent(self, idpackage, extended = False): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveContent: retrieving Content for package ID "+str(idpackage)) + self.createContentIndex() # FIXME: remove this with 1.0 + extstring = '' if extended: extstring = ",type" @@ -2164,6 +2166,17 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isProtectAvailable: "+protect+" available.") return result[0] + def isFileAvailable(self,file): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isFileAvailable: called.") + self.createContentIndex() # FIXME: remove this with 1.0 + self.cursor.execute('SELECT idpackage FROM content WHERE file = "'+file+'"') + result = self.cursor.fetchone() + if not result: + dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isFileAvailable: "+file+" not available.") + return False + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isFileAvailable: "+file+" available.") + return True + def isSourceAvailable(self,source): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSourceAvailable: called.") self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = "'+source+'"') @@ -2574,8 +2587,7 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllFiles: called.") self.cursor.execute('SELECT file FROM content') if clean: - x = self.fetchall2set(self.cursor.fetchall()) - return x + return self.fetchall2set(self.cursor.fetchall()) else: return self.fetchall2list(self.cursor.fetchall()) @@ -2763,6 +2775,9 @@ class etpDatabase: self.cursor.execute('CREATE TABLE counters ( counter INTEGER PRIMARY KEY, idpackage INTEGER );') self.commitChanges() + def createContentIndex(self): + self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex ON content ( file )') + def regenerateCountersTable(self, output = False): self.createCountersTable() # assign a counter to an idpackage diff --git a/libraries/dumpTools.py b/libraries/dumpTools.py index af17b0604..1957b667e 100644 --- a/libraries/dumpTools.py +++ b/libraries/dumpTools.py @@ -19,38 +19,54 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ''' - import sys -import commands -sys.path.append('../libraries') from xml.dom import minidom from entropyConstants import * +curname = '' + ''' @description: dump object to file @input: name of the object, object @output: status code ''' def dumpobj(name,object): - doc = minidom.Document() - structure = doc.createElement("structure") - doc.appendChild(structure) - data = doc.createElement("data") - structure.appendChild(data) - text = doc.createTextNode(unicode(object)) - data.appendChild(text) - # etpConst['dumpstoragedir'] + curname = name try: - if not os.path.isdir(etpConst['dumpstoragedir']): - os.makedirs(etpConst['dumpstoragedir']) - f = open(etpConst['dumpstoragedir']+"/"+name+".dmp","w") #FIXME add check - f.writelines(doc.toprettyxml(indent=" ")) - f.flush() - f.close() - except: - raise IOError,"can't write to file" + doc = minidom.Document() + structure = doc.createElement("structure") + doc.appendChild(structure) + data = doc.createElement("data") + structure.appendChild(data) + text = doc.createTextNode(unicode(object)) + data.appendChild(text) + # etpConst['dumpstoragedir'] + try: + if not os.path.isdir(etpConst['dumpstoragedir']): + os.makedirs(etpConst['dumpstoragedir']) + f = open(etpConst['dumpstoragedir']+"/"+name+".dmp","w") #FIXME add check + f.writelines(doc.toprettyxml(indent=" ")) + f.flush() + f.close() + except: + raise IOError,"can't write to file "+name + except KeyboardInterrupt: + import signal + signal.signal(signal.SIGTERM, removeobj) + signal.signal(signal.SIGQUIT, removeobj) + signal.signal(signal.SIGINT, removeobj) + signal.signal(signal.SIGHUP, removeobj) + dumpobj(name,object) + sys.exit(1) +def removeobj(): + if curname: + try: + os.remove(etpConst['dumpstoragedir']+"/"+curname+".dmp") + except: + pass + ''' @description: load object from a file @input: name of the object diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index e54576135..348dbfd93 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -281,6 +281,8 @@ CREATE TABLE neededreference ( library VARCHAR ); +CREATE INDEX contentindex ON content ( file ); + """ # Entropy directories specifications @@ -437,6 +439,9 @@ etpCache = { 'dbInfo': 'info_', # used by the database controller as prefix to the cache files belonging to etpDatabase class (info retrival) 'atomMatch': 'atomMatchCache', # used to store info about repository dependencies solving 'generateDependsTree': 'generateDependsTreeCache', # used to store info about removal dependencies + 'install': 'resume_install', # resume cache (install) + 'remove': 'resume_remove', # resume cache (remove) + 'world': 'resume_world', # resume cache (world) } # byte sizes of disk caches @@ -467,15 +472,12 @@ global atomMatchCache atomMatchCache = {} global atomClientMatchCache atomClientMatchCache = {} -global contentCache -contentCache = {} global generateDependsTreeCache generateDependsTreeCache = {} def const_resetCache(): dbCacheStore.clear() atomMatchCache.clear() atomClientMatchCache.clear() - contentCache.clear() generateDependsTreeCache.clear() # handle Entropy Version @@ -628,8 +630,8 @@ if os.path.isfile(etpConst['repositoriesconf']): etpRepositories[reponame]['dbcformat'] = dbformat etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch'] # initialize CONFIG_PROTECT - will be filled the first time the db will be opened - etpRepositories[reponame]['configprotect'] = set() - etpRepositories[reponame]['configprotectmask'] = set() + etpRepositories[reponame]['configprotect'] = None + etpRepositories[reponame]['configprotectmask'] = None elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): branch = line.split("|")[1] etpConst['branch'] = branch diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 935da004c..8cced7fa5 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -26,7 +26,7 @@ from entropyConstants import * import os import re from sys import exit, stdout, getfilesystemencoding -import threading, time +import threading, time, tarfile # Instantiate the databaseStatus: import databaseTools @@ -333,10 +333,19 @@ def md5string(string): return m.hexdigest() # used by equo, this function retrieves the new safe Gentoo-aware file path -def allocateMaskedFile(file): +def allocateMaskedFile(file, fromfile): + + # check if file and tofile are equal + if os.path.isfile(file) and os.path.isfile(fromfile): + old = md5sum(fromfile) + new = md5sum(file) + if old == new: + return file, False + counter = -1 newfile = "" previousfile = "" + while 1: counter += 1 txtcounter = str(counter) @@ -354,13 +363,22 @@ def allocateMaskedFile(file): if not newfile: newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file) else: - if os.path.exists(previousfile): + + if os.path.exists(previousfile): + + # compare fromfile with previousfile + new = md5sum(fromfile) + old = md5sum(previousfile) + if new == old: + return previousfile, False + # compare old and new, if they match, suggest previousfile directly new = md5sum(file) old = md5sum(previousfile) if (new == old): - newfile = previousfile - return newfile + return previousfile, False + + return newfile, True def extractElog(file): @@ -1041,12 +1059,45 @@ def compressTarBz2(storepath,pathtocompress): # tar.bz2 uncompress function... def uncompressTarBz2(filepath, extractPath = None): - entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"uncompressTarBz2: called. ") + entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"uncompressTarBz2: called.") if extractPath is None: extractPath = os.path.dirname(filepath) - cmd = "tar xjf "+filepath+" -C "+extractPath - rc = spawnCommand(cmd, "&> /dev/null") - return rc + tar = tarfile.open(filepath,"r:bz2") + + directories = [] + for tarinfo in tar: + if tarinfo.isdir(): + # Extract directory with a safe mode, so that + # all files below can be extracted as well. + try: + os.makedirs(os.path.join(extractPath, tarinfo.name), 0777) + except EnvironmentError: + pass + directories.append(tarinfo) + else: + tar.extract(tarinfo, extractPath) + + # Reverse sort directories. + directories.sort(lambda a, b: cmp(a.name, b.name)) + directories.reverse() + + origpath = extractPath + # Set correct owner, mtime and filemode on directories. + for tarinfo in directories: + extractPath = os.path.join(extractPath, tarinfo.name) + try: + tar.chown(tarinfo, extractPath) + tar.utime(tarinfo, extractPath) + tar.chmod(tarinfo, extractPath) + except tarfile.ExtractError, e: + if tar.errorlevel > 1: + raise + + tar.close() + if os.listdir(origpath): + return 0 + else: + return -1 def bytesIntoHuman(bytes): size = str(round(float(bytes)/1024,1)) @@ -1230,7 +1281,6 @@ def writeNewBranch(branch): def quickpkg(pkgdata,dirpath): entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"quickpkg: called -> "+str(pkgdata)+" | dirpath: "+dirpath) - import tarfile import stat import databaseTools