diff --git a/client/repositoriesTools.py b/client/repositoriesTools.py index 04cfeac97..ba40a10c5 100644 --- a/client/repositoriesTools.py +++ b/client/repositoriesTools.py @@ -25,32 +25,29 @@ ## Repositories Tools # -import os from entropyConstants import * from clientConstants import * from outputTools import * import entropyTools -from remoteTools import downloadData, getOnlineContent -import dumpTools import exceptionTools def repositories(options): - + # Options available for all the packages submodules myopts = options[1:] equoRequestForceUpdate = False rc = 0 for opt in myopts: - if (opt == "--force"): - equoRequestForceUpdate = True + if (opt == "--force"): + equoRequestForceUpdate = True if (options[0] == "update"): - rc = syncRepositories(forceUpdate = equoRequestForceUpdate) + rc = syncRepositories(forceUpdate = equoRequestForceUpdate) elif (options[0] == "status"): - for repo in etpRepositories: - showRepositoryInfo(repo) + for repo in etpRepositories: + showRepositoryInfo(repo) elif (options[0] == "repoinfo"): - showRepositories() + showRepositories() else: rc = -10 return rc @@ -60,32 +57,32 @@ def showRepositories(): print_info(darkred(" * ")+darkgreen("Active Repositories:")) repoNumber = 0 for repo in etpRepositories: - repoNumber += 1 - print_info(blue("\t#"+str(repoNumber))+bold(" "+etpRepositories[repo]['description'])) - sourcecount = 0 - for pkgrepo in etpRepositories[repo]['packages']: - sourcecount += 1 - print_info(red("\t\tPackages Mirror #"+str(sourcecount)+" : ")+darkgreen(pkgrepo)) - print_info(red("\t\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database'])) - print_info(red("\t\tRepository name: ")+bold(repo)) - print_info(red("\t\tRepository database path: ")+blue(etpRepositories[repo]['dbpath'])) + repoNumber += 1 + print_info(blue("\t#"+str(repoNumber))+bold(" "+etpRepositories[repo]['description'])) + sourcecount = 0 + for pkgrepo in etpRepositories[repo]['packages']: + sourcecount += 1 + print_info(red("\t\tPackages Mirror #"+str(sourcecount)+" : ")+darkgreen(pkgrepo)) + print_info(red("\t\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database'])) + print_info(red("\t\tRepository name: ")+bold(repo)) + print_info(red("\t\tRepository database path: ")+blue(etpRepositories[repo]['dbpath'])) return 0 def showRepositoryInfo(reponame): repoNumber = 0 for repo in etpRepositories: - repoNumber += 1 - if repo == reponame: - break + repoNumber += 1 + if repo == reponame: + break print_info(blue("#"+str(repoNumber))+bold(" "+etpRepositories[reponame]['description'])) if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasefile']): - status = "active" + status = "active" else: - status = "never synced" + status = "never synced" print_info(darkgreen("\tStatus: ")+darkred(status)) urlcount = 0 for repourl in etpRepositories[reponame]['packages'][::-1]: - urlcount += 1 + urlcount += 1 print_info(red("\tPackages URL #"+str(urlcount)+": ")+darkgreen(repourl)) print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[reponame]['database'])) print_info(red("\tRepository name: ")+bold(reponame)) @@ -101,187 +98,143 @@ def showRepositoryInfo(reponame): # @returns int>0 if the file exists def getRepositoryRevision(reponame): if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): - f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r") - revision = int(f.readline().strip()) - f.close() + f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r") + revision = int(f.readline().strip()) + f.close() else: - revision = -1 + revision = -1 return revision -# @returns -1 if the file is not available -# @returns int>0 if the revision has been retrieved -def getOnlineRepositoryRevision(reponame): - url = etpRepositories[reponame]['database']+"/"+etpConst['etpdatabaserevisionfile'] - status = getOnlineContent(url) - if (status): - status = status[0].strip() - return int(status) - else: - return -1 - # @returns -1 if the file does not exist # @returns int>0 if the file exists def getRepositoryDbFileHash(reponame): if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile']): - f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") - mhash = f.readline().strip().split()[0] - f.close() + f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") + mhash = f.readline().strip().split()[0] + f.close() else: - mhash = "-1" + mhash = "-1" return mhash def syncRepositories(reponames = [], forceUpdate = False): - # check if I am root - if (not entropyTools.isRoot()): - print_error(red("\t You must run this application as root.")) - return 1 - - # check etpRepositories - if len(etpRepositories) == 0: - if (not etpUi['quiet']): - print_error(darkred(" * ")+red("No repositories specified in ")+etpConst['repositoriesconf']) - return 127 + # load repository class + try: + repoConn = repositoryController(reponames, forceUpdate) + except exceptionTools.PermissionDenied: + print_error(red("\t You must run this application as root.")) + return 1 + except exceptionTools.MissingParameter: + print_error(darkred(" * ")+red("No repositories specified in ")+etpConst['repositoriesconf']) + return 127 + except exceptionTools.OnlineMirrorError: + print_error(darkred(" @@ ")+darkgreen("You are not connected to the Internet. You should.")) + return 126 + except Exception, e: + print_error(darkred(" @@ ")+darkgreen("Unhandled exception: %s" % (str(e),))) + return 2 + # let's dance! if (not etpUi['quiet']): print_info(darkred(" @@ ")+darkgreen("Repositories syncronization...")) + repoNumber = 0 - syncErrors = False - - if (not reponames): - for x in etpRepositories: - reponames.append(x) - - dbupdated = False - - # Test network connectivity - conntest = getOnlineContent("http://svn.sabayonlinux.org") - if not conntest: - print_info(darkred(" @@ ")+darkgreen("You are not connected to the Internet. You should.")) - return 2 - - for repo in reponames: - - repoNumber += 1 - - if (not etpUi['quiet']): - print_info(blue(" #"+str(repoNumber))+bold(" "+etpRepositories[repo]['description'])) - print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database'])) - print_info(red("\tDatabase local path: ")+darkgreen(etpRepositories[repo]['dbpath'])) - - # check if database is already updated to the latest revision - onlinestatus = getOnlineRepositoryRevision(repo) - if (onlinestatus != -1): - localstatus = getRepositoryRevision(repo) - if (localstatus == onlinestatus) and (not forceUpdate): - if (not etpUi['quiet']): - print_info(bold("\tAttention: ")+red("database is already up to date.")) - continue - - # get database lock - rc = downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile'],"/dev/null") - if rc != "-3": # cannot download database - if (not etpUi['quiet']): - print_error(bold("\tATTENTION -> ")+red("repository is being updated. Try again in a few minutes.")) - syncErrors = True - continue - - # database is going to be updated - dbupdated = True - # clear database interface cache belonging to this repository - dumpTools.dumpobj(etpCache['dbInfo']+repo,{}) - - cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat']) - if cmethod == None: raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.") - - # starting to download - if (not etpUi['quiet']): - print_info(red("\tDownloading database ")+darkgreen(etpConst[cmethod[2]])+red(" ...")) - # create dir if it doesn't exist - if not os.path.isdir(etpRepositories[repo]['dbpath']): - if (not etpUi['quiet']): - print_info(red("\t\tCreating database directory...")) - os.makedirs(etpRepositories[repo]['dbpath']) - # download - rc = downloadData(etpRepositories[repo]['database']+"/"+etpConst[cmethod[2]],etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]) - if rc in ("-1","-2","-3"): + for repo in repoConn.reponames: + + repoNumber += 1 + + if (not etpUi['quiet']): + print_info(blue(" #"+str(repoNumber))+bold(" "+etpRepositories[repo]['description'])) + print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database'])) + print_info(red("\tDatabase local path: ")+darkgreen(etpRepositories[repo]['dbpath'])) + + # check if database is already updated to the latest revision + update = repoConn.isRepositoryUpdatable(repo) + if not update: + if not etpUi['quiet']: print_info(bold("\tAttention: ")+red("database is already up to date.")) + continue + + # get database lock + unlocked = repoConn.isRepositoryUnlocked(repo) + if not unlocked: + if not etpUi['quiet']: print_error(bold("\tATTENTION -> ")+red("repository is being updated. Try again in a few minutes.")) + continue + + # database is going to be updated + repoConn.dbupdated = True + # clear database interface cache belonging to this repository + repoConn.clearRepositoryCache(repo) + cmethod = repoConn.validateCompressionMethod(repo) + repoConn.ensureRepositoryPath(repo) + + # starting to download + if (not etpUi['quiet']): + print_info(red("\tDownloading database ")+darkgreen(etpConst[cmethod[2]])+red(" ...")) + + down_status = repoConn.downloadDatabase(repo, cmethod[2]) + if not down_status: print_info(bold("\tAttention: ")+red("database does not exist online.")) - return 128 - - if (not etpUi['quiet']): - print_info(red("\tUnpacking database to ")+darkgreen(etpConst['etpdatabasefile'])+red(" ...")) - eval("entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]) - # download etpdatabasehashfile - if (not etpUi['quiet']): - print_info(red("\tDownloading checksum ")+darkgreen(etpConst['etpdatabasehashfile'])+red(" ...")) - downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasehashfile'],etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) - # checking checksum - if (not etpUi['quiet']): - print_info(red("\tChecking downloaded database ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."), back = True) - f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") - md5hash = f.readline().strip() - md5hash = md5hash.split()[0] - f.close() - rc = entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash) - if rc: - if (not etpUi['quiet']): - print_info(red("\tDownloaded database status: ")+bold("OK")) - else: - if (not etpUi['quiet']): - print_error(red("\tDownloaded database status: ")+darkred("ERROR")) - print_error(red("\t An error occured while checking database integrity")) - # delete all - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]) - if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): - os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) - syncErrors = True - continue - - # download etpdatabaserevisionfile - if (not etpUi['quiet']): - print_info(red("\tDownloading revision ")+darkgreen(etpConst['etpdatabaserevisionfile'])+red(" ...")) - downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile'],etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) - - if (not etpUi['quiet']): - print_info(red("\tUpdated repository revision: ")+bold(str(getRepositoryRevision(repo)))) - print_info(darkgreen("\tUpdate completed")) + continue - if syncErrors: - if (not etpUi['quiet']): - print_warning(darkred(" @@ ")+red("Something bad happened. Please have a look.")) - return 128 + # unpack database + if (not etpUi['quiet']): + print_info(red("\tUnpacking database to ")+darkgreen(etpConst['etpdatabasefile'])+red(" ...")) + # unpack database + repoConn.unpackDownloadedDatabase(repo,cmethod) + + # download checksum + if (not etpUi['quiet']): + print_info(red("\tDownloading checksum ")+darkgreen(etpConst['etpdatabasehashfile'])+red(" ...")) + down_status = repoConn.downloadChecksum(repo) + if not down_status: + print_warning(red("\tCannot fetch checksum. Cannot verify database integrity !")) + else: + # verify checksum + if (not etpUi['quiet']): + print_info(red("\tChecking downloaded database ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."), back = True) + db_status = repoConn.verifyDatabaseChecksum(repo) + if db_status == -1: + print_warning(red("\tCannot open digest. Cannot verify database integrity !")) + elif db_status: + if (not etpUi['quiet']): + print_info(red("\tDownloaded database status: ")+bold("OK")) + else: + if (not etpUi['quiet']): + print_error(red("\tDownloaded database status: ")+darkred("ERROR")) + print_error(red("\t An error occured while checking database integrity")) + # delete all + repoConn.removeRepositoryFiles(repo, cmethod[2]) + repoConn.syncErrors = True + continue + + # download revision + if (not etpUi['quiet']): + print_info(red("\tDownloading revision ")+darkgreen(etpConst['etpdatabaserevisionfile'])+red(" ...")) + rev_status = repoConn.downloadDatabaseRevision(repo) + if not rev_status: + print_warning(red("\tCannot download repository revision, don't ask me why !")) + else: + if not etpUi['quiet']: print_info(red("\tUpdated repository revision: ")+bold(str(getRepositoryRevision(repo)))) + + print_info(darkgreen("\tUpdate completed")) + + repoConn.closeTransactions() + + if repoConn.syncErrors: + if (not etpUi['quiet']): + print_warning(darkred(" @@ ")+red("Something bad happened. Please have a look.")) + del repoConn + return 128 - if (dbupdated): - - # safely clean ram caches - atomMatchCache.clear() - dumpTools.dumpobj(etpCache['atomMatch'],{}) - generateDependsTreeCache.clear() - dumpTools.dumpobj(etpCache['generateDependsTree'],{}) - for dbinfo in dbCacheStore: - dbCacheStore[dbinfo].clear() - dumpTools.dumpobj(dbinfo,{}) - - # clean caches - import cacheTools - cacheTools.generateCache(depcache = True, configcache = False) - - # clean resume caches - dumpTools.dumpobj(etpCache['install'],{}) - dumpTools.dumpobj(etpCache['world'],{}) - dumpTools.dumpobj(etpCache['remove'],[]) - # tell if a new equo release is available import equoTools from databaseTools import openClientDatabase try: clientDbconn = openClientDatabase(xcache = False) except exceptionTools.SystemDatabaseError: + del repoConn return 0 - + matches = clientDbconn.searchPackages("app-admin/equo") if matches: equo_match = "<="+matches[0][0] @@ -291,7 +244,190 @@ def syncRepositories(reponames = [], forceUpdate = False): print_warning(darkred(" !! ")+blue("A new version of ")+bold("equo")+blue(" is available. Please ")+bold("install it")+blue(" before any other package.")) del matches del equo_unsatisfied + clientDbconn.closeDB() del clientDbconn + del repoConn - return 0 \ No newline at end of file + return 0 + +# +# repository control class, that's it +# +class repositoryController: + + def __init__(self, reponames = [], forceUpdate = False): + + self.reponames = reponames + self.forceUpdate = forceUpdate + self.syncErrors = False + self.dbupdated = False + + import remoteTools + import dumpTools + self.remoteTools = remoteTools + self.dumpTools = dumpTools + + # check if I am root + if (not entropyTools.isRoot()): + raise exceptionTools.PermissionDenied("PermissionDenied: not allowed as user.") + + # check etpRepositories + if not etpRepositories: + raise exceptionTools.MissingParameter("MissingParameter: no repositories specified in %s" % (etpConst['repositoriesconf'],)) + + # Test network connectivity + conntest = self.remoteTools.getOnlineContent("http://svn.sabayonlinux.org") + if not conntest: + raise exceptionTools.OnlineMirrorError("OnlineMirrorError: you are not connected to the Internet. You should.") + + if (not self.reponames): + for x in etpRepositories: + self.reponames.append(x) + + def validateRepositoryId(self, repoid): + if repoid not in self.reponames: + raise exceptionTools.InvalidData("InvalidData: repository is not listed in self.reponames") + + # @returns -1 if the file is not available + # @returns int>0 if the revision has been retrieved + def getOnlineRepositoryRevision(self, repo): + + self.validateRepositoryId(repo) + + url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile'] + status = self.remoteTools.getOnlineContent(url) + if (status): + status = status[0].strip() + return int(status) + else: + return -1 + + def isRepositoryUpdatable(self, repo): + + self.validateRepositoryId(repo) + + onlinestatus = self.getOnlineRepositoryRevision(repo) + if (onlinestatus != -1): + localstatus = getRepositoryRevision(repo) + if (localstatus == onlinestatus) and (not self.forceUpdate): + return False + return True + + def isRepositoryUnlocked(self, repo): + + self.validateRepositoryId(repo) + + rc = self.remoteTools.downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile'],"/dev/null") + if rc != "-3": # cannot download database + self.syncErrors = True + return False + return True + + def clearRepositoryCache(self, repo): + + self.validateRepositoryId(repo) + + self.dumpTools.dumpobj(etpCache['dbInfo']+repo,{}) + + def validateCompressionMethod(self, repo): + + self.validateRepositoryId(repo) + + cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat']) + if cmethod == None: + raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.") + return cmethod + + def ensureRepositoryPath(self, repo): + + self.validateRepositoryId(repo) + + # create dir if it doesn't exist + if not os.path.isdir(etpRepositories[repo]['dbpath']): + os.makedirs(etpRepositories[repo]['dbpath']) + + # this function can be reimplemented + def downloadDatabase(self, repo, dbfilenameid): + + self.validateRepositoryId(repo) + + rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst[dbfilenameid], etpRepositories[repo]['dbpath'] + "/" + etpConst[dbfilenameid]) + if rc in ("-1","-2","-3"): + return False + return True + + def removeRepositoryFiles(self, repo, dbfilenameid): + + self.validateRepositoryId(repo) + + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]) + if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']): + os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']) + + def unpackDownloadedDatabase(self, repo, cmethod): + + self.validateRepositoryId(repo) + + import entropyTools + path = eval("entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]) + return path + + def downloadChecksum(self, repo): + + self.validateRepositoryId(repo) + + rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile'], etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile']) + if rc in ("-1","-2","-3"): + return False + return True + + def verifyDatabaseChecksum(self, repo): + + self.validateRepositoryId(repo) + + import entropyTools + try: + f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r") + md5hash = f.readline().strip() + md5hash = md5hash.split()[0] + f.close() + except: + return -1 + rc = entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash) + return rc + + def downloadDatabaseRevision(self, repo): + + self.validateRepositoryId(repo) + + rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile'], etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile']) + if rc in ("-1","-2","-3"): + return False + return True + + def closeTransactions(self): + + if not self.dbupdated: + return + + # safely clean ram caches + atomMatchCache.clear() + self.dumpTools.dumpobj(etpCache['atomMatch'],{}) + generateDependsTreeCache.clear() + self.dumpTools.dumpobj(etpCache['generateDependsTree'],{}) + for dbinfo in dbCacheStore: + dbCacheStore[dbinfo].clear() + self.dumpTools.dumpobj(dbinfo,{}) + + # clean caches + import cacheTools + cacheTools.generateCache(depcache = True, configcache = False) + + # clean resume caches + self.dumpTools.dumpobj(etpCache['install'],{}) + self.dumpTools.dumpobj(etpCache['world'],{}) + self.dumpTools.dumpobj(etpCache['remove'],[]) diff --git a/conf/repositories.conf b/conf/repositories.conf index 2e1515f59..313402a3d 100644 --- a/conf/repositories.conf +++ b/conf/repositories.conf @@ -7,10 +7,10 @@ # repository|servername (no spaces!)|server description| <...>| # # example: -# repository|sabayonlinux.org|Sabayon Linux Official Repository|http://www.sabayonlinux.org/packages|http://www.sabayonlinux.org/database +## repository|sabayonlinux.org|Sabayon Linux Official Repository|http://www.sabayonlinux.org/packages|http://www.sabayonlinux.org/database # # Sabayon Linux Official Repository -repository|sabayonlinux.org|Sabayon Linux Official Repository|http://svn.sabayonlinux.org/entropy http://cross-lfs.sabayonlinux.org/entropy http://mirror.holland.sabayonlinux.org/entropy http://na.mirror.garr.it/mirrors/sabayonlinux/entropy|http://svn.sabayonlinux.org/entropy#bz2 +repository|sabayonlinux.org|Sabayon Linux Official Repository|http://svn.sabayonlinux.org/entropy|http://svn.sabayonlinux.org/entropy#bz2 repository|sabayonlinux.org||http://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/entropy/| repository|sabayonlinux.org||http://cross-lfs.sabayonlinux.org/entropy| repository|sabayonlinux.org||http://mirror.holland.sabayonlinux.org/entropy| diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index b99b678a3..62061c53a 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -344,6 +344,7 @@ etpFileTransfer = { 'elapsed': 0.0, 'transferpollingtime': float(1)/4 # 0.25secs = 4Hz } +etpFileTransferMetadata = {} # disk caching dictionary etpCache = { @@ -453,6 +454,7 @@ portageFakeDbApi = {} # Client packages/database repositories etpRepositories = {} +etpRepositoriesExcluded = {} etpRepositoriesOrder = set() # remote section @@ -781,16 +783,26 @@ def initConfig_entropyConstants(rootdir): f = open(etpConst['repositoriesconf'],"r") repositoriesconf = f.readlines() f.close() - + # setup product first for line in repositoriesconf: if (line.strip().find("product|") != -1) and (not line.strip().startswith("#")) and (len(line.strip().split("|")) == 2): etpConst['product'] = line.strip().split("|")[1] - + for line in repositoriesconf: line = line.strip() # populate etpRepositories - if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5): + if (line.find("repository|") != -1) and (len(line.split("|")) == 5): + + excluded = False + myRepodata = etpRepositories + if line.startswith("##"): + continue + elif line.startswith("#"): + excluded = True + myRepodata = etpRepositoriesExcluded + line = line[1:] + reponame = line.split("|")[1] repodesc = line.split("|")[2] repopackages = line.split("|")[3] @@ -804,24 +816,37 @@ def initConfig_entropyConstants(rootdir): except: pass repodatabase = repodatabase[:dbformatcolon] - if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")): - stated = etpRepositories.get(reponame) - if stated == None: - etpRepositories[reponame] = {} - etpRepositories[reponame]['description'] = repodesc - etpRepositories[reponame]['packages'] = [] - etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch'] - etpRepositories[reponame]['dbcformat'] = dbformat - etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch'] + + if ((repopackages.startswith("http://") or repopackages.startswith("ftp://")) and \ + (repodatabase.startswith("http://") or repodatabase.startswith("ftp://"))) or \ + ((not repodatabase) and (myRepodata.has_key(reponame))): + + if not myRepodata.has_key(reponame): + myRepodata[reponame] = {} + myRepodata[reponame]['description'] = repodesc + myRepodata[reponame]['packages'] = [] + myRepodata[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch'] + myRepodata[reponame]['dbcformat'] = dbformat + myRepodata[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch'] + + myRepodata[reponame]['dbrevision'] = "0" + dbrevision_file = os.path.join(myRepodata[reponame]['dbpath'],etpConst['etpdatabaserevisionfile']) + if os.path.isfile(dbrevision_file): + rev_file = open(dbrevision_file,"r") + myRepodata[reponame]['dbrevision'] = rev_file.readline().strip() + rev_file.close() + del rev_file + # initialize CONFIG_PROTECT - will be filled the first time the db will be opened - etpRepositories[reponame]['configprotect'] = None - etpRepositories[reponame]['configprotectmask'] = None - - ordercount += 1 - etpRepositoriesOrder.add((ordercount,reponame)) + myRepodata[reponame]['configprotect'] = None + myRepodata[reponame]['configprotectmask'] = None + + if not excluded: + ordercount += 1 + etpRepositoriesOrder.add((ordercount,reponame)) for x in repopackages.split(): - etpRepositories[reponame]['packages'].append(x+"/"+etpConst['product']) + myRepodata[reponame]['packages'].append(x+"/"+etpConst['product']) elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): branch = line.split("|")[1] diff --git a/libraries/remoteTools.py b/libraries/remoteTools.py index 74f935bf0..9e41469d5 100644 --- a/libraries/remoteTools.py +++ b/libraries/remoteTools.py @@ -85,10 +85,8 @@ def downloadData(url, pathToSave, bufferSize = 8192, checksum = True, showSpeed # start scheduler if (showSpeed): - etpFileTransfer['datatransfer'] = 0 - etpFileTransfer['gather'] = 0 - etpFileTransfer['elapsed'] = 0.0 - speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransfer['transferpollingtime']) + etpFileTransferMetadata[url] = etpFileTransfer.copy() + speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransferMetadata[url]['transferpollingtime']) speedUpdater.setName("download") speedUpdater.start() @@ -133,6 +131,10 @@ def downloadData(url, pathToSave, bufferSize = 8192, checksum = True, showSpeed if (showSpeed): speedUpdater.kill() socket.setdefaulttimeout(2) + # remove status + # XXX: move this action somewhere else + del etpFileTransferMetadata[url] + return rc # Get the content of an online page @@ -218,8 +220,8 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = Tru for y in range(diffbarsize): bartext += " " if (showSpeed): - etpFileTransfer['gather'] = f.tell() - bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransfer['datatransfer']))+"/sec" + etpFileTransferMetadata[url]['gather'] = f.tell() + bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransferMetadata[url]['datatransfer']))+"/sec" else: bartext += "]" average = str(average) @@ -231,6 +233,6 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = Tru def __updateSpeedInfo(): - etpFileTransfer['elapsed'] += etpFileTransfer['transferpollingtime'] + etpFileTransferMetadata[url]['elapsed'] += etpFileTransferMetadata[url]['transferpollingtime'] # we have the diff size - etpFileTransfer['datatransfer'] = etpFileTransfer['gather'] / etpFileTransfer['elapsed'] + etpFileTransferMetadata[url]['datatransfer'] = etpFileTransferMetadata[url]['gather'] / etpFileTransferMetadata[url]['elapsed'] diff --git a/spritz/src/entropyapi.py b/spritz/src/entropyapi.py index 33454b2e9..a732b7e55 100644 --- a/spritz/src/entropyapi.py +++ b/spritz/src/entropyapi.py @@ -19,10 +19,6 @@ import logging -#import yum -#import yum.Errors as Errors -#from yum.update_md import UpdateMetadata - from misc import const,cleanMarkupSting from i18n import _ from yumgui import format_number @@ -31,6 +27,10 @@ from callbacks import * from dialogs import questionDialog from urlgrabber.grabber import URLGrabError +# Entropy Imports +from entropyConstants import * +from clientConstants import * +import repositoriesTools class YumexYumHandler: def __init__(self,recent,settings,progress,mainwin,parser): @@ -487,3 +487,16 @@ class YumexYumHandler: return out +class repositoryGuiController(repositoriesTools.repositoryController): + """ hello world """ + + # reimplementing download function + # this function can be reimplemented + def downloadDatabase(self, repo, dbfilenameid): + + self.validateRepositoryId(repo) + + rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst[dbfilenameid], etpRepositories[repo]['dbpath'] + "/" + etpConst[dbfilenameid]) + if rc in ("-1","-2","-3"): + return False + return True diff --git a/spritz/src/gui.py b/spritz/src/gui.py index 952a185f3..f8e9fe9b1 100644 --- a/spritz/src/gui.py +++ b/spritz/src/gui.py @@ -225,7 +225,7 @@ class YumexProgress: def setTotal( self, now, total ): self.total.setProgress( now, total ) - def set_progress( self, frac, text=None ): + def set_progress( self, frac, text=None ): if self.parent.quitNow: self.parent.exitNow() # Skip if fraction not have changed diff --git a/spritz/src/views.py b/spritz/src/views.py index 7fc7a1747..4d14f1768 100644 --- a/spritz/src/views.py +++ b/spritz/src/views.py @@ -439,7 +439,6 @@ class YumexRepoView: self.headers = [_('Repository'),_('Filename')] self.store = self.setup_view() - def on_toggled( self, widget, path): """ Repo select/unselect handler """ iter = self.store.get_iter( path ) @@ -448,31 +447,25 @@ class YumexRepoView: def setup_view( self ): """ Create models and columns for the Repo TextView """ - store = gtk.ListStore( 'gboolean', gobject.TYPE_STRING,gobject.TYPE_STRING,'gboolean') + store = gtk.ListStore( 'gboolean', gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING) self.view.set_model( store ) + # Setup Selection Column cell1 = gtk.CellRendererToggle() # Selection cell1.set_property( 'activatable', True ) - column1 = gtk.TreeViewColumn( " ", cell1 ) + column1 = gtk.TreeViewColumn( "Active", cell1 ) column1.add_attribute( cell1, "active", 0 ) column1.set_resizable( True ) - column1.set_sort_column_id( -1 ) + column1.set_sort_column_id( -1 ) self.view.append_column( column1 ) - cell1.connect( "toggled", self.on_toggled ) - # Setup resent column - cell2 = gtk.CellRendererPixbuf() # gpgcheck - cell2.set_property( 'stock-id', gtk.STOCK_DIALOG_AUTHENTICATION ) - column2 = gtk.TreeViewColumn( "", cell2 ) - column2.set_cell_data_func( cell2, self.new_pixbuf ) - column2.set_sizing( gtk.TREE_VIEW_COLUMN_FIXED ) - column2.set_fixed_width( 20 ) - column2.set_sort_column_id( -1 ) - self.view.append_column( column2 ) - column2.set_clickable( True ) - + cell1.connect( "toggled", self.on_toggled ) + + # Setup revision column + self.create_text_column( _('Revision'),1 ) + # Setup reponame & repofile column's - self.create_text_column( _('Repository Identifier'),1 ) - self.create_text_column( _('Description'),2 ) + self.create_text_column( _('Repository Identifier'),2 ) + self.create_text_column( _('Description'),3 ) self.view.set_search_column( 1 ) self.view.set_reorderable( False ) return store @@ -481,7 +474,7 @@ class YumexRepoView: cell = gtk.CellRendererText() # Size Column column = gtk.TreeViewColumn( hdr, cell, text=colno ) column.set_resizable( True ) - self.view.append_column( column ) + self.view.append_column( column ) def populate(self): """ Populate a repo liststore with data """ @@ -490,7 +483,10 @@ class YumexRepoView: repos.sort() for repo in etpRepositoriesOrder: repodata = etpRepositories[repo[1]] - self.store.append([1,repo[1],repodata['description'],"200x"]) + self.store.append([1,repodata['dbrevision'],repo[1],repodata['description']]) + # excluded ones + for repo in etpRepositoriesExcluded: + self.store.append([0,etpRepositoriesExcluded['dbrevision'],repo,etpRepositoriesExcluded['description']]) def new_pixbuf( self, column, cell, model, iter ): gpg = model.get_value( iter, 3 ) @@ -503,7 +499,7 @@ class YumexRepoView: selected = [] for elem in self.store: state = elem[0] - name = elem[1] + name = elem[2] if state: selected.append( name ) return selected @@ -570,7 +566,7 @@ class YumexPluginView: column1.add_attribute( cell1, "active", 0 ) column1.set_resizable( True ) self.view.append_column( column1 ) - cell1.connect( "toggled", self.on_toggled, model ) + cell1.connect( "toggled", self.on_toggled, model ) cell2 = gtk.CellRendererText() column2= gtk.TreeViewColumn( _( "Plugin" ), cell2, text=1 ) column2.set_resizable( True ) diff --git a/spritz/src/yumex.glade b/spritz/src/yumex.glade index 2a75f6532..54fe81e8f 100644 --- a/spritz/src/yumex.glade +++ b/spritz/src/yumex.glade @@ -323,8 +323,8 @@ True True - GTK_POLICY_ALWAYS - GTK_POLICY_ALWAYS + GTK_POLICY_AUTOMATIC + GTK_POLICY_AUTOMATIC GTK_SHADOW_IN GTK_CORNER_TOP_LEFT @@ -357,16 +357,16 @@ 6 - + True True True GTK_RELIEF_NORMAL True - + - + True 0.5 0.5 @@ -378,15 +378,15 @@ 0 - + True False 2 - + True - gtk-cancel + gtk-refresh 4 0.5 0.5 @@ -401,9 +401,9 @@ - + True - _Deselect All + Update Repositories True False GTK_JUSTIFY_LEFT @@ -430,19 +430,6 @@ - - - - True - True - True - gtk-refresh - True - GTK_RELIEF_NORMAL - True - - - 5 diff --git a/spritz/src/yumex.py b/spritz/src/yumex.py index 05be3be14..32133f9f2 100755 --- a/spritz/src/yumex.py +++ b/spritz/src/yumex.py @@ -24,9 +24,13 @@ import logging import traceback # Entropy Imports +sys.path.append("../../libraries") +sys.path.append("../../client") sys.path.append("/usr/lib/entropy/libraries") sys.path.append("/usr/lib/entropy/client") from entropyConstants import * +from clientConstants import * +import exceptionTools # GTK Imports import gtk,gobject @@ -38,7 +42,7 @@ from yumgui import * # yumex imports import filters -from entropyapi import YumexYumHandler +from entropyapi import YumexYumHandler, repositoryGuiController from gui import YumexGUI from dialogs import * from misc import const,YumexOptions,YumexProfile @@ -105,7 +109,7 @@ class YumexController(Controller): repos = self.repoView.get_selected() self.setPage('output') self.logger.info( "Enabled repositories : %s" % ",".join(repos)) - self.setupYum() + self.updateRepositories(repos) def on_repoDeSelect_clicked(self,widget): self.repoView.deselect_all() @@ -116,13 +120,9 @@ class YumexController(Controller): self.queueView.deleteSelected() def on_queueQuickAdd_activate(self,widget): - if self.initYum: # Yum has to be initialised - txt = widget.get_text() - arglist = txt.split(' ') - self.doQuickAdd(arglist[0],arglist[1:]) - else: - self.setStatus(_('Yum has not been initialized yet')) - widget.set_text('') + txt = widget.get_text() + arglist = txt.split(' ') + self.doQuickAdd(arglist[0],arglist[1:]) def on_queueProcess_clicked( self, widget ): """ Process Queue Button Handler """ @@ -143,7 +143,7 @@ class YumexController(Controller): else: self.queue.clear() # Clear package queue self.queueView.refresh() # Refresh Package Queue - self.setupYum() + #self.setupYum() def on_queueSave_clicked( self, widget ): dialog = gtk.FileChooserDialog(title=None,action=gtk.FILE_CHOOSER_ACTION_SAVE, @@ -399,11 +399,7 @@ class YumexApplication(YumexController,YumexGUI): # setup Repositories self.setupRepoView() self.firstTime = True - self.initYum = False - if self.settings.autorefresh: - self.setupYum() - else: - self.setPage('repos') + self.setPage('repos') def startWorking(self): self.isWorking = True @@ -417,25 +413,65 @@ class YumexApplication(YumexController,YumexGUI): normalCursor(self.ui.main) gtkEventThread.endProcessing() - def setupYum(self): + def updateRepositories(self, repos): self.setPage('output') self.startWorking() - self.progress.total.setup( const.SETUP_PROGRESS_STEPS ) - # Run yum setup in background thread + + # set steps + self.progress.total.setup( range(len(repos)) ) + self.progressLog(_('Initializing Repository module...')) + + # 1 + import repositoriesTools try: - if self.firstTime: - self.yumbase._setupBase() - self.firstTime = False - else: - self.yumbase._setupAgain() + repoConn = repositoryGuiController(repos, forceUpdate = True) + except exceptionTools.PermissionDenied: + self.progressLog(_('You must run this application as root')) + return 1 + except exceptionTools.MissingParameter: + self.progressLog(_('No repositories specified in %s') % (etpConst['repositoriesconf'],)) + return 127 + except exceptionTools.OnlineMirrorError: + self.progressLog(_('You are not connected to the Internet. You should.')) + return 126 except Exception, e: - self.endWorking() - errorMessage( self.ui.main, _( "Error" ), _( "Error in Yum Setup" ), str(e) ) - self.logger.error(str(e)) - return - # prepare package lists - self.progress.total.next() # -> List setup - self.progressLog(_('Building Package Lists')) + self.progressLog(_('Unhandled exception: %s') % (str(e),)) + return 2 + + # 2 + error = False + for repo in repoConn.reponames: + + self.progressLog(_('Fetching repository id: %s') % (repo,)) + update = repoConn.isRepositoryUpdatable(repo) + if not update: + self.progressLog(_('Attention: database already up to date for: %s') % (repo,)) + time.sleep(1) + error = True + continue + + unlocked = repoConn.isRepositoryUnlocked(repo) + if not unlocked: + self.progressLog(_('Attention: repository %s is being updated. Try again in a few minutes.') % (repo,)) + time.sleep(1) + error = True + continue + + # database is going to be updated + repoConn.dbupdated = True + # clear database interface cache belonging to this repository + repoConn.clearRepositoryCache(repo) + cmethod = repoConn.validateCompressionMethod(repo) + repoConn.ensureRepositoryPath(repo) + + self.progressLog(_('Downloading database for %s') % (repo,)) + # download + down_status = repoConn.downloadDatabase(repo, cmethod[2]) + if not down_status: + self.progressLog(_('Attention: repository %s does not exist online.') % (repo,)) + error = True + continue + #self.yumbase.populatePackages(['installed','updates','available']) self.progressLog(_('Building Package Lists Completed')) self.progressLog(_('Building Groups Lists')) @@ -446,9 +482,10 @@ class YumexApplication(YumexController,YumexGUI): self.doProgress = True #self.addPackages() self.doProgress = False - self.setPage('packages') - self.initYum = True + if not error: + self.setPage('repos') self.endWorking() + initConfig_entropyConstants(etpSys['rootdir']) def setupRepoView(self): @@ -465,7 +502,7 @@ class YumexApplication(YumexController,YumexGUI): masks = [action] self.pkgView.store.clear() allpkgs = [] - if self.doProgress: self.progress.total.next() # -> Get lists + if self.doProgress: self.progress.total.next() # -> Get lists for flt in masks: msg = _('Getting packages : %s' ) % flt self.progressLog(msg) @@ -473,7 +510,7 @@ class YumexApplication(YumexController,YumexGUI): pkgs = self.yumbase.getPackages(flt) self.progressLog(_('Found %d %s packages') % (len(pkgs),flt)) allpkgs.extend(pkgs) - if self.doProgress: self.progress.total.next() # -> Sort Lists + if self.doProgress: self.progress.total.next() # -> Sort Lists self.progressLog(_('Sorting packages')) allpkgs.sort() self.progressLog(_('Population view with packages'))