- started to work out spritz dependencies and update repositories function

- reimplemented repositoriesTools in a OO way, creating repositoryController class git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@961 cd1c1023-2f26-0410-ae45-c471fc1f0318
2007-12-29 01:55:01 +00:00
parent 891a64f2a6
commit 34995bce94
9 changed files with 489 additions and 293 deletions
@@ -25,32 +25,29 @@
 ##   Repositories Tools
 #

-import os
 from entropyConstants import *
 from clientConstants import *
 from outputTools import *
 import entropyTools
-from remoteTools import downloadData, getOnlineContent
-import dumpTools
 import exceptionTools

 def repositories(options):
-    
+
    # Options available for all the packages submodules
    myopts = options[1:]
    equoRequestForceUpdate = False
    rc = 0
    for opt in myopts:
-	if (opt == "--force"):
-	    equoRequestForceUpdate = True
+        if (opt == "--force"):
+            equoRequestForceUpdate = True

    if (options[0] == "update"):
-	rc = syncRepositories(forceUpdate = equoRequestForceUpdate)
+        rc = syncRepositories(forceUpdate = equoRequestForceUpdate)
    elif (options[0] == "status"):
-	for repo in etpRepositories:
-	    showRepositoryInfo(repo)
+        for repo in etpRepositories:
+            showRepositoryInfo(repo)
    elif (options[0] == "repoinfo"):
-	showRepositories()
+        showRepositories()
    else:
        rc = -10
    return rc
@@ -60,32 +57,32 @@ def showRepositories():
    print_info(darkred(" * ")+darkgreen("Active Repositories:"))
    repoNumber = 0
    for repo in etpRepositories:
-	repoNumber += 1
-	print_info(blue("\t#"+str(repoNumber))+bold(" "+etpRepositories[repo]['description']))
-	sourcecount = 0
-	for pkgrepo in etpRepositories[repo]['packages']:
-	    sourcecount += 1
-	    print_info(red("\t\tPackages Mirror #"+str(sourcecount)+" : ")+darkgreen(pkgrepo))
-	print_info(red("\t\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database']))
-	print_info(red("\t\tRepository name: ")+bold(repo))
-	print_info(red("\t\tRepository database path: ")+blue(etpRepositories[repo]['dbpath']))
+        repoNumber += 1
+        print_info(blue("\t#"+str(repoNumber))+bold(" "+etpRepositories[repo]['description']))
+        sourcecount = 0
+        for pkgrepo in etpRepositories[repo]['packages']:
+            sourcecount += 1
+            print_info(red("\t\tPackages Mirror #"+str(sourcecount)+" : ")+darkgreen(pkgrepo))
+        print_info(red("\t\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database']))
+        print_info(red("\t\tRepository name: ")+bold(repo))
+        print_info(red("\t\tRepository database path: ")+blue(etpRepositories[repo]['dbpath']))
    return 0

 def showRepositoryInfo(reponame):
    repoNumber = 0
    for repo in etpRepositories:
-	repoNumber += 1
-	if repo == reponame:
-	    break
+        repoNumber += 1
+        if repo == reponame:
+            break
    print_info(blue("#"+str(repoNumber))+bold(" "+etpRepositories[reponame]['description']))
    if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasefile']):
-	status = "active"
+        status = "active"
    else:
-	status = "never synced"
+        status = "never synced"
    print_info(darkgreen("\tStatus: ")+darkred(status))
    urlcount = 0
    for repourl in etpRepositories[reponame]['packages'][::-1]:
-	urlcount += 1
+        urlcount += 1
        print_info(red("\tPackages URL #"+str(urlcount)+": ")+darkgreen(repourl))
    print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[reponame]['database']))
    print_info(red("\tRepository name: ")+bold(reponame))
@@ -101,187 +98,143 @@ def showRepositoryInfo(reponame):
 # @returns int>0 if the file exists
 def getRepositoryRevision(reponame):
    if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
-	f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r")
-	revision = int(f.readline().strip())
-	f.close()
+        f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r")
+        revision = int(f.readline().strip())
+        f.close()
    else:
-	revision = -1
+        revision = -1
    return revision

-# @returns -1 if the file is not available
-# @returns int>0 if the revision has been retrieved
-def getOnlineRepositoryRevision(reponame):
-    url = etpRepositories[reponame]['database']+"/"+etpConst['etpdatabaserevisionfile']
-    status = getOnlineContent(url)
-    if (status):
-	status = status[0].strip()
-	return int(status)
-    else:
-	return -1
-
 # @returns -1 if the file does not exist
 # @returns int>0 if the file exists
 def getRepositoryDbFileHash(reponame):
    if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
-	f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
-	mhash = f.readline().strip().split()[0]
-	f.close()
+        f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
+        mhash = f.readline().strip().split()[0]
+        f.close()
    else:
-	mhash = "-1"
+        mhash = "-1"
    return mhash

 def syncRepositories(reponames = [], forceUpdate = False):

-    # check if I am root
-    if (not entropyTools.isRoot()):
-	print_error(red("\t You must run this application as root."))
-	return 1
-
-    # check etpRepositories
-    if len(etpRepositories) == 0:
-	if (not etpUi['quiet']):
-	    print_error(darkred(" * ")+red("No repositories specified in ")+etpConst['repositoriesconf'])
-	return 127
+    # load repository class
+    try:
+        repoConn = repositoryController(reponames, forceUpdate)
+    except exceptionTools.PermissionDenied:
+        print_error(red("\t You must run this application as root."))
+        return 1
+    except exceptionTools.MissingParameter:
+        print_error(darkred(" * ")+red("No repositories specified in ")+etpConst['repositoriesconf'])
+        return 127
+    except exceptionTools.OnlineMirrorError:
+        print_error(darkred(" @@ ")+darkgreen("You are not connected to the Internet. You should."))
+        return 126
+    except Exception, e:
+        print_error(darkred(" @@ ")+darkgreen("Unhandled exception: %s" % (str(e),)))
+        return 2

+    # let's dance!
    if (not etpUi['quiet']):
        print_info(darkred(" @@ ")+darkgreen("Repositories syncronization..."))
+
    repoNumber = 0
-    syncErrors = False
-    
-    if (not reponames):
-	for x in etpRepositories:
-	    reponames.append(x)
-    
-    dbupdated = False
-    
-    # Test network connectivity
-    conntest = getOnlineContent("http://svn.sabayonlinux.org")
-    if not conntest:
-	print_info(darkred(" @@ ")+darkgreen("You are not connected to the Internet. You should."))
-	return 2
-    
-    for repo in reponames:
-	
-	repoNumber += 1
-	
-	if (not etpUi['quiet']):
-	    print_info(blue("  #"+str(repoNumber))+bold(" "+etpRepositories[repo]['description']))
-	    print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database']))
-	    print_info(red("\tDatabase local path: ")+darkgreen(etpRepositories[repo]['dbpath']))
-	
-	# check if database is already updated to the latest revision
-	onlinestatus = getOnlineRepositoryRevision(repo)
-	if (onlinestatus != -1):
-	    localstatus = getRepositoryRevision(repo)
-	    if (localstatus == onlinestatus) and (not forceUpdate):
-		if (not etpUi['quiet']):
-		    print_info(bold("\tAttention: ")+red("database is already up to date."))
-		continue
-	
-	# get database lock
-	rc = downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile'],"/dev/null")
-	if rc != "-3": # cannot download database
-	    if (not etpUi['quiet']):
-	        print_error(bold("\tATTENTION -> ")+red("repository is being updated. Try again in a few minutes."))
-	    syncErrors = True
-	    continue
-	
-	# database is going to be updated
-	dbupdated = True
-	# clear database interface cache belonging to this repository
-	dumpTools.dumpobj(etpCache['dbInfo']+repo,{})
-	
-	cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat'])
-	if cmethod == None: raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
-	
-	# starting to download
-	if (not etpUi['quiet']):
-	    print_info(red("\tDownloading database ")+darkgreen(etpConst[cmethod[2]])+red(" ..."))
-	# create dir if it doesn't exist
-	if not os.path.isdir(etpRepositories[repo]['dbpath']):
-	    if (not etpUi['quiet']):
-	        print_info(red("\t\tCreating database directory..."))
-	    os.makedirs(etpRepositories[repo]['dbpath'])
-	# download
-	rc = downloadData(etpRepositories[repo]['database']+"/"+etpConst[cmethod[2]],etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]])
-        if rc in ("-1","-2","-3"):
+    for repo in repoConn.reponames:
+
+        repoNumber += 1
+
+        if (not etpUi['quiet']):
+            print_info(blue("  #"+str(repoNumber))+bold(" "+etpRepositories[repo]['description']))
+            print_info(red("\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database']))
+            print_info(red("\tDatabase local path: ")+darkgreen(etpRepositories[repo]['dbpath']))
+
+        # check if database is already updated to the latest revision
+        update = repoConn.isRepositoryUpdatable(repo)
+        if not update:
+            if not etpUi['quiet']: print_info(bold("\tAttention: ")+red("database is already up to date."))
+            continue
+
+        # get database lock
+        unlocked = repoConn.isRepositoryUnlocked(repo)
+        if not unlocked:
+            if not etpUi['quiet']: print_error(bold("\tATTENTION -> ")+red("repository is being updated. Try again in a few minutes."))
+            continue
+
+        # database is going to be updated
+        repoConn.dbupdated = True
+        # clear database interface cache belonging to this repository
+        repoConn.clearRepositoryCache(repo)
+        cmethod = repoConn.validateCompressionMethod(repo)
+        repoConn.ensureRepositoryPath(repo)
+
+        # starting to download
+        if (not etpUi['quiet']):
+            print_info(red("\tDownloading database ")+darkgreen(etpConst[cmethod[2]])+red(" ..."))
+
+        down_status = repoConn.downloadDatabase(repo, cmethod[2])
+        if not down_status:
            print_info(bold("\tAttention: ")+red("database does not exist online."))
-            return 128
-	
-	if (not etpUi['quiet']):
-	    print_info(red("\tUnpacking database to ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."))
-	eval("entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]])
-	# download etpdatabasehashfile
-	if (not etpUi['quiet']):
-	    print_info(red("\tDownloading checksum ")+darkgreen(etpConst['etpdatabasehashfile'])+red(" ..."))
-	downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasehashfile'],etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'])
-	# checking checksum
-	if (not etpUi['quiet']):
-	    print_info(red("\tChecking downloaded database ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."), back = True)
-	f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
-	md5hash = f.readline().strip()
-	md5hash = md5hash.split()[0]
-	f.close()
-	rc = entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash)
-	if rc:
-	    if (not etpUi['quiet']):
-	        print_info(red("\tDownloaded database status: ")+bold("OK"))
-	else:
-	    if (not etpUi['quiet']):
-	        print_error(red("\tDownloaded database status: ")+darkred("ERROR"))
-	        print_error(red("\t An error occured while checking database integrity"))
-	    # delete all
-	    if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
-		os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'])
-	    if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]):
-		os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]])
-	    if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
-		os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
-	    syncErrors = True
-	    continue
-	
-	# download etpdatabaserevisionfile
-	if (not etpUi['quiet']):
-	    print_info(red("\tDownloading revision ")+darkgreen(etpConst['etpdatabaserevisionfile'])+red(" ..."))
-	downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile'],etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
-	
-	if (not etpUi['quiet']):
-	    print_info(red("\tUpdated repository revision: ")+bold(str(getRepositoryRevision(repo))))
-	print_info(darkgreen("\tUpdate completed"))
+            continue

-    if syncErrors:
-	if (not etpUi['quiet']):
-	    print_warning(darkred(" @@ ")+red("Something bad happened. Please have a look."))
-	return 128
+        # unpack database
+        if (not etpUi['quiet']):
+            print_info(red("\tUnpacking database to ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."))
+        # unpack database
+        repoConn.unpackDownloadedDatabase(repo,cmethod)
+
+        # download checksum
+        if (not etpUi['quiet']):
+            print_info(red("\tDownloading checksum ")+darkgreen(etpConst['etpdatabasehashfile'])+red(" ..."))
+        down_status = repoConn.downloadChecksum(repo)
+        if not down_status:
+            print_warning(red("\tCannot fetch checksum. Cannot verify database integrity !"))
+        else:
+            # verify checksum
+            if (not etpUi['quiet']):
+                print_info(red("\tChecking downloaded database ")+darkgreen(etpConst['etpdatabasefile'])+red(" ..."), back = True)
+            db_status = repoConn.verifyDatabaseChecksum(repo)
+            if db_status == -1:
+                print_warning(red("\tCannot open digest. Cannot verify database integrity !"))
+            elif db_status:
+                if (not etpUi['quiet']):
+                    print_info(red("\tDownloaded database status: ")+bold("OK"))
+            else:
+                if (not etpUi['quiet']):
+                    print_error(red("\tDownloaded database status: ")+darkred("ERROR"))
+                    print_error(red("\t An error occured while checking database integrity"))
+                # delete all
+                repoConn.removeRepositoryFiles(repo, cmethod[2])
+                repoConn.syncErrors = True
+                continue
+
+        # download revision
+        if (not etpUi['quiet']):
+            print_info(red("\tDownloading revision ")+darkgreen(etpConst['etpdatabaserevisionfile'])+red(" ..."))
+        rev_status = repoConn.downloadDatabaseRevision(repo)
+        if not rev_status:
+            print_warning(red("\tCannot download repository revision, don't ask me why !"))
+        else:
+            if not etpUi['quiet']: print_info(red("\tUpdated repository revision: ")+bold(str(getRepositoryRevision(repo))))
+
+        print_info(darkgreen("\tUpdate completed"))
+
+    repoConn.closeTransactions()
+
+    if repoConn.syncErrors:
+        if (not etpUi['quiet']):
+            print_warning(darkred(" @@ ")+red("Something bad happened. Please have a look."))
+        del repoConn
+        return 128

-    if (dbupdated):
-	
-	# safely clean ram caches
-	atomMatchCache.clear()
-	dumpTools.dumpobj(etpCache['atomMatch'],{})
-	generateDependsTreeCache.clear()
-	dumpTools.dumpobj(etpCache['generateDependsTree'],{})
-        for dbinfo in dbCacheStore:
-            dbCacheStore[dbinfo].clear()
-            dumpTools.dumpobj(dbinfo,{})
-        
-	# clean caches
-        import cacheTools
-        cacheTools.generateCache(depcache = True, configcache = False)
-        
-        # clean resume caches
-        dumpTools.dumpobj(etpCache['install'],{})
-        dumpTools.dumpobj(etpCache['world'],{})
-        dumpTools.dumpobj(etpCache['remove'],[])
-        
    # tell if a new equo release is available
    import equoTools
    from databaseTools import openClientDatabase
    try:
        clientDbconn = openClientDatabase(xcache = False)
    except exceptionTools.SystemDatabaseError:
+        del repoConn
        return 0
-    
+
    matches = clientDbconn.searchPackages("app-admin/equo")
    if matches:
        equo_match = "<="+matches[0][0]
@@ -291,7 +244,190 @@ def syncRepositories(reponames = [], forceUpdate = False):
            print_warning(darkred(" !! ")+blue("A new version of ")+bold("equo")+blue(" is available. Please ")+bold("install it")+blue(" before any other package."))
        del matches
        del equo_unsatisfied
+
    clientDbconn.closeDB()
    del clientDbconn
+    del repoConn

-    return 0
+    return 0
+
+#
+# repository control class, that's it
+#
+class repositoryController:
+
+    def __init__(self, reponames = [], forceUpdate = False):
+
+        self.reponames = reponames
+        self.forceUpdate = forceUpdate
+        self.syncErrors = False
+        self.dbupdated = False
+
+        import remoteTools
+        import dumpTools
+        self.remoteTools = remoteTools
+        self.dumpTools = dumpTools
+
+        # check if I am root
+        if (not entropyTools.isRoot()):
+            raise exceptionTools.PermissionDenied("PermissionDenied: not allowed as user.")
+
+        # check etpRepositories
+        if not etpRepositories:
+            raise exceptionTools.MissingParameter("MissingParameter: no repositories specified in %s" % (etpConst['repositoriesconf'],))
+
+        # Test network connectivity
+        conntest = self.remoteTools.getOnlineContent("http://svn.sabayonlinux.org")
+        if not conntest:
+            raise exceptionTools.OnlineMirrorError("OnlineMirrorError: you are not connected to the Internet. You should.")
+
+        if (not self.reponames):
+            for x in etpRepositories:
+                self.reponames.append(x)
+
+    def validateRepositoryId(self, repoid):
+        if repoid not in self.reponames:
+            raise exceptionTools.InvalidData("InvalidData: repository is not listed in self.reponames")
+
+    # @returns -1 if the file is not available
+    # @returns int>0 if the revision has been retrieved
+    def getOnlineRepositoryRevision(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile']
+        status = self.remoteTools.getOnlineContent(url)
+        if (status):
+            status = status[0].strip()
+            return int(status)
+        else:
+            return -1
+
+    def isRepositoryUpdatable(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        onlinestatus = self.getOnlineRepositoryRevision(repo)
+        if (onlinestatus != -1):
+            localstatus = getRepositoryRevision(repo)
+            if (localstatus == onlinestatus) and (not self.forceUpdate):
+                return False
+        return True
+
+    def isRepositoryUnlocked(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        rc = self.remoteTools.downloadData(etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile'],"/dev/null")
+        if rc != "-3": # cannot download database
+            self.syncErrors = True
+            return False
+        return True
+
+    def clearRepositoryCache(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        self.dumpTools.dumpobj(etpCache['dbInfo']+repo,{})
+
+    def validateCompressionMethod(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat'])
+        if cmethod == None:
+            raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
+        return cmethod
+
+    def ensureRepositoryPath(self, repo):
+
+        self.validateRepositoryId(repo)
+
+	# create dir if it doesn't exist
+	if not os.path.isdir(etpRepositories[repo]['dbpath']):
+	    os.makedirs(etpRepositories[repo]['dbpath'])
+
+    # this function can be reimplemented
+    def downloadDatabase(self, repo, dbfilenameid):
+
+        self.validateRepositoryId(repo)
+
+	rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] +   "/" + etpConst[dbfilenameid], etpRepositories[repo]['dbpath'] + "/" + etpConst[dbfilenameid])
+        if rc in ("-1","-2","-3"):
+            return False
+        return True
+
+    def removeRepositoryFiles(self, repo, dbfilenameid):
+
+        self.validateRepositoryId(repo)
+
+        if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
+            os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'])
+        if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]):
+            os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid])
+        if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
+            os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
+
+    def unpackDownloadedDatabase(self, repo, cmethod):
+
+        self.validateRepositoryId(repo)
+
+        import entropyTools
+        path = eval("entropyTools."+cmethod[1])(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]])
+        return path
+
+    def downloadChecksum(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile'], etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile'])
+        if rc in ("-1","-2","-3"):
+            return False
+        return True
+
+    def verifyDatabaseChecksum(self, repo):
+
+        self.validateRepositoryId(repo)
+
+        import entropyTools
+        try:
+            f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
+            md5hash = f.readline().strip()
+            md5hash = md5hash.split()[0]
+            f.close()
+        except:
+            return -1
+	rc = entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasefile'],md5hash)
+        return rc
+
+    def downloadDatabaseRevision(self, repo):
+
+        self.validateRepositoryId(repo)
+
+	rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile'], etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile'])
+        if rc in ("-1","-2","-3"):
+            return False
+        return True
+
+    def closeTransactions(self):
+
+        if not self.dbupdated:
+            return
+
+        # safely clean ram caches
+        atomMatchCache.clear()
+        self.dumpTools.dumpobj(etpCache['atomMatch'],{})
+        generateDependsTreeCache.clear()
+        self.dumpTools.dumpobj(etpCache['generateDependsTree'],{})
+        for dbinfo in dbCacheStore:
+            dbCacheStore[dbinfo].clear()
+            self.dumpTools.dumpobj(dbinfo,{})
+    
+        # clean caches
+        import cacheTools
+        cacheTools.generateCache(depcache = True, configcache = False)
+    
+        # clean resume caches
+        self.dumpTools.dumpobj(etpCache['install'],{})
+        self.dumpTools.dumpobj(etpCache['world'],{})
+        self.dumpTools.dumpobj(etpCache['remove'],[])
@@ -7,10 +7,10 @@
 #    repository|servername (no spaces!)|server description|<packages mirror 1> <packages mirror 2> <...>|<database uri>
 #
 #    example:
-#    repository|sabayonlinux.org|Sabayon Linux Official Repository|http://www.sabayonlinux.org/packages|http://www.sabayonlinux.org/database
+##   repository|sabayonlinux.org|Sabayon Linux Official Repository|http://www.sabayonlinux.org/packages|http://www.sabayonlinux.org/database
 #
 # Sabayon Linux Official Repository
-repository|sabayonlinux.org|Sabayon Linux Official Repository|http://svn.sabayonlinux.org/entropy http://cross-lfs.sabayonlinux.org/entropy http://mirror.holland.sabayonlinux.org/entropy http://na.mirror.garr.it/mirrors/sabayonlinux/entropy|http://svn.sabayonlinux.org/entropy#bz2
+repository|sabayonlinux.org|Sabayon Linux Official Repository|http://svn.sabayonlinux.org/entropy|http://svn.sabayonlinux.org/entropy#bz2
 repository|sabayonlinux.org||http://ftp.cc.uoc.gr/mirrors/linux/SabayonLinux/entropy/|
 repository|sabayonlinux.org||http://cross-lfs.sabayonlinux.org/entropy|
 repository|sabayonlinux.org||http://mirror.holland.sabayonlinux.org/entropy|
@@ -344,6 +344,7 @@ etpFileTransfer = {
    'elapsed': 0.0,
    'transferpollingtime': float(1)/4 # 0.25secs = 4Hz
 }
+etpFileTransferMetadata = {}

 # disk caching dictionary
 etpCache = {
@@ -453,6 +454,7 @@ portageFakeDbApi = {}

 # Client packages/database repositories
 etpRepositories = {}
+etpRepositoriesExcluded = {}
 etpRepositoriesOrder = set()

 # remote section
@@ -781,16 +783,26 @@ def initConfig_entropyConstants(rootdir):
        f = open(etpConst['repositoriesconf'],"r")
        repositoriesconf = f.readlines()
        f.close()
-        
+
        # setup product first
        for line in repositoriesconf:
            if (line.strip().find("product|") != -1) and (not line.strip().startswith("#")) and (len(line.strip().split("|")) == 2):
                etpConst['product'] = line.strip().split("|")[1]
-        
+
        for line in repositoriesconf:
            line = line.strip()
            # populate etpRepositories
-            if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5):
+            if (line.find("repository|") != -1) and (len(line.split("|")) == 5):
+
+                excluded = False
+                myRepodata = etpRepositories
+                if line.startswith("##"):
+                    continue
+                elif line.startswith("#"):
+                    excluded = True
+                    myRepodata = etpRepositoriesExcluded
+                    line = line[1:]
+
                reponame = line.split("|")[1]
                repodesc = line.split("|")[2]
                repopackages = line.split("|")[3]
@@ -804,24 +816,37 @@ def initConfig_entropyConstants(rootdir):
                        except:
                            pass
                    repodatabase = repodatabase[:dbformatcolon]
-                if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")):
-                    stated = etpRepositories.get(reponame)
-                    if stated == None:
-                        etpRepositories[reponame] = {}
-                        etpRepositories[reponame]['description'] = repodesc
-                        etpRepositories[reponame]['packages'] = []
-                        etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch']
-                        etpRepositories[reponame]['dbcformat'] = dbformat
-                        etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch']
+
+                if ((repopackages.startswith("http://") or repopackages.startswith("ftp://")) and \
+                        (repodatabase.startswith("http://") or repodatabase.startswith("ftp://"))) or \
+                         ((not repodatabase) and (myRepodata.has_key(reponame))):
+
+                    if not myRepodata.has_key(reponame):
+                        myRepodata[reponame] = {}
+                        myRepodata[reponame]['description'] = repodesc
+                        myRepodata[reponame]['packages'] = []
+                        myRepodata[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch']
+                        myRepodata[reponame]['dbcformat'] = dbformat
+                        myRepodata[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch']
+
+                        myRepodata[reponame]['dbrevision'] = "0"
+                        dbrevision_file = os.path.join(myRepodata[reponame]['dbpath'],etpConst['etpdatabaserevisionfile'])
+                        if os.path.isfile(dbrevision_file):
+                            rev_file = open(dbrevision_file,"r")
+                            myRepodata[reponame]['dbrevision'] = rev_file.readline().strip()
+                            rev_file.close()
+                            del rev_file
+
                        # initialize CONFIG_PROTECT - will be filled the first time the db will be opened
-                        etpRepositories[reponame]['configprotect'] = None
-                        etpRepositories[reponame]['configprotectmask'] = None
-                    
-                    ordercount += 1
-                    etpRepositoriesOrder.add((ordercount,reponame))
+                        myRepodata[reponame]['configprotect'] = None
+                        myRepodata[reponame]['configprotectmask'] = None
+
+                        if not excluded:
+                            ordercount += 1
+                            etpRepositoriesOrder.add((ordercount,reponame))

                    for x in repopackages.split():
-                        etpRepositories[reponame]['packages'].append(x+"/"+etpConst['product'])
+                        myRepodata[reponame]['packages'].append(x+"/"+etpConst['product'])

            elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2):
                branch = line.split("|")[1]
@@ -85,10 +85,8 @@ def downloadData(url, pathToSave, bufferSize = 8192, checksum = True, showSpeed

    # start scheduler
    if (showSpeed):
-	etpFileTransfer['datatransfer'] = 0
-	etpFileTransfer['gather'] = 0
-        etpFileTransfer['elapsed'] = 0.0
-	speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransfer['transferpollingtime'])
+        etpFileTransferMetadata[url] = etpFileTransfer.copy()
+	speedUpdater = entropyTools.TimeScheduled(__updateSpeedInfo,etpFileTransferMetadata[url]['transferpollingtime'])
 	speedUpdater.setName("download")
 	speedUpdater.start()
    
@@ -133,6 +131,10 @@ def downloadData(url, pathToSave, bufferSize = 8192, checksum = True, showSpeed
    if (showSpeed):
 	speedUpdater.kill()
        socket.setdefaulttimeout(2)
+        # remove status
+        # XXX: move this action somewhere else
+        del etpFileTransferMetadata[url]
+
    return rc

 # Get the content of an online page
@@ -218,8 +220,8 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = Tru
 	for y in range(diffbarsize):
 	    bartext += " "
 	if (showSpeed):
-	    etpFileTransfer['gather'] = f.tell()
-	    bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransfer['datatransfer']))+"/sec"
+	    etpFileTransferMetadata[url]['gather'] = f.tell()
+	    bartext += "] => "+str(entropyTools.bytesIntoHuman(etpFileTransferMetadata[url]['datatransfer']))+"/sec"
 	else:
 	    bartext += "]"
 	average = str(average)
@@ -231,6 +233,6 @@ def __downloadFileCommitData(f, buf, output = True, maxsize = 0, showSpeed = Tru


 def __updateSpeedInfo():
-    etpFileTransfer['elapsed'] += etpFileTransfer['transferpollingtime']
+    etpFileTransferMetadata[url]['elapsed'] += etpFileTransferMetadata[url]['transferpollingtime']
    # we have the diff size
-    etpFileTransfer['datatransfer'] = etpFileTransfer['gather'] / etpFileTransfer['elapsed']
+    etpFileTransferMetadata[url]['datatransfer'] = etpFileTransferMetadata[url]['gather'] / etpFileTransferMetadata[url]['elapsed']
@@ -19,10 +19,6 @@

 import logging

-#import yum
-#import yum.Errors as Errors
-#from yum.update_md import UpdateMetadata
-
 from misc import const,cleanMarkupSting
 from i18n import _
 from yumgui import format_number
@@ -31,6 +27,10 @@ from callbacks import *
 from dialogs import questionDialog
 from urlgrabber.grabber import URLGrabError

+# Entropy Imports
+from entropyConstants import *
+from clientConstants import *
+import repositoriesTools

 class YumexYumHandler:
    def __init__(self,recent,settings,progress,mainwin,parser):
@@ -487,3 +487,16 @@ class YumexYumHandler:
        
        return out
        
+class repositoryGuiController(repositoriesTools.repositoryController):
+    """ hello world """
+    
+    # reimplementing download function
+    # this function can be reimplemented
+    def downloadDatabase(self, repo, dbfilenameid):
+
+        self.validateRepositoryId(repo)
+
+	rc = self.remoteTools.downloadData(etpRepositories[repo]['database'] +   "/" + etpConst[dbfilenameid], etpRepositories[repo]['dbpath'] + "/" + etpConst[dbfilenameid])
+        if rc in ("-1","-2","-3"):
+            return False
+        return True
@@ -225,7 +225,7 @@ class YumexProgress:
    def setTotal( self, now, total ):
        self.total.setProgress( now, total )
    
-    def set_progress( self, frac, text=None ): 
+    def set_progress( self, frac, text=None ):
        if self.parent.quitNow:
            self.parent.exitNow()
        # Skip if fraction not have changed
@@ -439,7 +439,6 @@ class YumexRepoView:
        self.headers = [_('Repository'),_('Filename')]
        self.store = self.setup_view()
    
-    
    def on_toggled( self, widget, path):
        """ Repo select/unselect handler """
        iter = self.store.get_iter( path )
@@ -448,31 +447,25 @@ class YumexRepoView:
                     
    def setup_view( self ):
        """ Create models and columns for the Repo TextView  """
-        store = gtk.ListStore( 'gboolean', gobject.TYPE_STRING,gobject.TYPE_STRING,'gboolean')
+        store = gtk.ListStore( 'gboolean', gobject.TYPE_STRING, gobject.TYPE_STRING, gobject.TYPE_STRING)
        self.view.set_model( store )
+        
        # Setup Selection Column
        cell1 = gtk.CellRendererToggle()    # Selection
        cell1.set_property( 'activatable', True )
-        column1 = gtk.TreeViewColumn( "    ", cell1 )
+        column1 = gtk.TreeViewColumn( "Active", cell1 )
        column1.add_attribute( cell1, "active", 0 )
        column1.set_resizable( True )
-        column1.set_sort_column_id( -1 )            
+        column1.set_sort_column_id( -1 )
        self.view.append_column( column1 )
-        cell1.connect( "toggled", self.on_toggled )     
-        # Setup resent column
-        cell2 = gtk.CellRendererPixbuf()    # gpgcheck
-        cell2.set_property( 'stock-id', gtk.STOCK_DIALOG_AUTHENTICATION )
-        column2 = gtk.TreeViewColumn( "", cell2 )
-        column2.set_cell_data_func( cell2, self.new_pixbuf )
-        column2.set_sizing( gtk.TREE_VIEW_COLUMN_FIXED )
-        column2.set_fixed_width( 20 )
-        column2.set_sort_column_id( -1 )
-        self.view.append_column( column2 )
-        column2.set_clickable( True )
-               
+        cell1.connect( "toggled", self.on_toggled )
+        
+        # Setup revision column
+        self.create_text_column( _('Revision'),1 )
+        
        # Setup reponame & repofile column's
-        self.create_text_column( _('Repository Identifier'),1 )
-        self.create_text_column( _('Description'),2 )
+        self.create_text_column( _('Repository Identifier'),2 )
+        self.create_text_column( _('Description'),3 )
        self.view.set_search_column( 1 )
        self.view.set_reorderable( False )
        return store
@@ -481,7 +474,7 @@ class YumexRepoView:
        cell = gtk.CellRendererText()    # Size Column
        column = gtk.TreeViewColumn( hdr, cell, text=colno )
        column.set_resizable( True )
-        self.view.append_column( column )        
+        self.view.append_column( column )

    def populate(self):
        """ Populate a repo liststore with data """
@@ -490,7 +483,10 @@ class YumexRepoView:
        repos.sort()
        for repo in etpRepositoriesOrder:
            repodata = etpRepositories[repo[1]]
-            self.store.append([1,repo[1],repodata['description'],"200x"])
+            self.store.append([1,repodata['dbrevision'],repo[1],repodata['description']])
+        # excluded ones
+        for repo in etpRepositoriesExcluded:
+            self.store.append([0,etpRepositoriesExcluded['dbrevision'],repo,etpRepositoriesExcluded['description']])

    def new_pixbuf( self, column, cell, model, iter ):
        gpg = model.get_value( iter, 3 )
@@ -503,7 +499,7 @@ class YumexRepoView:
        selected = []
        for elem in self.store:
            state = elem[0]
-            name = elem[1]
+            name = elem[2]
            if state:
                selected.append( name )
        return selected
@@ -570,7 +566,7 @@ class YumexPluginView:
        column1.add_attribute( cell1, "active", 0 )
        column1.set_resizable( True )
        self.view.append_column( column1 )
-        cell1.connect( "toggled", self.on_toggled, model )             
+        cell1.connect( "toggled", self.on_toggled, model )
        cell2 = gtk.CellRendererText()
        column2= gtk.TreeViewColumn( _( "Plugin" ), cell2, text=1 )
        column2.set_resizable( True )
@@ -323,8 +323,8 @@
 		    <widget class="GtkScrolledWindow" id="scrolledwindow6">
 		      <property name="visible">True</property>
 		      <property name="can_focus">True</property>
-		      <property name="hscrollbar_policy">GTK_POLICY_ALWAYS</property>
-		      <property name="vscrollbar_policy">GTK_POLICY_ALWAYS</property>
+		      <property name="hscrollbar_policy">GTK_POLICY_AUTOMATIC</property>
+		      <property name="vscrollbar_policy">GTK_POLICY_AUTOMATIC</property>
 		      <property name="shadow_type">GTK_SHADOW_IN</property>
 		      <property name="window_placement">GTK_CORNER_TOP_LEFT</property>

@@ -357,16 +357,16 @@
 		      <property name="spacing">6</property>

 		      <child>
-			<widget class="GtkButton" id="repoDeSelect">
+			<widget class="GtkButton" id="repoRefresh">
 			  <property name="visible">True</property>
 			  <property name="can_default">True</property>
 			  <property name="can_focus">True</property>
 			  <property name="relief">GTK_RELIEF_NORMAL</property>
 			  <property name="focus_on_click">True</property>
-			  <signal name="clicked" handler="on_repoDeSelect_clicked"/>
+			  <signal name="clicked" handler="on_repoRefresh_clicked" last_modification_time="Sat, 29 Dec 2007 00:35:25 GMT"/>

 			  <child>
-			    <widget class="GtkAlignment" id="alignment7">
+			    <widget class="GtkAlignment" id="alignment54">
 			      <property name="visible">True</property>
 			      <property name="xalign">0.5</property>
 			      <property name="yalign">0.5</property>
@@ -378,15 +378,15 @@
 			      <property name="right_padding">0</property>

 			      <child>
-				<widget class="GtkHBox" id="hbox8">
+				<widget class="GtkHBox" id="hbox87">
 				  <property name="visible">True</property>
 				  <property name="homogeneous">False</property>
 				  <property name="spacing">2</property>

 				  <child>
-				    <widget class="GtkImage" id="image7">
+				    <widget class="GtkImage" id="image192">
 				      <property name="visible">True</property>
-				      <property name="stock">gtk-cancel</property>
+				      <property name="stock">gtk-refresh</property>
 				      <property name="icon_size">4</property>
 				      <property name="xalign">0.5</property>
 				      <property name="yalign">0.5</property>
@@ -401,9 +401,9 @@
 				  </child>

 				  <child>
-				    <widget class="GtkLabel" id="label13">
+				    <widget class="GtkLabel" id="label132">
 				      <property name="visible">True</property>
-				      <property name="label" translatable="yes">_Deselect All</property>
+				      <property name="label" translatable="yes">Update Repositories</property>
 				      <property name="use_underline">True</property>
 				      <property name="use_markup">False</property>
 				      <property name="justify">GTK_JUSTIFY_LEFT</property>
@@ -430,19 +430,6 @@
 			  </child>
 			</widget>
 		      </child>
-
-		      <child>
-			<widget class="GtkButton" id="repoRefresh">
-			  <property name="visible">True</property>
-			  <property name="can_default">True</property>
-			  <property name="can_focus">True</property>
-			  <property name="label">gtk-refresh</property>
-			  <property name="use_stock">True</property>
-			  <property name="relief">GTK_RELIEF_NORMAL</property>
-			  <property name="focus_on_click">True</property>
-			  <signal name="clicked" handler="on_repoRefresh_clicked"/>
-			</widget>
-		      </child>
 		    </widget>
 		    <packing>
 		      <property name="padding">5</property>
@@ -24,9 +24,13 @@ import logging
 import traceback

 # Entropy Imports
+sys.path.append("../../libraries")
+sys.path.append("../../client")
 sys.path.append("/usr/lib/entropy/libraries")
 sys.path.append("/usr/lib/entropy/client")
 from entropyConstants import *
+from clientConstants import *
+import exceptionTools

 # GTK Imports
 import gtk,gobject
@@ -38,7 +42,7 @@ from yumgui import *

 # yumex imports
 import filters
-from entropyapi import YumexYumHandler
+from entropyapi import YumexYumHandler, repositoryGuiController
 from gui import YumexGUI
 from dialogs import *
 from misc import const,YumexOptions,YumexProfile
@@ -105,7 +109,7 @@ class YumexController(Controller):
        repos = self.repoView.get_selected()
        self.setPage('output')
        self.logger.info( "Enabled repositories : %s" % ",".join(repos))
-        self.setupYum()
+        self.updateRepositories(repos)

    def on_repoDeSelect_clicked(self,widget):
        self.repoView.deselect_all()
@@ -116,13 +120,9 @@ class YumexController(Controller):
        self.queueView.deleteSelected()

    def on_queueQuickAdd_activate(self,widget):
-        if self.initYum: # Yum has to be initialised
-            txt = widget.get_text()
-            arglist = txt.split(' ')
-            self.doQuickAdd(arglist[0],arglist[1:])
-        else:
-            self.setStatus(_('Yum has not been initialized yet'))
-            widget.set_text('')
+        txt = widget.get_text()
+        arglist = txt.split(' ')
+        self.doQuickAdd(arglist[0],arglist[1:])
        
    def on_queueProcess_clicked( self, widget ):
        """ Process Queue Button Handler """
@@ -143,7 +143,7 @@ class YumexController(Controller):
            else:
                self.queue.clear()       # Clear package queue    
                self.queueView.refresh() # Refresh Package Queue 
-                self.setupYum()
+                #self.setupYum()

    def on_queueSave_clicked( self, widget ):
        dialog = gtk.FileChooserDialog(title=None,action=gtk.FILE_CHOOSER_ACTION_SAVE,
@@ -399,11 +399,7 @@ class YumexApplication(YumexController,YumexGUI):
        # setup Repositories
        self.setupRepoView()
        self.firstTime = True
-        self.initYum = False
-        if self.settings.autorefresh:
-            self.setupYum()
-        else:
-            self.setPage('repos')
+        self.setPage('repos')

    def startWorking(self):
        self.isWorking = True
@@ -417,25 +413,65 @@ class YumexApplication(YumexController,YumexGUI):
        normalCursor(self.ui.main)
        gtkEventThread.endProcessing()
            
-    def setupYum(self):
+    def updateRepositories(self, repos):
        self.setPage('output')
        self.startWorking()
-        self.progress.total.setup( const.SETUP_PROGRESS_STEPS )        
-        # Run yum setup in background thread
+        
+        # set steps
+        self.progress.total.setup( range(len(repos)) )
+        self.progressLog(_('Initializing Repository module...'))
+        
+        # 1
+        import repositoriesTools
        try:
-            if self.firstTime:
-                self.yumbase._setupBase()
-                self.firstTime = False
-            else:
-                self.yumbase._setupAgain()
+            repoConn = repositoryGuiController(repos, forceUpdate = True)
+        except exceptionTools.PermissionDenied:
+            self.progressLog(_('You must run this application as root'))
+            return 1
+        except exceptionTools.MissingParameter:
+            self.progressLog(_('No repositories specified in %s') % (etpConst['repositoriesconf'],))
+            return 127
+        except exceptionTools.OnlineMirrorError:
+            self.progressLog(_('You are not connected to the Internet. You should.'))
+            return 126
        except Exception, e:
-            self.endWorking()
-            errorMessage( self.ui.main, _( "Error" ), _( "Error in Yum Setup" ), str(e) )
-            self.logger.error(str(e))
-            return         
-        # prepare package lists
-        self.progress.total.next() # -> List setup        
-        self.progressLog(_('Building Package Lists'))
+            self.progressLog(_('Unhandled exception: %s') % (str(e),))
+            return 2
+        
+        # 2
+        error = False
+        for repo in repoConn.reponames:
+
+            self.progressLog(_('Fetching repository id: %s') % (repo,))
+            update = repoConn.isRepositoryUpdatable(repo)
+            if not update:
+                self.progressLog(_('Attention: database already up to date for: %s') % (repo,))
+                time.sleep(1)
+                error = True
+                continue
+
+            unlocked = repoConn.isRepositoryUnlocked(repo)
+            if not unlocked:
+                self.progressLog(_('Attention: repository %s is being updated. Try again in a few minutes.') % (repo,))
+                time.sleep(1)
+                error = True
+                continue
+
+            # database is going to be updated
+            repoConn.dbupdated = True
+            # clear database interface cache belonging to this repository
+            repoConn.clearRepositoryCache(repo)
+            cmethod = repoConn.validateCompressionMethod(repo)
+            repoConn.ensureRepositoryPath(repo)
+
+            self.progressLog(_('Downloading database for %s') % (repo,))
+            # download
+            down_status = repoConn.downloadDatabase(repo, cmethod[2])
+            if not down_status:
+                self.progressLog(_('Attention: repository %s does not exist online.') % (repo,))
+                error = True
+                continue
+        
        #self.yumbase.populatePackages(['installed','updates','available'])
        self.progressLog(_('Building Package Lists Completed'))
        self.progressLog(_('Building Groups Lists'))
@@ -446,9 +482,10 @@ class YumexApplication(YumexController,YumexGUI):
        self.doProgress = True
        #self.addPackages()
        self.doProgress = False
-        self.setPage('packages')
-        self.initYum = True
+        if not error:
+            self.setPage('repos')
        self.endWorking()
+        initConfig_entropyConstants(etpSys['rootdir'])
        
    
    def setupRepoView(self):
@@ -465,7 +502,7 @@ class YumexApplication(YumexController,YumexGUI):
            masks = [action]
        self.pkgView.store.clear()
        allpkgs = []
-        if self.doProgress: self.progress.total.next() # -> Get lists                
+        if self.doProgress: self.progress.total.next() # -> Get lists
        for flt in masks:
            msg = _('Getting packages : %s' ) % flt
            self.progressLog(msg)
@@ -473,7 +510,7 @@ class YumexApplication(YumexController,YumexGUI):
            pkgs = self.yumbase.getPackages(flt)
            self.progressLog(_('Found %d %s packages') % (len(pkgs),flt))
            allpkgs.extend(pkgs)
-        if self.doProgress: self.progress.total.next() # -> Sort Lists        
+        if self.doProgress: self.progress.total.next() # -> Sort Lists
        self.progressLog(_('Sorting packages'))
        allpkgs.sort()
        self.progressLog(_('Population view with packages'))