entropy/libraries/entropy.py

#!/usr/bin/python
'''
    # DESCRIPTION:
    # Entropy Object Oriented Interface

    Copyright (C) 2007-2008 Fabio Erculiani

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
'''

from __future__ import with_statement
import shutil, commands, urllib2, time, thread, copy, subprocess
from entropyConstants import *
from outputTools import TextInterface, \
    print_info, print_warning, print_error, \
    red, brown, blue, green, purple, darkgreen, \
    darkred, bold, darkblue, readtext, nocolor
import exceptionTools
from entropy_i18n import _

try: # try with sqlite3 from python 2.5 - default one
    from sqlite3 import dbapi2
except ImportError: # fallback to embedded pysqlite
    try:
        from pysqlite2 import dbapi2
    except ImportError, e:
        raise exceptionTools.SystemError(
            "%s. %s: %s" % (
                _("Entropy needs a working sqlite+pysqlite or Python compiled with sqlite support"),
                _("Error"),
                e,
            )
        )

class urlFetcher:

    def __init__(self, url, pathToSave, checksum = True, showSpeed = True, resume = True, abort_check_func = None, disallow_redirect = False):

        import entropyTools, socket
        self.entropyTools, self.socket = entropyTools, socket
        self.url = url
        self.resume = resume
        self.url = self.encodeUrl(self.url)
        self.pathToSave = pathToSave
        self.checksum = checksum
        self.showSpeed = showSpeed
        self.initVars()
        self.progress = None
        self.abort_check_func = abort_check_func
        self.disallow_redirect = disallow_redirect
        self.user_agent = "Entropy/%s (compatible; %s; %s: %s %s %s)" % (
                                        etpConst['entropyversion'],
                                        "Entropy",
                                        os.path.basename(self.url),
                                        os.uname()[0],
                                        os.uname()[4],
                                        os.uname()[2],
        )
        self.extra_header_data = {}
        self.setup_resume_support()
        self.setup_proxy()

    def setup_resume_support(self):
        # resume support
        if os.path.isfile(self.pathToSave) and os.access(self.pathToSave,os.R_OK) and self.resume:
            self.localfile = open(self.pathToSave,"awb")
            self.localfile.seek(0,2)
            self.startingposition = int(self.localfile.tell())
            self.resumed = True
        else:
            if os.path.lexists(self.pathToSave) and not self.entropyTools.is_valid_path(self.pathToSave):
                try:
                    os.remove(self.pathToSave)
                except OSError: # I won't stop you here
                    pass
            self.localfile = open(self.pathToSave,"wb")

    def setup_proxy(self):

        # setup proxy, doing here because config is dynamic
        mydict = {}
        if etpConst['proxy']['ftp']:
            mydict['ftp'] = etpConst['proxy']['ftp']
        if etpConst['proxy']['http']:
            mydict['http'] = etpConst['proxy']['http']
        if mydict:
            mydict['username'] = etpConst['proxy']['username']
            mydict['password'] = etpConst['proxy']['password']
            self.entropyTools.add_proxy_opener(urllib2, mydict)
        else:
            # unset
            urllib2._opener = None

    def encodeUrl(self, url):
        import urllib
        url = os.path.join(os.path.dirname(url),urllib.quote(os.path.basename(url)))
        return url

    def initVars(self):
        self.resumed = False
        self.bufferSize = 8192
        self.status = None
        self.remotefile = None
        self.downloadedsize = 0
        self.average = 0
        self.remotesize = 0
        self.oldaverage = 0.0
        # transfer status data
        self.startingposition = 0
        self.datatransfer = 0
        self.time_remaining = "(infinite)"
        self.elapsed = 0.0
        self.updatestep = 0.2
        self.speedlimit = etpConst['downloadspeedlimit'] # kbytes/sec
        self.transferpollingtime = float(1)/4

    def download(self):
        self.initVars()
        self.speedUpdater = self.entropyTools.TimeScheduled(
                    self.updateSpeedInfo,
                    self.transferpollingtime
        )
        self.speedUpdater.setName("download::"+self.url+str(abs(hash(os.urandom(20))))) # set unique ID to thread, hopefully
        self.speedUpdater.start()

        # set timeout
        self.socket.setdefaulttimeout(20)


        if self.url.startswith("http://"):
            headers = { 'User-Agent' : self.user_agent }
            req = urllib2.Request(self.url, self.extra_header_data, headers)
        else:
            req = self.url

        u_agent_error = False
        while 1:
            # get file size if available
            try:
                self.remotefile = urllib2.urlopen(req)
            except KeyboardInterrupt:
                self.close()
                raise
            except urllib2.HTTPError, e:
                if (e.code == 405) and not u_agent_error:
                    # server doesn't like our user agent
                    req = self.url
                    u_agent_error = True
                    continue
                self.close()
                self.status = "-3"
                return self.status
            except:
                self.close()
                self.status = "-3"
                return self.status
            break


        try:
            self.remotesize = int(self.remotefile.headers.get("content-length"))
            self.remotefile.close()
        except KeyboardInterrupt:
            self.close()
            raise
        except:
            pass

        # handle user stupidity
        try:
            request = self.url
            if ((self.startingposition > 0) and (self.remotesize > 0)) and (self.startingposition < self.remotesize):
                try:
                    request = urllib2.Request(
                        self.url,
                        headers = {
                            "Range" : "bytes=" + str(self.startingposition) + "-" + str(self.remotesize) 
                        }
                    )
                except KeyboardInterrupt:
                    self.close()
                    raise
                except:
                    pass
            elif (self.startingposition == self.remotesize):
                return self.prepare_return()
            else:
                self.localfile = open(self.pathToSave,"wb")
            self.remotefile = urllib2.urlopen(request)
        except KeyboardInterrupt:
            self.close()
            raise
        except:
            self.close()
            self.status = "-3"
            return self.status

        if self.remotesize > 0:
            self.remotesize = float(int(self.remotesize))/1024

        if self.disallow_redirect and (self.url != self.remotefile.geturl()):
            self.close()
            self.status = "-3"
            return self.status

        rsx = "x"
        while rsx != '':
            try:
                rsx = self.remotefile.read(self.bufferSize)
                if self.abort_check_func != None:
                    self.abort_check_func()
            except KeyboardInterrupt:
                self.close()
                raise
            except:
                # python 2.4 timeouts go here
                self.close()
                self.status = "-3"
                return self.status
            self.commitData(rsx)
            if self.showSpeed:
                self.updateProgress()
                self.oldaverage = self.average
            if self.speedlimit:
                while self.datatransfer > self.speedlimit*1024:
                    time.sleep(0.1)
                    if self.showSpeed:
                        self.updateProgress()
                        self.oldaverage = self.average

        # kill thread
        self.close()

        return self.prepare_return()


    def prepare_return(self):
        if self.checksum:
            self.status = self.entropyTools.md5sum(self.pathToSave)
            return self.status
        else:
            self.status = "-2"
            return self.status

    def commitData(self, mybuffer):
        # writing file buffer
        self.localfile.write(mybuffer)
        # update progress info
        self.downloadedsize = self.localfile.tell()
        kbytecount = float(self.downloadedsize)/1024
        self.average = int((kbytecount/self.remotesize)*100)

    def updateProgress(self):

        mytxt = _("[F]")
        eta_txt = _("ETA")
        sec_txt = _("sec") # as in XX kb/sec

        currentText = darkred("    %s: " % (mytxt,)) + \
            darkgreen(str(round(float(self.downloadedsize)/1024,1))) + "/" + \
            red(str(round(self.remotesize,1))) + " kB"
        # create progress bar
        barsize = 10
        bartext = "["
        curbarsize = 1
        if self.average > self.oldaverage+self.updatestep:
            averagesize = (self.average*barsize)/100
            while averagesize > 0:
                curbarsize += 1
                bartext += "="
                averagesize -= 1
            bartext += ">"
            diffbarsize = barsize-curbarsize
            while diffbarsize > 0:
                bartext += " "
                diffbarsize -= 1
            if self.showSpeed:
                bartext += "] => %s" % (self.entropyTools.bytesIntoHuman(self.datatransfer),)
                bartext += "/%s : %s: %s" % (sec_txt,eta_txt,self.time_remaining,)
            else:
                bartext += "]"
            average = str(self.average)
            if len(average) < 2:
                average = " "+average
            currentText += " <->  "+average+"% "+bartext
            print_info(currentText,back = True)


    def close(self):
        try:
            self.localfile.flush()
            self.localfile.close()
        except:
            pass
        try:
            self.remotefile.close()
        except:
            pass
        self.speedUpdater.kill()
        self.socket.setdefaulttimeout(2)

    def updateSpeedInfo(self):
        self.elapsed += self.transferpollingtime
        # we have the diff size
        self.datatransfer = (self.downloadedsize-self.startingposition) / self.elapsed
        try:
            self.time_remaining = int(round((int(round(self.remotesize*1024,0))-int(round(self.downloadedsize,0)))/self.datatransfer,0))
            self.time_remaining = self.entropyTools.convertSecondsToFancyOutput(self.time_remaining)
        except:
            self.time_remaining = "(%s)" % (_("infinite"),)

class EntropyCacher:

    import entropyTools
    import dumpTools
    import threading
    def __init__(self):
        self.alive = False
        self.CacheBuffer = self.entropyTools.lifobuffer()
        self.CacheWriter = self.entropyTools.TimeScheduled(self.Cacher,0.5)
        self.CacheLock = self.threading.Lock()

    def start(self):
        self.alive = True
        self.CacheWriter.start()

    def sync(self, wait = False):
        if not self.alive: return
        wd = 1000
        while self.CacheBuffer.is_filled() and ((wd > 0) or wait):
            if not wait: wd -= 1
            time.sleep(0.01)

    def push(self, key, data, async = True):
        if not self.alive: return
        if async:
            with self.CacheLock:
                self.CacheBuffer.push((key,data,))
        else:
            self.dumpTools.dumpobj(key,data)

    def pop(self, key):
        with self.CacheLock:
            l_o = self.dumpTools.loadobj
            if not l_o: return
            return l_o(key)

    def Cacher(self):
        while 1:
            if not self.alive: break
            with self.CacheLock:
                data = self.CacheBuffer.pop()
            if data == None: break
            key, data = data
            d_o = self.dumpTools.dumpobj
            if not d_o: break
            d_o(key,data)

    def __del__(self):
        self.stop()

    def stop(self):
        if hasattr(self,"CacheBuffer"):
            if self.CacheBuffer and self.alive:
                wd = 500
                while self.CacheBuffer.is_filled() and wd:
                    wd -= 1
                    time.sleep(0.01)
                self.CacheBuffer.clear()
        self.alive = False
        if hasattr(self,"CacheWriter"):
            self.CacheWriter.kill()

'''
    Main Entropy (client side) package management class
'''
class EquoInterface(TextInterface):

    def __init__(self, indexing = True, noclientdb = 0, xcache = True, user_xcache = False, repo_validation = True, load_ugc = True, url_fetcher = urlFetcher):

        # modules import
        import dumpTools, entropyTools
        self.dumpTools = dumpTools
        self.entropyTools = entropyTools

        self.atomMatchCacheKey = etpCache['atomMatch']
        self.dbapi2 = dbapi2 # export for third parties
        self.FileUpdates = None
        self.repoDbCache = {}
        self.securityCache = {}
        self.QACache = {}
        self.spmCache = {}
        self.mirrorDownloadFailures = {}
        self.repo_error_messages_cache = set()
        self.package_match_validator_cache = {}
        self.memoryDbInstances = {}
        self.validRepositories = []

        self.clientLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[client]")

        self.urlFetcher = url_fetcher # in this way, can be reimplemented (so you can override updateProgress)
        self.progress = None # supporting external updateProgress stuff, you can point self.progress to your progress bar
                             # and reimplement updateProgress
        self.clientDbconn = None
        self.safe_mode = 0
        self.FtpInterface = FtpInterface # for convenience
        self.indexing = indexing
        self.repo_validation = repo_validation
        self.noclientdb = False
        self.openclientdb = True
        if noclientdb in (False,0):
            self.noclientdb = False
        elif noclientdb in (True,1):
            self.noclientdb = True
        elif noclientdb == 2:
            self.noclientdb = True
            self.openclientdb = False
        self.xcache = xcache
        shell_xcache = os.getenv("ETP_NOCACHE")
        if shell_xcache:
            self.xcache = False
        self.Cacher = EntropyCacher()
        # needs to be started here otherwise repository cache will be
        # always dropped, trust me, it will be stopped later in the init
        # if xcache is disabled.
        self.Cacher.start()

        # now if we are on live, we should disable it
        # are we running on a livecd? (/proc/cmdline has "cdroot")
        if self.entropyTools.islive():
            self.xcache = False
        elif (not self.entropyTools.is_user_in_entropy_group()) and not user_xcache:
            self.xcache = False
        elif not user_xcache:
            self.validate_repositories_cache()

        if not self.xcache and (self.entropyTools.is_user_in_entropy_group()):
            try: self.purge_cache(False)
            except: pass

        if not self.xcache: self.Cacher.stop()

        if self.openclientdb:
            self.openClientDatabase()
        self.FileUpdates = self.FileUpdatesInterfaceLoader()

        # setup package settings (masking and other stuff)
        self.SystemSettings = SystemSettings(self)
        # settle settings, especially package masking ones
        # useful for third party softwares that use this interface directly
        self.SystemSettings.scan()

        if self.repo_validation:
            self.validate_repositories()

        # load User Generated Content Interface
        self.UGC = None
        if load_ugc:
            self.UGC = UGCClientInterface(self)

    def destroy(self):
        if hasattr(self,'clientDbconn'):
            if self.clientDbconn != None:
                self.clientDbconn.closeDB()
                del self.clientDbconn
        if hasattr(self,'FileUpdates'):
            del self.FileUpdates
        if hasattr(self,'clientLog'):
            self.clientLog.close()
        if hasattr(self,'Cacher'):
            self.Cacher.stop()

        self.closeAllRepositoryDatabases(mask_clear = False)
        self.closeAllSecurity()
        self.closeAllQA()

    def __del__(self):
        self.destroy()

    def reload_constants(self):
        initConfig_entropyConstants(etpSys['rootdir'])
        initConfig_clientConstants()

    def validate_repositories(self):
        self.mirrorDownloadFailures.clear()
        self.repo_error_messages_cache.clear()
        self.package_match_validator_cache.clear()
        # valid repositories
        del self.validRepositories[:]
        for repoid in etpRepositoriesOrder:
            # open database
            try:
                dbc = self.openRepositoryDatabase(repoid)
                dbc.listConfigProtectDirectories()
                dbc.validateDatabase()
                self.validRepositories.append(repoid)
            except exceptionTools.RepositoryError:
                t = _("Repository") + " " + repoid + " " + _("is not available") + ". " + _("Cannot validate")
                t2 = _("Please update your repositories now in order to remove this message!")
                self.updateProgress(
                                    darkred(t),
                                    importance = 1,
                                    type = "warning"
                                   )
                self.updateProgress(
                                    purple(t2),
                                    header = bold("!!! "),
                                    importance = 1,
                                    type = "warning"
                                   )
                continue # repo not available
            except (dbapi2.OperationalError,dbapi2.DatabaseError,exceptionTools.SystemDatabaseError,):
                t = _("Repository") + " " + repoid + " " + _("is corrupted") + ". " + _("Cannot validate")
                self.updateProgress(
                                    darkred(t),
                                    importance = 1,
                                    type = "warning"
                                   )
                continue
        # to avoid having zillions of open files when loading a lot of EquoInterfaces
        self.closeAllRepositoryDatabases(mask_clear = False)

    def init_generic_memory_repository(self, repoid, description, package_mirrors = []):
        dbc = self.openMemoryDatabase(dbname = repoid)
        self.memoryDbInstances[repoid] = dbc

        # add to etpRepositories
        repodata = {
            'repoid': repoid,
            'in_memory': True,
            'description': description,
            'packages': package_mirrors,
            'dbpath': ':memory:',
        }
        self.addRepository(repodata)

        return dbc

    def setup_default_file_perms(self, filepath):
        # setup file permissions
        os.chmod(filepath,0664)
        if etpConst['entropygid'] != None:
            os.chown(filepath,-1,etpConst['entropygid'])

    def _resources_run_create_lock(self):
        self.create_pid_file_lock(etpConst['locks']['using_resources'])

    def _resources_run_remove_lock(self):
        if os.path.isfile(etpConst['locks']['using_resources']):
            os.remove(etpConst['locks']['using_resources'])

    def _resources_run_check_lock(self):
        rc = self.check_pid_file_lock(etpConst['locks']['using_resources'])
        return rc

    def check_pid_file_lock(self, pidfile):
        if not os.path.isfile(pidfile):
            return False # not locked
        f = open(pidfile)
        s_pid = f.readline().strip()
        f.close()
        try:
            s_pid = int(s_pid)
        except ValueError:
            return False # not locked
        # is it our pid?
        mypid = os.getpid()
        if (s_pid != mypid) and os.path.isdir("%s/proc/%s" % (etpConst['systemroot'],s_pid,)):
            # is it running
            return True # locked
        return False

    def create_pid_file_lock(self, pidfile, mypid = None):
        lockdir = os.path.dirname(pidfile)
        if not os.path.isdir(lockdir):
            os.makedirs(lockdir,0775)
        const_setup_perms(lockdir,etpConst['entropygid'])
        if mypid == None:
            mypid = os.getpid()
        f = open(pidfile,"w")
        f.write(str(mypid))
        f.flush()
        f.close()

    def application_lock_check(self, silent = False):
        # check if another instance is running
        etpConst['applicationlock'] = False
        const_setupEntropyPid(just_read = True)
        locked = self.entropyTools.applicationLockCheck(option = None, gentle = True, silent = True)
        if locked:
            if not silent:
                self.updateProgress(
                    red(_("Another Entropy instance is currently active, cannot satisfy your request.")),
                    importance = 1,
                    type = "error",
                    header = darkred(" @@ ")
                )
            return True
        return False

    def lock_check(self, check_function):

        lock_count = 0
        max_lock_count = 600
        sleep_seconds = 0.5

        # check lock file
        while 1:
            locked = check_function()
            if not locked:
                if lock_count > 0:
                    self.updateProgress(
                        blue(_("Resources unlocked, let's go!")),
                        importance = 1,
                        type = "info",
                        header = darkred(" @@ ")
                    )
                break
            if lock_count >= max_lock_count:
                mycalc = max_lock_count*sleep_seconds/60
                self.updateProgress(
                    blue(_("Resources still locked after %s minutes, giving up!")) % (mycalc,),
                    importance = 1,
                    type = "warning",
                    header = darkred(" @@ ")
                )
                return True # gave up
            lock_count += 1
            self.updateProgress(
                blue(_("Resources locked, sleeping %s seconds, check #%s/%s")) % (
                        sleep_seconds,
                        lock_count,
                        max_lock_count,
                ),
                importance = 1,
                type = "warning",
                header = darkred(" @@ "),
                back = True
            )
            time.sleep(sleep_seconds)
        return False # yay!

    def validate_repositories_cache(self):
        # is the list of repos changed?
        cached = self.Cacher.pop(etpCache['repolist'])
        if cached == None:
            # invalidate matching cache
            try: self.repository_move_clear_cache()
            except IOError: pass
        elif isinstance(cached,tuple):
            difflist = [x for x in cached if x not in etpRepositoriesOrder]
            for repoid in difflist:
                try: self.repository_move_clear_cache(repoid)
                except IOError: pass
        self.store_repository_list_cache()

    def store_repository_list_cache(self):
        self.Cacher.push(etpCache['repolist'],tuple(etpRepositoriesOrder)[:], async = False)

    def backup_setting(self, setting_name):
        if etpConst.has_key(setting_name):
            myinst = etpConst[setting_name]
            if type(etpConst[setting_name]) in (list,tuple):
                myinst = etpConst[setting_name][:]
            elif type(etpConst[setting_name]) in (dict,set):
                myinst = etpConst[setting_name].copy()
            else:
                myinst = etpConst[setting_name]
            etpConst['backed_up'].update({setting_name: myinst})
        else:
            t = _("Nothing to backup in etpConst with %s key") % (setting_name,)
            raise exceptionTools.InvalidData("InvalidData: %s" % (t,))

    def set_priority(self, low = 0):
        return const_setNiceLevel(low)

    def reloadRepositoriesConfigProtect(self, repositories = None):
        if repositories == None:
            repositories = self.validRepositories
        for repoid in repositories:
            dbconn = self.openRepositoryDatabase(repoid)
            self.loadRepositoryConfigProtect(repoid, dbconn)

    def switchChroot(self, chroot = ""):
        # clean caches
        self.purge_cache()
        self.closeAllRepositoryDatabases()
        if chroot.endswith("/"):
            chroot = chroot[:-1]
        etpSys['rootdir'] = chroot
        self.reload_constants()
        self.validate_repositories()
        self.reopenClientDbconn()
        if chroot:
            try: self.clientDbconn.resetTreeupdatesDigests()
            except: pass
        # I don't think it's safe to keep them open
        # isn't it?
        self.closeAllSecurity()
        self.closeAllQA()

    def Security(self):
        chroot = etpConst['systemroot']
        cached = self.securityCache.get(chroot)
        if cached != None:
            return cached
        cached = SecurityInterface(self)
        self.securityCache[chroot] = cached
        return cached

    def QA(self):
        chroot = etpConst['systemroot']
        cached = self.QACache.get(chroot)
        if cached != None:
            return cached
        cached = QAInterface(self)
        self.QACache[chroot] = cached
        return cached

    def closeAllQA(self):
        self.QACache.clear()

    def closeAllSecurity(self):
        self.securityCache.clear()

    def reopenClientDbconn(self):
        self.clientDbconn.closeDB()
        self.openClientDatabase()

    def closeAllRepositoryDatabases(self, mask_clear = True):
        for item in self.repoDbCache:
            self.repoDbCache[item].closeDB()
        self.repoDbCache.clear()
        if mask_clear: self.SystemSettings.clear()

    def openClientDatabase(self):

        def load_db_from_ram():
            self.safe_mode = etpConst['safemodeerrors']['clientdb']
            mytxt = "%s, %s" % (_("System database not found or corrupted"),_("running in safe mode using empty database from RAM"),)
            self.updateProgress(
                darkred(mytxt),
                importance = 1,
                type = "warning",
                header = bold("!!!"),
            )
            conn = self.openMemoryDatabase(dbname = etpConst['clientdbid'])
            return conn

        if not os.path.isdir(os.path.dirname(etpConst['etpdatabaseclientfilepath'])):
            os.makedirs(os.path.dirname(etpConst['etpdatabaseclientfilepath']))

        if (not self.noclientdb) and (not os.path.isfile(etpConst['etpdatabaseclientfilepath'])):
            conn = load_db_from_ram()
            self.entropyTools.printTraceback(f = self.clientLog)
        else:
            conn = EntropyDatabaseInterface(
                readOnly = False,
                dbFile = etpConst['etpdatabaseclientfilepath'],
                clientDatabase = True,
                dbname = etpConst['clientdbid'],
                xcache = self.xcache,
                indexing = self.indexing,
                OutputInterface = self,
                ServiceInterface = self
            )
            # validate database
            if not self.noclientdb:
                try:
                    conn.validateDatabase()
                except exceptionTools.SystemDatabaseError:
                    try:
                        conn.closeDB()
                    except:
                        pass
                    self.entropyTools.printTraceback(f = self.clientLog)
                    conn = load_db_from_ram()

        if not etpConst['dbconfigprotect']:

            if conn.doesTableExist('configprotect') and conn.doesTableExist('configprotectreference'):
                etpConst['dbconfigprotect'] = conn.listConfigProtectDirectories()
            if conn.doesTableExist('configprotectmask') and conn.doesTableExist('configprotectreference'):
                etpConst['dbconfigprotectmask'] = conn.listConfigProtectDirectories(mask = True)

            etpConst['dbconfigprotect'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']]
            etpConst['dbconfigprotectmask'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']]

            etpConst['dbconfigprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpConst['dbconfigprotect']]
            etpConst['dbconfigprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpConst['dbconfigprotectmask']]

        self.clientDbconn = conn
        return self.clientDbconn

    def clientDatabaseSanityCheck(self):
        self.updateProgress(
            darkred(_("Sanity Check") + ": " + _("system database")),
            importance = 2,
            type = "warning"
        )
        idpkgs = self.clientDbconn.listAllIdpackages()
        length = len(idpkgs)
        count = 0
        errors = False
        scanning_txt = _("Scanning...")
        for x in idpkgs:
            count += 1
            self.updateProgress(
                                    darkgreen(scanning_txt),
                                    importance = 0,
                                    type = "info",
                                    back = True,
                                    count = (count,length),
                                    percent = True
                                )
            try:
                self.clientDbconn.getPackageData(x)
            except Exception ,e:
                self.entropyTools.printTraceback()
                errors = True
                self.updateProgress(
                    darkred(_("Errors on idpackage %s, error: %s")) % (x,str(e)),
                    importance = 0,
                    type = "warning"
                )

        if not errors:
            t = _("Sanity Check") + ": %s" % (bold(_("PASSED")),)
            self.updateProgress(
                darkred(t),
                importance = 2,
                type = "warning"
            )
            return 0
        else:
            t = _("Sanity Check") + ": %s" % (bold(_("CORRUPTED")),)
            self.updateProgress(
                darkred(t),
                importance = 2,
                type = "warning"
            )
            return -1

    def openRepositoryDatabase(self, repoid):
        t_ident = 1 # thread.get_ident() disabled for now
        if not self.repoDbCache.has_key((repoid,etpConst['systemroot'],t_ident,)):
            dbconn = self.loadRepositoryDatabase(repoid, xcache = self.xcache, indexing = self.indexing)
            try:
                dbconn.checkDatabaseApi()
            except:
                pass
            self.repoDbCache[(repoid,etpConst['systemroot'],t_ident,)] = dbconn
            return dbconn
        else:
            return self.repoDbCache.get((repoid,etpConst['systemroot'],t_ident,))

    def is_installed_idpackage_in_system_mask(self, idpackage):
        if idpackage in self.SystemSettings['repos_system_mask_installed']:
            return True
        return False

    '''
    @description: open the repository database
    @input repositoryName: name of the client database
    @input xcache: loads on-disk cache
    @input indexing: indexes SQL tables
    @output: database class instance
    NOTE: DO NOT USE THIS DIRECTLY, BUT USE EquoInterface.openRepositoryDatabase
    '''
    def loadRepositoryDatabase(self, repoid, xcache = True, indexing = True):

        if isinstance(repoid,basestring):
            if repoid.endswith(etpConst['packagesext']):
                xcache = False

        if repoid not in etpRepositories:
            t = _("bad repository id specified")
            if repoid not in self.repo_error_messages_cache:
                self.updateProgress(
                    darkred(t),
                    importance = 2,
                    type = "warning"
                )
                self.repo_error_messages_cache.add(repoid)
            raise exceptionTools.RepositoryError("RepositoryError: %s" % (t,))

        dbfile = etpRepositories[repoid]['dbpath']+"/"+etpConst['etpdatabasefile']
        if not os.path.isfile(dbfile):
            t = _("Repository %s hasn't been downloaded yet.") % (repoid,)
            if repoid not in self.repo_error_messages_cache:
                self.updateProgress(
                    darkred(t),
                    importance = 2,
                    type = "warning"
                )
                self.repo_error_messages_cache.add(repoid)
            raise exceptionTools.RepositoryError("RepositoryError: %s" % (t,))

        conn = EntropyDatabaseInterface(
            readOnly = True,
            dbFile = dbfile,
            clientDatabase = True,
            dbname = etpConst['dbnamerepoprefix']+repoid,
            xcache = xcache,
            indexing = indexing,
            OutputInterface = self,
            ServiceInterface = self
        )
        # initialize CONFIG_PROTECT
        if (etpRepositories[repoid]['configprotect'] == None) or \
            (etpRepositories[repoid]['configprotectmask'] == None):
                self.loadRepositoryConfigProtect(repoid, conn)

        if (repoid not in etpConst['client_treeupdatescalled']) and (self.entropyTools.is_user_in_entropy_group()) and (not repoid.endswith(etpConst['packagesext'])):
            updated = False
            try:
                updated = conn.clientUpdatePackagesData(self.clientDbconn)
            except (dbapi2.OperationalError, dbapi2.DatabaseError):
                pass
            if updated:
                self.clear_dump_cache(etpCache['world_update'])
                self.clear_dump_cache(etpCache['world'])
                self.clear_dump_cache(etpCache['install'])
                self.clear_dump_cache(etpCache['remove'])
                self.calculate_world_updates(use_cache = False)
        return conn

    def loadRepositoryConfigProtect(self, repoid, dbconn):

        try:
            etpRepositories[repoid]['configprotect'] = dbconn.listConfigProtectDirectories()
        except (dbapi2.OperationalError, dbapi2.DatabaseError):
            etpRepositories[repoid]['configprotect'] = []
        try:
            etpRepositories[repoid]['configprotectmask'] = dbconn.listConfigProtectDirectories(mask = True)
        except (dbapi2.OperationalError, dbapi2.DatabaseError):
            etpRepositories[repoid]['configprotectmask'] = []

        etpRepositories[repoid]['configprotect'] = [etpConst['systemroot']+x for x in etpRepositories[repoid]['configprotect']]
        etpRepositories[repoid]['configprotectmask'] = [etpConst['systemroot']+x for x in etpRepositories[repoid]['configprotectmask']]

        etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotect']]
        etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotectmask']]

    def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False, indexing_override = None, skipChecks = False):
        if xcache == None:
            xcache = self.xcache
        if indexing_override != None:
            indexing = indexing_override
        else:
            indexing = self.indexing
        if dbname == None:
            dbname = etpConst['genericdbid']
        return EntropyDatabaseInterface(
            readOnly = readOnly,
            dbFile = dbfile,
            clientDatabase = True,
            dbname = dbname,
            xcache = xcache,
            indexing = indexing,
            OutputInterface = self,
            skipChecks = skipChecks
        )

    def openMemoryDatabase(self, dbname = None):
        if dbname == None:
            dbname = etpConst['genericdbid']
        dbc = EntropyDatabaseInterface(
            readOnly = False,
            dbFile = ':memory:',
            clientDatabase = True,
            dbname = dbname,
            xcache = False,
            indexing = False,
            OutputInterface = self,
            skipChecks = True
        )
        dbc.initializeDatabase()
        return dbc

    def backupDatabase(self, dbpath, backup_dir = None, silent = False, compress_level = 9):

        if compress_level not in range(1,10):
            compress_level = 9

        backup_dir = os.path.dirname(dbpath)
        if not backup_dir: backup_dir = os.path.dirname(dbpath)
        dbname = os.path.basename(dbpath)
        bytes_required = 1024000*300
        if not (os.access(backup_dir,os.W_OK) and \
                os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \
                os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)):
            if not silent:
                mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "error",
                    header = red(" @@ ")
                )
            return False, mytxt

        def get_ts():
            from datetime import datetime
            ts = datetime.fromtimestamp(time.time())
            return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second)

        comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),)
        comp_dbpath = os.path.join(backup_dir,comp_dbname)
        if not silent:
            mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )
        import bz2
        try:
            self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level)
        except:
            if not silent:
                self.entropyTools.printTraceback()
            return False, _("Unable to compress")

        if not silent:
            mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )
        return True, _("All fine")

    def restoreDatabase(self, backup_path, db_destination, silent = False):

        bytes_required = 1024000*300
        if not (os.access(os.path.dirname(db_destination),os.W_OK) and \
                os.path.isdir(os.path.dirname(db_destination)) and os.path.isfile(backup_path) and \
                os.access(backup_path,os.R_OK) and self.entropyTools.check_required_space(os.path.dirname(db_destination), bytes_required)):
            if not silent:
                mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")),blue(backup_path),darkred(_("permission denied")),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "error",
                    header = red(" @@ ")
                )
            return False, mytxt

        if not silent:
            mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")),blue(os.path.basename(backup_path)),blue(db_destination),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )

        import bz2
        try:
            self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File)
        except:
            if not silent:
                self.entropyTools.printTraceback()
            return False, _("Unable to unpack")

        if not silent:
            mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")),blue(db_destination),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )
        self.purge_cache()
        return True, _("All fine")

    def list_backedup_client_databases(self, client_dbdir = None):
        if not client_dbdir:
            client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath'])
        return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \
                    if x.startswith(etpConst['dbbackupprefix']) and \
                    os.access(os.path.join(client_dbdir,x),os.R_OK)
        ]

    def get_branch_from_download_relative_uri(self, db_download_uri):
        return db_download_uri.split("/")[2]

    '''
       Cache stuff :: begin
    '''
    def purge_cache(self, showProgress = True, client_purge = True):
        if self.entropyTools.is_user_in_entropy_group():
            skip = set()
            if not client_purge:
                skip.add("/"+etpCache['dbMatch']+"/"+etpConst['clientdbid']) # it's ok this way
                skip.add("/"+etpCache['dbSearch']+"/"+etpConst['clientdbid']) # it's ok this way
            for key in etpCache:
                if showProgress:
                    self.updateProgress(
                        darkred(_("Cleaning %s => dumps...")) % (etpCache[key],),
                        importance = 1,
                        type = "warning",
                        back = True
                    )
                self.clear_dump_cache(etpCache[key], skip = skip)

            if showProgress:
                self.updateProgress(
                    darkgreen(_("Cache is now empty.")),
                    importance = 2,
                    type = "info"
                )

    def generate_cache(self, depcache = True, configcache = True, client_purge = True, install_queue = True):
        # clean first of all
        self.purge_cache(client_purge = client_purge)
        if depcache:
            self.do_depcache(do_install_queue = install_queue)
        if configcache:
            self.do_configcache()

    def do_configcache(self):
        self.updateProgress(
            darkred(_("Configuration files")),
            importance = 2,
            type = "warning"
        )
        self.updateProgress(
            red(_("Scanning hard disk")),
            importance = 1,
            type = "warning"
        )
        self.FileUpdates.scanfs(dcache = False, quiet = True)
        self.updateProgress(
            darkred(_("Cache generation complete.")),
            importance = 2,
            type = "info"
        )

    def do_depcache(self, do_install_queue = True):

        self.updateProgress(
            darkgreen(_("Resolving metadata")),
            importance = 1,
            type = "warning"
        )
        # we can barely ignore any exception from here
        # especially cases where client db does not exist
        try:
            update, remove, fine = self.calculate_world_updates()
            del fine, remove
            if do_install_queue:
                self.retrieveInstallQueue(update, False, False)
            self.calculate_available_packages()
        except:
            pass

        self.updateProgress(
            darkred(_("Dependencies cache filled.")),
            importance = 2,
            type = "warning"
        )

    def clear_dump_cache(self, dump_name, skip = []):
        self.Cacher.sync(wait = True)
        dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name)
        dump_dir = os.path.dirname(dump_path)
        #dump_file = os.path.basename(dump_path)
        for currentdir, subdirs, files in os.walk(dump_dir):
            path = os.path.join(dump_dir,currentdir)
            if skip:
                found = False
                for myskip in skip:
                    if path.find(myskip) != -1:
                        found = True
                        break
                if found: continue
            for item in files:
                if item.endswith(etpConst['cachedumpext']):
                    item = os.path.join(path,item)
                    try: os.remove(item)
                    except OSError: pass
            if not os.listdir(path):
                os.rmdir(path)

    '''
       Cache stuff :: end
    '''

    def unused_packages_test(self, dbconn = None):
        if dbconn == None: dbconn = self.clientDbconn
        return [x for x in dbconn.retrieveUnusedIdpackages() if self.validatePackageRemoval(x)]

    def dependencies_test(self, dbconn = None):

        if dbconn == None:
            dbconn = self.clientDbconn
        # get all the installed packages
        installedPackages = dbconn.listAllIdpackages()

        deps_not_matched = set()
        # now look
        length = len(installedPackages)
        count = 0
        for xidpackage in installedPackages:
            count += 1
            atom = dbconn.retrieveAtom(xidpackage)
            self.updateProgress(
                darkgreen(_("Checking %s") % (bold(atom),)),
                importance = 0,
                type = "info",
                back = True,
                count = (count,length),
                header = darkred(" @@ ")
            )

            xdeps = dbconn.retrieveDependencies(xidpackage)
            needed_deps = set()
            for xdep in xdeps:
                xmatch = dbconn.atomMatch(xdep)
                if xmatch[0] == -1:
                    needed_deps.add(xdep)

            deps_not_matched |= needed_deps

        return deps_not_matched

    def find_belonging_dependency(self, matched_atoms):
        crying_atoms = set()
        for atom in matched_atoms:
            for repo in self.validRepositories:
                rdbconn = self.openRepositoryDatabase(repo)
                riddep = rdbconn.searchDependency(atom)
                if riddep != -1:
                    ridpackages = rdbconn.searchIdpackageFromIddependency(riddep)
                    for i in ridpackages:
                        i,r = rdbconn.idpackageValidator(i)
                        if i == -1:
                            continue
                        iatom = rdbconn.retrieveAtom(i)
                        crying_atoms.add((iatom,repo))
        return crying_atoms

    def get_licenses_to_accept(self, install_queue):
        if not install_queue:
            return {}
        licenses = {}
        for match in install_queue:
            repoid = match[1]
            dbconn = self.openRepositoryDatabase(repoid)
            wl = self.SystemSettings['repos_license_whitelist'].get(repoid)
            if not wl:
                continue
            keys = dbconn.retrieveLicensedataKeys(match[0])
            for key in keys:
                if key not in wl:
                    found = self.clientDbconn.isLicenseAccepted(key)
                    if found:
                        continue
                    if not licenses.has_key(key):
                        licenses[key] = set()
                    licenses[key].add(match)
        return licenses

    def get_text_license(self, license_name, repoid):
        dbconn = self.openRepositoryDatabase(repoid)
        text = dbconn.retrieveLicenseText(license_name)
        tempfile = self.entropyTools.getRandomTempFile()
        f = open(tempfile,"w")
        f.write(text)
        f.flush()
        f.close()
        return tempfile

    def get_file_viewer(self):
        viewer = None
        if os.access("/usr/bin/less",os.X_OK):
            viewer = "/usr/bin/less"
        elif os.access("/bin/more",os.X_OK):
            viewer = "/bin/more"
        if not viewer:
            viewer = self.get_file_editor()
        return viewer

    def get_file_editor(self):
        editor = None
        if os.getenv("EDITOR"):
            editor = "$EDITOR"
        elif os.access("/bin/nano",os.X_OK):
            editor = "/bin/nano"
        elif os.access("/bin/vi",os.X_OK):
            editor = "/bin/vi"
        elif os.access("/usr/bin/vi",os.X_OK):
            editor = "/usr/bin/vi"
        elif os.access("/usr/bin/emacs",os.X_OK):
            editor = "/usr/bin/emacs"
        elif os.access("/bin/emacs",os.X_OK):
            editor = "/bin/emacs"
        return editor

    def libraries_test(self, dbconn = None, broken_symbols = False, task_bombing_func = None):

        if dbconn == None:
            dbconn = self.clientDbconn

        self.updateProgress(
            blue(_("Libraries test")),
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )

        if not etpConst['systemroot']:
            myroot = "/"
        else:
            myroot = etpConst['systemroot']+"/"
        # run ldconfig first
        subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True)
        # open /etc/ld.so.conf
        if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"):
            self.updateProgress(
                blue(_("Cannot find "))+red(etpConst['systemroot']+"/etc/ld.so.conf"),
                importance = 1,
                type = "error",
                header = red(" @@ ")
            )
            return {},set(),-1

        ldpaths = set(self.entropyTools.collectLinkerPaths())
        ldpaths |= self.entropyTools.collectPaths()
        # speed up when /usr/lib is a /usr/lib64 symlink
        if "/usr/lib64" in ldpaths and "/usr/lib" in ldpaths:
            if os.path.realpath("/usr/lib64") == "/usr/lib":
                ldpaths.discard("/usr/lib")
        # some crappy packages put shit here too
        ldpaths.add("/usr/share")
        # always force /usr/libexec too
        ldpaths.add("/usr/libexec")

        executables = set()
        total = len(ldpaths)
        count = 0
        sys_root_len = len(etpConst['systemroot'])
        for ldpath in ldpaths:
            if callable(task_bombing_func): task_bombing_func()
            count += 1
            self.updateProgress(
                blue("Tree: ")+red(etpConst['systemroot']+ldpath),
                importance = 0,
                type = "info",
                count = (count,total),
                back = True,
                percent = True,
                header = "  "
            )
            ldpath = ldpath.encode(sys.getfilesystemencoding())
            mywalk_iter = os.walk(etpConst['systemroot']+ldpath)

            def mywimf(dt):

                currentdir, subdirs, files = dt

                def mymf(item):
                    filepath = os.path.join(currentdir,item)
                    if filepath in etpConst['libtest_files_blacklist']:
                        return 0
                    if not os.access(filepath,os.R_OK):
                        return 0
                    if not os.path.isfile(filepath):
                        return 0
                    if not self.entropyTools.is_elf_file(filepath):
                        return 0
                    return filepath[sys_root_len:]

                return set([x for x in map(mymf,files) if type(x) != int])

            for x in map(mywimf,mywalk_iter): executables |= x

        self.updateProgress(
            blue(_("Collecting broken executables")),
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )
        t = red(_("Attention")) + ": " + \
            blue(_("don't worry about libraries that are shown here but not later."))
        self.updateProgress(
            t,
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        myQA = self.QA()

        plain_brokenexecs = set()
        total = len(executables)
        count = 0
        scan_txt = blue("%s ..." % (_("Scanning libraries"),))
        for executable in executables:
            if callable(task_bombing_func): task_bombing_func()
            count += 1
            if (count%10 == 0) or (count == total) or (count == 1):
                self.updateProgress(
                    scan_txt,
                    importance = 0,
                    type = "info",
                    count = (count,total),
                    back = True,
                    percent = True,
                    header = "  "
                )
            myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable)
            def mymf2(mylib):
                return not myQA.resolve_dynamic_library(mylib, executable)
            mylibs = set(filter(mymf2,myelfs))
            broken_sym_found = set()
            if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable)
            if not (mylibs or broken_sym_found):
                continue

            if mylibs:
                alllibs = blue(' :: ').join(list(mylibs))
                self.updateProgress(
                    red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]",
                    importance = 1,
                    type = "info",
                    percent = True,
                    count = (count,total),
                    header = "  "
                )
            elif broken_sym_found:
                allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)])
                if len(allsyms) > 50: allsyms = brown(_('various broken symbols'))
                self.updateProgress(
                    red(etpConst['systemroot']+executable)+" { "+allsyms+" }",
                    importance = 1,
                    type = "info",
                    percent = True,
                    count = (count,total),
                    header = "  "
                )
            plain_brokenexecs.add(executable)

        del executables
        packagesMatched = {}

        if not etpSys['serverside']:

            self.updateProgress(
                blue(_("Matching broken libraries/executables")),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
            matched = set()
            for brokenlib in plain_brokenexecs:
                idpackages = self.clientDbconn.searchBelongs(brokenlib)
                for idpackage in idpackages:
                    key, slot = self.clientDbconn.retrieveKeySlot(idpackage)
                    mymatch = self.atomMatch(key, matchSlot = slot)
                    if mymatch[0] == -1:
                        matched.add(brokenlib)
                        continue
                    cmpstat = self.get_package_action(mymatch)
                    if cmpstat == 0:
                        continue
                    if not packagesMatched.has_key(brokenlib):
                        packagesMatched[brokenlib] = set()
                    packagesMatched[brokenlib].add(mymatch)
                    matched.add(brokenlib)
            plain_brokenexecs -= matched

        return packagesMatched,plain_brokenexecs,0

    def move_to_branch(self, branch, pretend = False):
        if pretend: return 0
        if branch != etpConst['branch']:
            # update configuration
            self.entropyTools.writeNewBranch(branch)
            # reset treeupdatesactions
            self.clientDbconn.resetTreeupdatesDigests()
            # clean cache
            self.purge_cache(showProgress = False)
            # reopen Client Database, this will make treeupdates to be re-read
            self.closeAllRepositoryDatabases()
            etpConst['branch'] = branch
            self.reload_constants()
            etpConst['branch'] = branch
            self.validate_repositories()
            self.reopenClientDbconn()
        return 0

    # tell if a new equo release is available, returns True or False
    def check_equo_updates(self):
        found, match = self.check_package_update("app-admin/equo", deep = True)
        return found

    '''
        @input: matched atom (idpackage,repoid)
        @output:
                upgrade: int(2)
                install: int(1)
                reinstall: int(0)
                downgrade: int(-1)
    '''
    def get_package_action(self, match):
        dbconn = self.openRepositoryDatabase(match[1])
        pkgkey, pkgslot = dbconn.retrieveKeySlot(match[0])
        results = self.clientDbconn.searchKeySlot(pkgkey, pkgslot)
        if not results: return 1

        installed_idpackage = results[0][0]
        pkgver, pkgtag, pkgrev = dbconn.getVersioningData(match[0])
        installedVer, installedTag, installedRev = self.clientDbconn.getVersioningData(installed_idpackage)
        pkgcmp = self.entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev))
        if pkgcmp == 0:
            return 0
        elif pkgcmp > 0:
            return 2
        return -1

    def get_meant_packages(self, search_term, from_installed = False, valid_repos = []):

        pkg_data = []
        atom_srch = False
        if "/" in search_term: atom_srch = True

        if not valid_repos: valid_repos = self.validRepositories
        if from_installed: valid_repos = [1]
        for repo in valid_repos:
            if isinstance(repo,basestring):
                dbconn = self.openRepositoryDatabase(repo)
            elif isinstance(repo,EntropyDatabaseInterface):
                dbconn = repo
            elif hasattr(self,'clientDbconn'):
                dbconn = self.clientDbconn
            else:
                continue
            pkg_data.extend([(x,repo,) for x in dbconn.searchSimilarPackages(search_term, atom = atom_srch)])

        return pkg_data

    # better to use key:slot
    def check_package_update(self, atom, deep = False):

        c_hash = "%s%s" % (etpCache['check_package_update'],hash("%s%s" % (atom,hash(deep),)),)
        if self.xcache:
            cached = self.Cacher.pop(c_hash)
            if cached != None:
                return cached

        found = False
        match = self.clientDbconn.atomMatch(atom)
        matched = None
        if match[0] != -1:
            myatom = self.clientDbconn.retrieveAtom(match[0])
            mytag = self.entropyTools.dep_gettag(myatom)
            myatom = self.entropyTools.remove_tag(myatom)
            myrev = self.clientDbconn.retrieveRevision(match[0])
            pkg_match = "="+myatom+"~"+str(myrev)
            if mytag != None:
                pkg_match += "#%s" % (mytag,)
            pkg_unsatisfied = self.get_unsatisfied_dependencies([pkg_match], deep_deps = deep)
            if pkg_unsatisfied:
                found = True
            del pkg_unsatisfied
            matched = self.atomMatch(pkg_match)
        del match

        if self.xcache:
            self.Cacher.push(c_hash,(found,matched)[:])

        return found, matched


    # @returns -1 if the file does not exist or contains bad data
    # @returns int>0 if the file exists
    def get_repository_revision(self, reponame):
        if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
            f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'],"r")
            try:
                revision = int(f.readline().strip())
            except:
                revision = -1
            f.close()
        else:
            revision = -1
        return revision

    def update_repository_revision(self, reponame):
        r = self.get_repository_revision(reponame)
        etpRepositories[reponame]['dbrevision'] = "0"
        if r != -1:
            etpRepositories[reponame]['dbrevision'] = str(r)

    # @returns -1 if the file does not exist
    # @returns int>0 if the file exists
    def get_repository_db_file_checksum(self, reponame):
        if os.path.isfile(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
            f = open(etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
            try:
                mhash = f.readline().strip().split()[0]
            except:
                mhash = "-1"
            f.close()
        else:
            mhash = "-1"
        return mhash

    def update_ugc_cache(self, repository):
        if not self.UGC.is_repository_eapi3_aware(repository):
            return None
        status = True

        votes_dict, err_msg = self.UGC.get_all_votes(repository)
        if isinstance(votes_dict,dict):
            self.UGC.UGCCache.save_vote_cache(repository, votes_dict)
        else:
            status = False

        downloads_dict, err_msg = self.UGC.get_all_downloads(repository)
        if isinstance(downloads_dict,dict):
            self.UGC.UGCCache.save_downloads_cache(repository, downloads_dict)
        else:
            status = False
        return status

    def __handle_multi_repo_matches(self, results, extended_results, valid_repos, open_db):

        packageInformation = {}
        versionInformation = {}
        # .tbz2 repos have always the precedence, so if we find them,
        # we should second what user wants, installing his tbz2
        tbz2repos = [x for x in results if x.endswith(etpConst['packagesext'])]
        if tbz2repos:
            del tbz2repos
            newrepos = results.copy()
            for x in newrepos:
                if x.endswith(etpConst['packagesext']):
                    continue
                del results[x]

        version_duplicates = set()
        versions = set()
        for repo in results:
            packageInformation[repo] = {}
            if extended_results:
                version = results[repo][1]
                packageInformation[repo]['versiontag'] = results[repo][2]
                packageInformation[repo]['revision'] = results[repo][3]
            else:
                dbconn = open_db(repo)
                packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(results[repo])
                packageInformation[repo]['revision'] = dbconn.retrieveRevision(results[repo])
                version = dbconn.retrieveVersion(results[repo])
            packageInformation[repo]['version'] = version
            versionInformation[version] = repo
            if version in versions:
                version_duplicates.add(version)
            versions.add(version)

        newerVersion = self.entropyTools.getNewerVersion(list(versions))[0]
        # if no duplicates are found or newer version is not in duplicates we're done
        if (not version_duplicates) or (newerVersion not in version_duplicates):
            reponame = versionInformation.get(newerVersion)
            return (results[reponame],reponame)

        # we have two repositories with >two packages with the same version
        # check package tag

        conflictingEntries = {}
        tags_duplicates = set()
        tags = set()
        tagsInfo = {}
        for repo in packageInformation:
            if packageInformation[repo]['version'] != newerVersion:
                continue
            conflictingEntries[repo] = {}
            versiontag = packageInformation[repo]['versiontag']
            if versiontag in tags:
                tags_duplicates.add(versiontag)
            tags.add(versiontag)
            tagsInfo[versiontag] = repo
            conflictingEntries[repo]['versiontag'] = versiontag
            conflictingEntries[repo]['revision'] = packageInformation[repo]['revision']

        # tags will always be != []
        newerTag = sorted(list(tags), reverse = True)[0]
        if newerTag not in tags_duplicates:
            reponame = tagsInfo.get(newerTag)
            return (results[reponame],reponame)

        # in this case, we have >two packages with the same version and tag
        # check package revision

        conflictingRevisions = {}
        revisions = set()
        revisions_duplicates = set()
        revisionInfo = {}
        for repo in conflictingEntries:
            if conflictingEntries[repo]['versiontag'] == newerTag:
                conflictingRevisions[repo] = {}
                versionrev = conflictingEntries[repo]['revision']
                if versionrev in revisions:
                    revisions_duplicates.add(versionrev)
                revisions.add(versionrev)
                revisionInfo[versionrev] = repo
                conflictingRevisions[repo]['revision'] = versionrev

        newerRevision = max(revisions)
        if newerRevision not in revisions_duplicates:
            reponame = revisionInfo.get(newerRevision)
            return (results[reponame],reponame)

        # final step, in this case we have >two packages with the same version, tag and revision
        # get the repository with the biggest priority

        for reponame in valid_repos:
            if reponame in conflictingRevisions:
                return (results[reponame],reponame)

    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, matchBranches = (), matchTag = None, packagesFilter = True,
            multiMatch = False, multiRepo = False, matchRevision = None, matchRepo = None,
            server_repos = [], serverInstance = None, extendedResults = False, useCache = True):

        # support match in repository from shell
        # atom@repo1,repo2,repo3
        atom, repos = self.entropyTools.dep_get_match_in_repos(atom)
        if (matchRepo == None) and (repos != None):
            matchRepo = repos

        u_hash = ""
        m_hash = ""
        k_ms = "//"
        k_mt = "@#@"
        k_mr = "-1"
        if isinstance(matchRepo,(list,tuple,set,)): u_hash = hash(frozenset(matchRepo))
        if isinstance(matchBranches,(list,tuple,set,)): m_hash = hash(frozenset(matchBranches))
        if isinstance(matchSlot,basestring): k_ms = matchSlot
        if isinstance(matchTag,basestring): k_mt = matchTag
        if isinstance(matchRevision,basestring): k_mr = matchRevision

        c_hash = "|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" % (
            atom,k_ms,k_mt,hash(packagesFilter),
            hash(frozenset(self.validRepositories)),
            hash(frozenset(etpRepositories)),
            hash(multiMatch),hash(multiRepo),hash(caseSensitive),
            k_mr,hash(extendedResults),
            u_hash, m_hash
        )
        c_hash = "%s%s" % (self.atomMatchCacheKey,hash(c_hash),)

        if self.xcache and useCache:
            cached = self.Cacher.pop(c_hash)
            if cached != None:
                return cached

        if server_repos:
            if not serverInstance:
                t = _("server_repos needs serverInstance")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (t,))
            valid_repos = server_repos[:]
        else:
            valid_repos = self.validRepositories
        if matchRepo and (type(matchRepo) in (list,tuple,set)):
            valid_repos = list(matchRepo)

        def open_db(repoid):
            if server_repos:
                dbconn = serverInstance.openServerDatabase(just_reading = True, repo = repoid)
            else:
                dbconn = self.openRepositoryDatabase(repoid)
            return dbconn

        repoResults = {}
        for repo in valid_repos:

            # search
            dbconn = open_db(repo)
            use_cache = useCache
            while 1:
                try:
                    query_data,query_rc = dbconn.atomMatch(
                        atom,
                        caseSensitive = caseSensitive,
                        matchSlot = matchSlot,
                        matchBranches = matchBranches,
                        matchTag = matchTag,
                        packagesFilter = packagesFilter,
                        matchRevision = matchRevision,
                        extendedResults = extendedResults,
                        useCache = use_cache
                    )
                    if query_rc == 0:
                        # package found, add to our dictionary
                        if extendedResults:
                            repoResults[repo] = (query_data[0],query_data[2],query_data[3],query_data[4])
                        else:
                            repoResults[repo] = query_data
                except TypeError:
                    if not use_cache:
                        raise
                    use_cache = False
                    continue
                break

        if extendedResults:
            dbpkginfo = ((-1,None,None,None),1)
        else:
            dbpkginfo = (-1,1)

        if multiRepo and repoResults:

            data = set()
            for repoid in repoResults:
                data.add((repoResults[repoid],repoid))
            dbpkginfo = (data,0)

        elif len(repoResults) == 1:
            # one result found
            repo = repoResults.keys()[0]
            dbpkginfo = (repoResults[repo],repo)

        elif len(repoResults) > 1:

            # we have to decide which version should be taken
            mypkginfo = self.__handle_multi_repo_matches(repoResults, extendedResults, valid_repos, open_db)
            if mypkginfo != None: dbpkginfo = mypkginfo

        # multimatch support
        if multiMatch:

            if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure
                if multiRepo:
                    data = set()
                    for q_id,q_repo in dbpkginfo[0]:
                        dbconn = open_db(q_repo)
                        query_data, query_rc = dbconn.atomMatch(
                            atom,
                            caseSensitive = caseSensitive,
                            matchSlot = matchSlot,
                            matchBranches = matchBranches,
                            matchTag = matchTag,
                            packagesFilter = packagesFilter,
                            multiMatch = True,
                            extendedResults = extendedResults
                        )
                        if extendedResults:
                            for item in query_data:
                                data.add(((item[0],item[2],item[3],item[4]),q_repo))
                        else:
                            for x in query_data: data.add((x,q_repo))
                    dbpkginfo = (data,0)
                else:
                    dbconn = open_db(dbpkginfo[1])
                    query_data, query_rc = dbconn.atomMatch(
                                                atom,
                                                caseSensitive = caseSensitive,
                                                matchSlot = matchSlot,
                                                matchBranches = matchBranches,
                                                matchTag = matchTag,
                                                packagesFilter = packagesFilter,
                                                multiMatch = True,
                                                extendedResults = extendedResults
                                               )
                    if extendedResults:
                        dbpkginfo = (set([((x[0],x[2],x[3],x[4]),dbpkginfo[1]) for x in query_data]),0)
                    else:
                        dbpkginfo = (set([(x,dbpkginfo[1]) for x in query_data]),0)

        if self.xcache and useCache:
            self.Cacher.push(c_hash,dbpkginfo[:])

        return dbpkginfo

    # expands package sets, and in future something more perhaps
    def packagesExpand(self, packages):
        new_packages = []

        for pkg_id in range(len(packages)):
            package = packages[pkg_id]

            # expand package sets
            if package.startswith(etpConst['packagesetprefix']):
                set_pkgs = sorted(list(self.packageSetExpand(package, raise_exceptions = False)))
                new_packages.extend([x for x in set_pkgs if x not in packages]) # atomMatch below will filter dupies
            else:
                new_packages.append(package)

        return new_packages

    def packageSetExpand(self, package_set, raise_exceptions = True):

        max_recursion_level = 50
        recursion_level = 0

        def do_expand(myset, recursion_level, max_recursion_level):
            recursion_level += 1
            if recursion_level > max_recursion_level:
                raise exceptionTools.InvalidPackageSet('InvalidPackageSet: corrupted, too many recursions: %s' % (myset,))
            set_data, set_rc = self.packageSetMatch(myset[len(etpConst['packagesetprefix']):])
            if not set_rc:
                raise exceptionTools.InvalidPackageSet('InvalidPackageSet: not found: %s' % (myset,))
            (set_from, package_set, mydata,) = set_data

            mypkgs = set()
            for fset in mydata: # recursively
                if fset.startswith(etpConst['packagesetprefix']):
                    mypkgs |= do_expand(fset, recursion_level, max_recursion_level)
                else:
                    mypkgs.add(fset)

            return mypkgs

        if not package_set.startswith(etpConst['packagesetprefix']):
            package_set = "%s%s" % (etpConst['packagesetprefix'],package_set,)

        try:
            mylist = do_expand(package_set, recursion_level, max_recursion_level)
        except exceptionTools.InvalidPackageSet:
            if raise_exceptions: raise
            mylist = set()

        return mylist

    def packageSetList(self, server_repos = [], serverInstance = None):
        return self.packageSetMatch('', server_repos = server_repos, serverInstance = serverInstance, search = True)[0]

    def packageSetSearch(self, package_set, server_repos = [], serverInstance = None):
        # search support
        if package_set == '*': package_set = ''
        return self.packageSetMatch(package_set, server_repos = server_repos, serverInstance = serverInstance, search = True)[0]

    def packageSetMatch(self, package_set, multiMatch = False, matchRepo = None, server_repos = [], serverInstance = None, search = False):

        # support match in repository from shell
        # set@repo1,repo2,repo3
        package_set, repos = self.entropyTools.dep_get_match_in_repos(package_set)
        if (matchRepo == None) and (repos != None):
            matchRepo = repos

        if server_repos:
            if not serverInstance:
                t = _("server_repos needs serverInstance")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (t,))
            valid_repos = server_repos[:]
        else:
            valid_repos = self.validRepositories

        if matchRepo and (type(matchRepo) in (list,tuple,set)):
            valid_repos = list(matchRepo)

        def open_db(repoid):
            if server_repos:
                dbconn = serverInstance.openServerDatabase(just_reading = True, repo = repoid)
            else:
                dbconn = self.openRepositoryDatabase(repoid)
            return dbconn

        # if we search, we return all the matches available
        if search: multiMatch = True

        set_data = []

        while 1:

            # check inside SystemSettings
            if not server_repos:
                if search:
                    mysets = [x for x in self.SystemSettings['system_package_sets'].keys() if (x.find(package_set) != -1)]
                    for myset in mysets:
                        mydata = self.SystemSettings['system_package_sets'].get(myset)
                        set_data.append((etpConst['userpackagesetsid'], unicode(myset), mydata.copy(),))
                else:
                    mydata = self.SystemSettings['system_package_sets'].get(package_set)
                    if mydata != None:
                        set_data.append((etpConst['userpackagesetsid'], unicode(package_set), mydata,))
                        if not multiMatch: break

            for repoid in valid_repos:
                dbconn = open_db(repoid)
                if search:
                    mysets = dbconn.searchSets(package_set)
                    for myset in mysets:
                        mydata = dbconn.retrievePackageSet(myset)
                        set_data.append((repoid, myset, mydata.copy(),))
                else:
                    mydata = dbconn.retrievePackageSet(package_set)
                    if mydata: set_data.append((repoid, package_set, mydata,))
                    if not multiMatch: break

            break

        if not set_data: return (),False
        if multiMatch: return set_data,True
        return set_data.pop(0),True

    def repository_move_clear_cache(self, repoid = None):
        self.clear_dump_cache(etpCache['world_available'])
        self.clear_dump_cache(etpCache['world_update'])
        self.clear_dump_cache(etpCache['check_package_update'])
        self.clear_dump_cache(etpCache['filter_satisfied_deps'])
        self.clear_dump_cache(self.atomMatchCacheKey)
        self.clear_dump_cache(etpCache['dep_tree'])
        if repoid != None:
            self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,))
            self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,))


    def addRepository(self, repodata):
        # update etpRepositories
        try:
            etpRepositories[repodata['repoid']] = {}
            etpRepositories[repodata['repoid']]['description'] = repodata['description']
            etpRepositories[repodata['repoid']]['configprotect'] = None
            etpRepositories[repodata['repoid']]['configprotectmask'] = None
        except KeyError:
            t = _("repodata dictionary is corrupted")
            raise exceptionTools.InvalidData("InvalidData: %s" % (t,))

        if repodata['repoid'].endswith(etpConst['packagesext']) or repodata.get('in_memory'): # dynamic repository
            try:
                # no need # etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:]
                etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:]
                smart_package = repodata.get('smartpackage')
                if smart_package != None: etpRepositories[repodata['repoid']]['smartpackage'] = smart_package
                etpRepositories[repodata['repoid']]['dbpath'] = repodata.get('dbpath')
                etpRepositories[repodata['repoid']]['pkgpath'] = repodata.get('pkgpath')
            except KeyError:
                raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted")
            # put at top priority, shift others
            etpRepositoriesOrder.insert(0,repodata['repoid'])
        else:
            # XXX it's boring to keep this in sync with entropyConstants stuff, solutions?
            etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:]
            etpRepositories[repodata['repoid']]['packages'] = [x+"/"+etpConst['product'] for x in repodata['plain_packages']]
            etpRepositories[repodata['repoid']]['plain_database'] = repodata['plain_database']
            etpRepositories[repodata['repoid']]['database'] = repodata['plain_database'] + \
                "/" + etpConst['product'] + "/database/" + etpConst['currentarch'] + "/" + etpConst['branch']
            if not repodata['dbcformat'] in etpConst['etpdatabasesupportedcformats']:
                repodata['dbcformat'] = etpConst['etpdatabasesupportedcformats'][0]
            etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat']
            etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + \
                "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch']  + "/" + etpConst['branch']
            # set dbrevision
            myrev = self.get_repository_revision(repodata['repoid'])
            if myrev == -1:
                myrev = 0
            etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev)
            if repodata.has_key("position"):
                etpRepositoriesOrder.insert(repodata['position'],repodata['repoid'])
            else:
                etpRepositoriesOrder.append(repodata['repoid'])
            if not repodata.has_key("service_port"):
                repodata['service_port'] = int(etpConst['socket_service']['port'])
            if not repodata.has_key("ssl_service_port"):
                repodata['ssl_service_port'] = int(etpConst['socket_service']['ssl_port'])
            etpRepositories[repodata['repoid']]['service_port'] = repodata['service_port']
            etpRepositories[repodata['repoid']]['ssl_service_port'] = repodata['ssl_service_port']
            self.repository_move_clear_cache(repodata['repoid'])
            # save new etpRepositories to file
            self.entropyTools.saveRepositorySettings(repodata)
            self.reload_constants()
        self.validate_repositories()

    def removeRepository(self, repoid, disable = False):

        # ensure that all dbs are closed
        self.closeAllRepositoryDatabases()

        done = False
        if etpRepositories.has_key(repoid):
            del etpRepositories[repoid]
            done = True

        if etpRepositoriesExcluded.has_key(repoid):
            del etpRepositoriesExcluded[repoid]
            done = True

        if done:

            if repoid in etpRepositoriesOrder:
                etpRepositoriesOrder.remove(repoid)

            self.repository_move_clear_cache(repoid)
            # save new etpRepositories to file
            repodata = {}
            repodata['repoid'] = repoid
            if disable:
                self.entropyTools.saveRepositorySettings(repodata, disable = True)
            else:
                self.entropyTools.saveRepositorySettings(repodata, remove = True)
            self.reload_constants()

        self.validate_repositories()

    def shiftRepository(self, repoid, toidx):
        # update etpRepositoriesOrder
        etpRepositoriesOrder.remove(repoid)
        etpRepositoriesOrder.insert(toidx,repoid)
        self.entropyTools.writeOrderedRepositoriesEntries()
        self.reload_constants()
        self.repository_move_clear_cache(repoid)
        self.validate_repositories()

    def enableRepository(self, repoid):
        self.repository_move_clear_cache(repoid)
        # save new etpRepositories to file
        repodata = {}
        repodata['repoid'] = repoid
        self.entropyTools.saveRepositorySettings(repodata, enable = True)
        self.reload_constants()
        self.validate_repositories()

    def disableRepository(self, repoid):
        # update etpRepositories
        done = False
        try:
            del etpRepositories[repoid]
            done = True
        except:
            pass

        if done:
            try:
                etpRepositoriesOrder.remove(repoid)
            except:
                pass
            # it's not vital to reset etpRepositoriesOrder counters

            self.repository_move_clear_cache(repoid)
            # save new etpRepositories to file
            repodata = {}
            repodata['repoid'] = repoid
            self.entropyTools.saveRepositorySettings(repodata, disable = True)
            self.reload_constants()
        self.validate_repositories()

    def get_unsatisfied_dependencies(self, dependencies, deep_deps = False, depcache = {}):

        if self.xcache:
            c_data = sorted(list(dependencies))
            client_checksum = self.clientDbconn.database_checksum()
            c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps))+client_checksum
            c_hash = "unsat_%s" % (hash(c_hash),)
            cached = self.dumpTools.loadobj(etpCache['filter_satisfied_deps']+c_hash)
            if cached != None: return cached

        cdb_am = self.clientDbconn.atomMatch
        am = self.atomMatch
        open_repo = self.openRepositoryDatabase
        intf_error = self.dbapi2.InterfaceError
        cdb_getversioning = self.clientDbconn.getVersioningData
        cdb_retrieveneededraw = self.clientDbconn.retrieveNeededRaw
        etp_cmp = self.entropyTools.entropyCompareVersions
        do_needed_check = False

        def fm_dep(dependency):

            cached = depcache.get(dependency)
            if cached != None: return cached

            ### conflict
            if dependency.startswith("!"):
                idpackage,rc = cdb_am(dependency[1:])
                if idpackage != -1:
                    depcache[dependency] = dependency
                    return dependency
                depcache[dependency] = 0
                return 0

            c_id,c_rc = cdb_am(dependency)
            if c_id == -1:
                depcache[dependency] = dependency
                return dependency

            #if not deep_deps and not do_needed_check:
            #    depcache[dependency] = 0
            #    return 0

            r_id,r_repo = am(dependency)
            if r_id == -1:
                depcache[dependency] = dependency
                return dependency

            if do_needed_check:
                dbconn = open_repo(r_repo)
                installed_needed = cdb_retrieveneededraw(c_id)
                repo_needed = dbconn.retrieveNeededRaw(r_id)
                if installed_needed != repo_needed:
                    return dependency
                #elif not deep_deps:
                #    return 0

            dbconn = open_repo(r_repo)
            try:
                repo_pkgver, repo_pkgtag, repo_pkgrev = dbconn.getVersioningData(r_id)
            except (intf_error,TypeError,):
                # package entry is broken
                return dependency

            try:
                installedVer, installedTag, installedRev = cdb_getversioning(c_id)
            except TypeError: # corrupted entry?
                installedVer = "0"
                installedTag = ''
                installedRev = 0

            vcmp = etp_cmp((repo_pkgver,repo_pkgtag,repo_pkgrev,), (installedVer,installedTag,installedRev,))
            if vcmp != 0:
                if not deep_deps and ((repo_pkgver,repo_pkgtag,) == (installedVer,installedTag,)) and (repo_pkgrev != installedRev):
                    depcache[dependency] = 0
                    return 0
                depcache[dependency] = dependency
                return dependency
            depcache[dependency] = 0
            return 0

        unsatisfied = map(fm_dep,dependencies)
        unsatisfied = set([x for x in unsatisfied if x != 0])

        if self.xcache:
            try:
                self.dumpTools.dumpobj(etpCache['filter_satisfied_deps']+c_hash,unsatisfied)
            except IOError:
                pass

        return unsatisfied

    def get_masked_package_reason(self, match):
        idpackage, repoid = match
        dbconn = self.openRepositoryDatabase(repoid)
        idpackage, idreason = dbconn.idpackageValidator(idpackage)
        masked = False
        if idpackage == -1: masked = True
        return masked, idreason, self.SystemSettings['pkg_masking_reasons'].get(idreason)

    def get_masked_packages_tree(self, match, atoms = False, flat = False, matchfilter = set()):

        if not isinstance(matchfilter,set):
            matchfilter = set()

        maskedtree = {}
        mybuffer = self.entropyTools.lifobuffer()
        depcache = set()
        treelevel = -1

        match_id, match_repo = match

        mydbconn = self.openRepositoryDatabase(match_repo)
        myatom = mydbconn.retrieveAtom(match_id)
        idpackage, idreason = mydbconn.idpackageValidator(match_id)
        if idpackage == -1:
            treelevel += 1
            if atoms:
                mydict = {myatom: idreason,}
            else:
                mydict = {match: idreason,}
            if flat:
                maskedtree.update(mydict)
            else:
                maskedtree[treelevel] = {}
                maskedtree[treelevel].update(mydict)

        mydeps = mydbconn.retrieveDependencies(match_id)
        for mydep in mydeps: mybuffer.push(mydep)
        mydep = mybuffer.pop()

        open_db = self.openRepositoryDatabase
        am = self.atomMatch
        while mydep:

            if mydep in depcache:
                mydep = mybuffer.pop()
                continue
            depcache.add(mydep)

            idpackage, repoid = am(mydep)
            if (idpackage, repoid) in matchfilter:
                mydep = mybuffer.pop()
                continue

            if idpackage != -1:
                # doing even here because atomMatch with packagesFilter = False can pull
                # something different
                matchfilter.add((idpackage, repoid))

            # collect masked
            if idpackage == -1:
                idpackage, repoid = am(mydep, packagesFilter = False)
                if idpackage != -1:
                    treelevel += 1
                    if not maskedtree.has_key(treelevel) and not flat:
                        maskedtree[treelevel] = {}
                    dbconn = open_db(repoid)
                    vidpackage, idreason = dbconn.idpackageValidator(idpackage)
                    if atoms:
                        mydict = {dbconn.retrieveAtom(idpackage): idreason}
                    else:
                        mydict = {(idpackage,repoid): idreason}
                    if flat: maskedtree.update(mydict)
                    else: maskedtree[treelevel].update(mydict)

            # push its dep into the buffer
            if idpackage != -1:
                matchfilter.add((idpackage, repoid))
                dbconn = open_db(repoid)
                owndeps = dbconn.retrieveDependencies(idpackage)
                for owndep in owndeps:
                    mybuffer.push(owndep)

            mydep = mybuffer.pop()

        return maskedtree


    def generate_dependency_tree(self, atomInfo, empty_deps = False, deep_deps = False, matchfilter = set(), flat = False, filter_unsat_cache = {}, treecache = set(), keyslotcache = set()):

        if not isinstance(matchfilter,set):
            matchfilter = set()

        mydbconn = self.openRepositoryDatabase(atomInfo[1])
        myatom = mydbconn.retrieveAtom(atomInfo[0])

        # caches
        # special events
        deps_not_found = set()
        conflicts = set()

        mydep = (1,myatom)
        mybuffer = self.entropyTools.lifobuffer()
        deptree = set()
        if atomInfo not in matchfilter:
            deptree.add((1,atomInfo))

        virgin = True
        open_repo = self.openRepositoryDatabase
        atom_match = self.atomMatch
        cdb_atom_match = self.clientDbconn.atomMatch
        lookup_conflict_replacement = self._lookup_conflict_replacement
        lookup_library_breakages = self._lookup_library_breakages
        lookup_inverse_dependencies = self._lookup_inverse_dependencies
        get_unsatisfied_deps = self.get_unsatisfied_dependencies

        def my_dep_filter(x):
            if x in treecache: return False
            if tuple(x.split(":")) in keyslotcache: return False
            return True

        while mydep:

            dep_level, dep_atom = mydep

            # already analyzed in this call
            if dep_atom in treecache:
                mydep = mybuffer.pop()
                continue
            treecache.add(dep_atom)

            if dep_atom == None: # corrupted entry
                mydep = mybuffer.pop()
                continue

            # conflicts
            if dep_atom[0] == "!":
                c_idpackage, xst = cdb_atom_match(dep_atom[1:])
                if c_idpackage != -1:
                    myreplacement = lookup_conflict_replacement(dep_atom[1:], c_idpackage, deep_deps = deep_deps)
                    if (myreplacement != None) and (myreplacement not in treecache):
                        mybuffer.push((dep_level+1,myreplacement))
                    else:
                        conflicts.add(c_idpackage)
                mydep = mybuffer.pop()
                continue

            # atom found?
            if virgin:
                virgin = False
                m_idpackage, m_repo = atomInfo
                dbconn = open_repo(m_repo)
                myidpackage, idreason = dbconn.idpackageValidator(m_idpackage)
                if myidpackage == -1: m_idpackage = -1
            else:
                m_idpackage, m_repo = atom_match(dep_atom)
            if m_idpackage == -1:
                deps_not_found.add(dep_atom)
                mydep = mybuffer.pop()
                continue

            # check if atom has been already pulled in
            matchdb = open_repo(m_repo)
            matchatom = matchdb.retrieveAtom(m_idpackage)
            matchkey, matchslot = matchdb.retrieveKeySlot(m_idpackage)
            if (dep_atom != matchatom) and (matchatom in treecache):
                mydep = mybuffer.pop()
                continue

            treecache.add(matchatom)

            # check if key + slot has been already pulled in
            if (matchslot,matchkey) in keyslotcache:
                mydep = mybuffer.pop()
                continue
            else:
                keyslotcache.add((matchslot,matchkey))

            match = (m_idpackage, m_repo,)
            # result already analyzed?
            if match in matchfilter:
                mydep = mybuffer.pop()
                continue

            # already analyzed by the calling function
            if match in matchfilter:
                mydep = mybuffer.pop()
                continue
            matchfilter.add(match)

            treedepth = dep_level+1

            # all checks passed, well done
            matchfilter.add(match)
            deptree.add((dep_level,match)) # add match

            # extra hooks
            cm_idpackage, cm_result = cdb_atom_match(matchkey, matchSlot = matchslot)
            if cm_idpackage != -1:
                broken_atoms = lookup_library_breakages(match, (cm_idpackage, cm_result,), deep_deps = deep_deps)
                inverse_deps = lookup_inverse_dependencies(match, (cm_idpackage, cm_result,))
                if inverse_deps:
                    deptree.remove((dep_level,match))
                    for ikey,islot in inverse_deps:
                        iks_str = '%s:%s' % (ikey,islot,)
                        if ((ikey,islot) not in keyslotcache) and (iks_str not in treecache):
                            mybuffer.push((dep_level,iks_str))
                            keyslotcache.add((ikey,islot))
                    deptree.add((treedepth,match))
                    treedepth += 1
                for x in broken_atoms:
                    if (tuple(x.split(":")) not in keyslotcache) and (x not in treecache):
                        mybuffer.push((treedepth,x))

            myundeps = filter(my_dep_filter,matchdb.retrieveDependenciesList(m_idpackage))
            if not empty_deps:
                myundeps = filter(my_dep_filter,get_unsatisfied_deps(myundeps, deep_deps, depcache = filter_unsat_cache))

            # PDEPENDs support
            if myundeps:
                post_deps = [x for x in matchdb.retrievePostDependencies(m_idpackage) if x in myundeps]
                myundeps = [x for x in myundeps if x not in post_deps]
                for x in post_deps: mybuffer.push((-1,x)) # always after the package itself

            for x in myundeps: mybuffer.push((treedepth,x))
            mydep = mybuffer.pop()

        if deps_not_found:
            return list(deps_not_found),-2

        if flat: return [x[1] for x in deptree],0

        newdeptree = {}
        for key,item in deptree:
            if key not in newdeptree: newdeptree[key] = set()
            newdeptree[key].add(item)
        # conflicts
        newdeptree[0] = conflicts

        return newdeptree,0 # note: newtree[0] contains possible conflicts


    def _lookup_system_mask_repository_deps(self):

        data = self.SystemSettings['repos_system_mask']
        if not data: return []
        mydata = []
        cached_items = set()
        for atom in data:
            mymatch = self.atomMatch(atom)
            if mymatch[0] == -1: # ignore missing ones intentionally
                continue
            if mymatch in cached_items:
                continue
            if mymatch not in mydata:
                # check if not found
                myaction = self.get_package_action(mymatch)
                if myaction != 0: mydata.append(mymatch) # upgrades, installs and downgrades allowed
            cached_items.add(mymatch)
        return mydata

    def _lookup_conflict_replacement(self, conflict_atom, client_idpackage, deep_deps):
        if self.entropyTools.isjustname(conflict_atom):
            return None
        conflict_match = self.atomMatch(conflict_atom)
        mykey, myslot = self.clientDbconn.retrieveKeySlot(client_idpackage)
        new_match = self.atomMatch(mykey, matchSlot = myslot)
        if (conflict_match == new_match) or (new_match[1] == 1):
            return None
        action = self.get_package_action(new_match)
        if (action == 0) and (not deep_deps):
            return None
        return "%s:%s" % (mykey,myslot,)

    def _lookup_inverse_dependencies(self, match, clientmatch):

        cmpstat = self.get_package_action(match)
        if cmpstat == 0: return set()

        keyslots = set()
        mydepends = self.clientDbconn.retrieveDepends(clientmatch[0])
        am = self.atomMatch
        cdb_rdeps = self.clientDbconn.retrieveDependencies
        cdb_rks = self.clientDbconn.retrieveKeySlot
        gpa = self.get_package_action
        keyslots_cache = set()
        match_cache = {}

        for idpackage in mydepends:
            key, slot = cdb_rks(idpackage)
            if (key,slot) in keyslots_cache: continue
            keyslots_cache.add((key,slot))
            if (key,slot) in keyslots: continue
            # grab its deps
            mydeps = cdb_rdeps(idpackage)
            found = False
            for mydep in mydeps:
                mymatch = match_cache.get(mydep, 0)
                if mymatch == 0:
                    mymatch = am(mydep)
                    match_cache[mydep] = mymatch
                if mymatch == match:
                    found = True
                    break
            if not found:
                mymatch = am(key, matchSlot = slot)
                if mymatch[0] == -1: continue
                cmpstat = gpa(mymatch)
                if cmpstat == 0: continue
                keyslots.add((key,slot))

        return keyslots

    def _lookup_library_breakages(self, match, clientmatch, deep_deps = False):

        # there is no need to update this cache when "match" will be installed, because at that point
        # clientmatch[0] will differ.
        c_hash = "%s|%s|%s" % (hash(tuple(match)),hash(deep_deps),hash(tuple(clientmatch)),)
        c_hash = "%s%s" % (etpCache['library_breakage'],hash(c_hash),)
        if self.xcache:
            cached = self.Cacher.pop(c_hash)
            if cached != None: return cached

        # these should be pulled in before
        repo_atoms = set()
        # these can be pulled in after
        client_atoms = set()

        matchdb = self.openRepositoryDatabase(match[1])
        reponeeded = matchdb.retrieveNeeded(match[0], extended = True, format = True)
        clientneeded = self.clientDbconn.retrieveNeeded(clientmatch[0], extended = True, format = True)
        repo_split = [x.split(".so")[0] for x in reponeeded]
        client_split = [x.split(".so")[0] for x in clientneeded]
        client_side = [x for x in clientneeded if (x not in reponeeded) and (x.split(".so")[0] in repo_split)]
        repo_side = [x for x in reponeeded if (x not in clientneeded) and (x.split(".so")[0] in client_split)]
        del clientneeded,client_split,repo_split

        # all the packages in client_side should be pulled in and updated
        client_idpackages = set()
        for needed in client_side: client_idpackages |= self.clientDbconn.searchNeeded(needed)

        client_keyslots = set()
        def mymf(idpackage):
            if idpackage == clientmatch[0]: return 0
            return self.clientDbconn.retrieveKeySlot(idpackage)
        client_keyslots = set([x for x in map(mymf,client_idpackages) if x != 0])

        # all the packages in repo_side should be pulled in too
        repodata = {}
        for needed in repo_side:
            repodata[needed] = reponeeded[needed]
        del repo_side,reponeeded

        repo_dependencies = matchdb.retrieveDependencies(match[0])
        matched_deps = set()
        matched_repos = set()
        for dependency in repo_dependencies:
            depmatch = self.atomMatch(dependency)
            if depmatch[0] == -1:
                continue
            matched_repos.add(depmatch[1])
            matched_deps.add(depmatch)

        matched_repos = [x for x in etpRepositoriesOrder if x in matched_repos]
        found_matches = set()
        for needed in repodata:
            for myrepo in matched_repos:
                mydbc = self.openRepositoryDatabase(myrepo)
                solved_needed = mydbc.resolveNeeded(needed, elfclass = repodata[needed])
                found = False
                for idpackage,myfile in solved_needed:
                    x = (idpackage,myrepo)
                    if x in matched_deps:
                        found_matches.add(x)
                        found = True
                        break
                if found:
                    break

        for idpackage,repo in found_matches:
            if not deep_deps:
                cmpstat = self.get_package_action((idpackage,repo))
                if cmpstat == 0:
                    continue
            mydbc = self.openRepositoryDatabase(repo)
            repo_atoms.add(mydbc.retrieveAtom(idpackage))

        for key, slot in client_keyslots:
            idpackage, repo = self.atomMatch(key, matchSlot = slot)
            if idpackage == -1:
                continue
            if not deep_deps:
                cmpstat = self.get_package_action((idpackage, repo))
                if cmpstat == 0:
                    continue
            mydbc = self.openRepositoryDatabase(repo)
            client_atoms.add(mydbc.retrieveAtom(idpackage))

        client_atoms |= repo_atoms

        if self.xcache:
            self.Cacher.push(c_hash,client_atoms.copy())

        return client_atoms


    def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False, quiet = False):

        c_hash = "%s%s" % (etpCache['dep_tree'],hash("%s|%s|%s|%s" % (
            hash(frozenset(sorted(matched_atoms))),hash(empty_deps),
            hash(deep_deps),self.clientDbconn.database_checksum(),
        )),)
        if self.xcache:
            cached = self.Cacher.pop(c_hash)
            if cached != None: return cached

        deptree = {}
        deptree[0] = set()

        atomlen = len(matched_atoms); count = 0
        error_generated = 0
        error_tree = set()

        # check if there are repositories needing some mandatory packages
        forced_matches = self._lookup_system_mask_repository_deps()
        if forced_matches:
            if isinstance(matched_atoms, list):
                matched_atoms = forced_matches + [x for x in matched_atoms if x not in forced_matches]
            elif isinstance(matched_atoms, set): # we cannot do anything about the order here
                matched_atoms |= set(forced_matches)

        sort_dep_text = _("Sorting dependencies")
        filter_unsat_cache = {}
        treecache = set()
        keyslotcache = set()
        matchfilter = set()
        for atomInfo in matched_atoms:

            if not quiet:
                count += 1
                if (count%10 == 0) or (count == atomlen) or (count == 1):
                    self.updateProgress(sort_dep_text, importance = 0, type = "info", back = True, header = ":: ", footer = " ::", percent = True, count = (count,atomlen))

            if atomInfo in matchfilter: continue
            newtree, result = self.generate_dependency_tree(
                atomInfo, empty_deps, deep_deps,
                matchfilter = matchfilter, filter_unsat_cache = filter_unsat_cache, treecache = treecache,
                keyslotcache = keyslotcache
            )

            if result == -2: # deps not found
                error_generated = -2
                error_tree |= set(newtree) # it is a list, we convert it into set and update error_tree
            elif (result != 0):
                return newtree, result
            elif newtree:
                # add conflicts
                max_parent_key = max(deptree)
                deptree[0] |= newtree.pop(0)
                levelcount = 0
                for mylevel in sorted(newtree.keys(), reverse = True):
                    levelcount += 1
                    deptree[max_parent_key+levelcount] = newtree.get(mylevel)

        if error_generated != 0:
            return error_tree,error_generated

        if self.xcache:
            self.Cacher.push(c_hash,(deptree,0)[:])

        return deptree,0

    def _filter_depends_multimatched_atoms(self, idpackage, depends, monotree):
        remove_depends = set()
        for d_idpackage in depends:
            mydeps = self.clientDbconn.retrieveDependencies(d_idpackage)
            for mydep in mydeps:
                matches, rslt = self.clientDbconn.atomMatch(mydep, multiMatch = True)
                if rslt == 1: continue
                if idpackage in matches and len(matches) > 1:
                    # are all in depends?
                    for mymatch in matches:
                        if mymatch not in depends and mymatch not in monotree:
                            remove_depends.add(d_idpackage)
                            break
        depends -= remove_depends
        return depends


    def generate_depends_tree(self, idpackages, deep = False):

        c_hash = "%s%s" % (etpCache['depends_tree'],hash("%s|%s" % (hash(tuple(sorted(idpackages))),hash(deep),),),)
        if self.xcache:
            cached = self.Cacher.pop(c_hash)
            if cached != None: return cached

        dependscache = set()
        treeview = set(idpackages)
        treelevel = set(idpackages)
        tree = {}
        treedepth = 0 # I start from level 1 because level 0 is idpackages itself
        tree[treedepth] = set(idpackages)
        monotree = set(idpackages) # monodimensional tree

        # check if dependstable is sane before beginning
        self.clientDbconn.retrieveDepends(idpackages[0])
        count = 0

        rem_dep_text = _("Calculating removable depends of")
        while 1:
            treedepth += 1
            tree[treedepth] = set()
            for idpackage in treelevel:

                count += 1
                p_atom = self.clientDbconn.retrieveAtom(idpackage)
                self.updateProgress(
                    blue(rem_dep_text + " %s" % (red(p_atom),)),
                    importance = 0,
                    type = "info",
                    back = True,
                    header = '|/-\\'[count%4]+" "
                )

                systempkg = not self.validatePackageRemoval(idpackage)
                if (idpackage in dependscache) or systempkg:
                    if idpackage in treeview:
                        treeview.remove(idpackage)
                    continue

                # obtain its depends
                depends = self.clientDbconn.retrieveDepends(idpackage)
                # filter already satisfied ones
                depends = set([x for x in depends if x not in monotree])
                depends = set([x for x in depends if self.validatePackageRemoval(x)])
                if depends:
                    depends = self._filter_depends_multimatched_atoms(idpackage, depends, monotree)
                if depends: # something depends on idpackage
                    tree[treedepth] |= depends
                    monotree |= depends
                    treeview |= depends
                elif deep: # if deep, grab its dependencies and check

                    mydeps = set()
                    for x in self.clientDbconn.retrieveDependencies(idpackage):
                        match = self.clientDbconn.atomMatch(x)
                        if match[0] != -1:
                            mydeps.add(match[0])

                    # now filter them
                    mydeps = [x for x in mydeps if x not in monotree and not (self.clientDbconn.isSystemPackage(x) or self.is_installed_idpackage_in_system_mask(x) )]
                    for x in mydeps:
                        mydepends = self.clientDbconn.retrieveDepends(x)
                        mydepends -= set([y for y in mydepends if y not in monotree])
                        if not mydepends:
                            tree[treedepth].add(x)
                            monotree.add(x)
                            treeview.add(x)

                dependscache.add(idpackage)
                if idpackage in treeview:
                    treeview.remove(idpackage)

            treelevel = treeview.copy()
            if not treelevel:
                if not tree[treedepth]:
                    del tree[treedepth] # probably the last one is empty then
                break

        # now filter newtree
        for count in sorted(tree.keys(), reverse = True):
            x = 0
            while x < count:
                tree[x] -= tree[count]
                x += 1

        if self.xcache:
            self.Cacher.push(c_hash,(tree,0)[:])
        return tree,0 # treeview is used to show deps while tree is used to run the dependency code.

    def list_repo_categories(self):
        categories = set()
        for repo in self.validRepositories:
            dbconn = self.openRepositoryDatabase(repo)
            catsdata = dbconn.listAllCategories()
            categories.update(set([x[1] for x in catsdata]))
        return categories

    def list_repo_packages_in_category(self, category):
        pkg_matches = []
        for repo in self.validRepositories:
            dbconn = self.openRepositoryDatabase(repo)
            catsdata = dbconn.searchPackagesByCategory(category, branch = etpConst['branch'])
            pkg_matches.extend([(x[1],repo,) for x in catsdata if (x[1],repo,) not in pkg_matches])
        return pkg_matches

    def get_category_description_data(self, category):

        data = {}
        for repo in self.validRepositories:
            try:
                dbconn = self.openRepositoryDatabase(repo)
            except exceptionTools.RepositoryError:
                continue
            try:
                data = dbconn.retrieveCategoryDescription(category)
            except (dbapi2.OperationalError, dbapi2.IntegrityError,):
                continue
            if data: break

        return data

    def list_installed_packages_in_category(self, category):
        pkg_matches = set([x[1] for x in self.clientDbconn.searchPackagesByCategory(category)])
        return pkg_matches

    def all_repositories_checksum(self):
        sum_hashes = ''
        for repo in self.validRepositories:
            try:
                dbconn = self.openRepositoryDatabase(repo)
            except (exceptionTools.RepositoryError):
                continue # repo not available
            try:
                sum_hashes += dbconn.database_checksum()
            except dbapi2.OperationalError:
                pass
        return sum_hashes

    def get_available_packages_chash(self, branch):
        repo_digest = self.all_repositories_checksum()
        # client digest not needed, cache is kept updated
        c_hash = "%s%s%s" % (hash(repo_digest),hash(branch),hash(tuple(self.validRepositories)),)
        return str(hash(c_hash))

    def get_available_packages_cache(self, branch = etpConst['branch'], myhash = None):
        if myhash == None:
            myhash = self.get_available_packages_chash(branch)
        disk_cache = self.Cacher.pop(etpCache['world_available'])
        try:
            if disk_cache['chash'] == myhash:
                return disk_cache['available']
        except (KeyError,TypeError,):
            return None

    # this function searches all the not installed packages available in the repositories
    def calculate_available_packages(self, use_cache = True):

        c_hash = self.get_available_packages_chash(etpConst['branch'])

        if use_cache and self.xcache:
            cached = self.get_available_packages_cache(myhash = c_hash)
            if cached != None:
                return cached

        available = []
        self.setTotalCycles(len(self.validRepositories))
        avail_dep_text = _("Calculating available packages for")
        for repo in self.validRepositories:
            try:
                dbconn = self.openRepositoryDatabase(repo)
                dbconn.validateDatabase()
            except (exceptionTools.RepositoryError,exceptionTools.SystemDatabaseError):
                self.cycleDone()
                continue
            idpackages = [  x for x in dbconn.listAllIdpackages(branch = etpConst['branch'], branch_operator = "<=", order_by = 'atom') \
                            if dbconn.idpackageValidator(x)[0] != -1  ]
            count = 0
            maxlen = len(idpackages)
            myavailable = []
            do_break = False
            for idpackage in idpackages:
                if do_break:
                    break
                count += 1
                if (count % 10 == 0) or (count == 1) or (count == maxlen):
                    self.updateProgress(
                        avail_dep_text + " %s" % (repo,),
                        importance = 0,
                        type = "info",
                        back = True,
                        header = "::",
                        count = (count,maxlen),
                        percent = True,
                        footer = " ::"
                    )
                # get key + slot
                try:
                    key, slot = dbconn.retrieveKeySlot(idpackage)
                    matches = self.clientDbconn.searchKeySlot(key, slot)
                except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
                    self.cycleDone()
                    do_break = True
                    continue
                if not matches: myavailable.append((idpackage,repo))
            available += myavailable[:]
            self.cycleDone()

        if self.xcache:
            data = {}
            data['chash'] = c_hash
            data['available'] = available
            self.Cacher.push(etpCache['world_available'],data.copy())
        return available

    def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None, ignore_spm_downgrades = False):
        if self.xcache:
            if db_digest == None: db_digest = self.all_repositories_checksum()
            c_hash = "%s%s" % (etpCache['world_update'],self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades),)
            disk_cache = self.Cacher.pop(c_hash)
            if disk_cache != None:
                try:
                    return disk_cache['r']
                except (KeyError, TypeError):
                    return None

    def get_world_update_cache_hash(self, db_digest, empty_deps, branch, ignore_spm_downgrades):
        c_hash = "%s%s%s%s%s%s" % ( 
            hash(db_digest),empty_deps,hash(tuple(self.validRepositories)),
            hash(tuple(etpRepositoriesOrder)), branch, ignore_spm_downgrades,
        )
        return str(hash(c_hash))

    def calculate_world_updates(
            self,
            empty_deps = False,
            branch = etpConst['branch'],
            ignore_spm_downgrades = etpConst['spm']['ignore-spm-downgrades'],
            use_cache = True
        ):

        db_digest = self.all_repositories_checksum()
        if use_cache and self.xcache:
            cached = self.get_world_update_cache(empty_deps = empty_deps, branch = branch, db_digest = db_digest, ignore_spm_downgrades = ignore_spm_downgrades)
            if cached != None: return cached

        update = []
        remove = []
        fine = []

        # get all the installed packages
        idpackages = self.clientDbconn.listAllIdpackages(order_by = 'atom')
        maxlen = len(idpackages)
        count = 0
        mytxt = _("Calculating world packages")
        for idpackage in idpackages:

            count += 1
            if (count%10 == 0) or (count == maxlen) or (count == 1):
                self.updateProgress(
                    mytxt,
                    importance = 0,
                    type = "info",
                    back = True,
                    header = ":: ",
                    count = (count,maxlen),
                    percent = True,
                    footer = " ::"
                )

            mystrictdata = self.clientDbconn.getStrictData(idpackage)
            # check against broken entries, or removed during iteration
            if mystrictdata == None:
                continue
            use_match_cache = True
            do_continue = False
            while 1:
                try:
                    match = self.atomMatch(
                        mystrictdata[0],
                        matchSlot = mystrictdata[1],
                        matchBranches = (branch,),
                        extendedResults = True,
                        useCache = use_match_cache
                    )
                except dbapi2.OperationalError:
                    # ouch, but don't crash here
                    do_continue = True
                    break
                try:
                    m_idpackage = match[0][0]
                except TypeError:
                    if not use_match_cache: raise
                    use_match_cache = False
                    continue
                break
            if do_continue: continue
            # now compare
            # version: mystrictdata[2]
            # tag: mystrictdata[3]
            # revision: mystrictdata[4]
            if (m_idpackage != -1):
                repoid = match[1]
                version = match[0][1]
                tag = match[0][2]
                revision = match[0][3]
                if empty_deps:
                    if (m_idpackage,repoid) not in update:
                        update.append((m_idpackage,repoid))
                    continue
                elif (mystrictdata[2] != version):
                    # different versions
                    if (m_idpackage,repoid) not in update:
                        update.append((m_idpackage,repoid))
                    continue
                elif (mystrictdata[3] != tag):
                    # different tags
                    if (m_idpackage,repoid) not in update:
                        update.append((m_idpackage,repoid))
                    continue
                elif (mystrictdata[4] != revision):
                    # different revision
                    if mystrictdata[4] == 9999 and ignore_spm_downgrades:
                        # no difference, we're ignoring revision 9999
                        fine.append(mystrictdata[5])
                        continue
                    else:
                        if (m_idpackage,repoid) not in update:
                            update.append((m_idpackage,repoid))
                        continue
                else:
                    # no difference
                    fine.append(mystrictdata[5])
                    continue

            # don't take action if it's just masked
            maskedresults = self.atomMatch(mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), packagesFilter = False)
            if maskedresults[0] == -1:
                remove.append(idpackage)
                # look for packages that would match key with any slot (for eg: gcc, kernel updates)
                matchresults = self.atomMatch(mystrictdata[0], matchBranches = (branch,))
                if matchresults[0] != -1:
                    m_action = self.get_package_action(matchresults)
                    if m_action > 0 and (matchresults not in update):
                        update.append(matchresults)

        if self.xcache:
            c_hash = self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades)
            data = {
                'r': (update, remove, fine,),
                'empty_deps': empty_deps,
            }
            self.Cacher.push("%s%s" % (etpCache['world_update'],c_hash,), data.copy())

        return update, remove, fine

    def get_match_conflicts(self, match):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        conflicts = dbconn.retrieveConflicts(m_id)
        found_conflicts = set()
        for conflict in conflicts:
            my_m_id, my_m_rc = self.clientDbconn.atomMatch(conflict)
            if my_m_id != -1:
                # check if the package shares the same slot
                match_data = dbconn.retrieveKeySlot(m_id)
                installed_match_data = self.clientDbconn.retrieveKeySlot(my_m_id)
                if match_data != installed_match_data:
                    found_conflicts.add(my_m_id)
        return found_conflicts

    def is_match_masked(self, match, live_check = True):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check)
        if idpackage != -1:
            return False
        return True

    def is_match_masked_by_user(self, match, live_check = True):
        # (query_status,masked?,)
        m_id, m_repo = match
        if m_repo not in self.validRepositories: return False
        dbconn = self.openRepositoryDatabase(m_repo)
        idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check)
        if idpackage != -1: return False #,False
        myr = self.SystemSettings['pkg_masking_reference']
        user_masks = [myr['user_package_mask'],myr['user_license_mask'],myr['user_live_mask']]
        if idreason in user_masks:
            return True #,True
        return False #,True

    def is_match_unmasked_by_user(self, match, live_check = True):
        # (query_status,unmasked?,)
        m_id, m_repo = match
        if m_repo not in self.validRepositories: return False
        dbconn = self.openRepositoryDatabase(m_repo)
        idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check)
        if idpackage == -1: return False #,False
        myr = self.SystemSettings['pkg_masking_reference']
        user_masks = [
            myr['user_package_unmask'],myr['user_live_unmask'],myr['user_package_keywords'],
            myr['user_repo_package_keywords_all'], myr['user_repo_package_keywords']
        ]
        if idreason in user_masks:
            return True #,True
        return False #,True

    def mask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False):
        if self.is_match_masked(match, live_check = False): return True
        methods = {
            'atom': self.mask_match_by_atom,
            'keyslot': self.mask_match_by_keyslot,
        }
        rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache)
        if dry_run: # inject if done "live"
            self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match)
            self.SystemSettings['live_packagemasking']['mask_matches'].add(match)
        return rc

    def unmask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False):
        if not self.is_match_masked(match, live_check = False): return True
        methods = {
            'atom': self.unmask_match_by_atom,
            'keyslot': self.unmask_match_by_keyslot,
        }
        rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache)
        if dry_run: # inject if done "live"
            self.SystemSettings['live_packagemasking']['unmask_matches'].add(match)
            self.SystemSettings['live_packagemasking']['mask_matches'].discard(match)
        return rc

    def _mask_unmask_match(self, match, method, methods_reference, dry_run = False, clean_all_cache = False):

        f = methods_reference.get(method)
        if not callable(f):
            raise exceptionTools.IncorrectParameter('IncorrectParameter: %s: %s' % (_("not a valid method"),method,) )

        self.Cacher.sync(wait = True)
        done = f(match, dry_run)
        if done: self.SystemSettings.clear()

        # clear atomMatch cache anyway
        if clean_all_cache and not dry_run:
            self.clear_dump_cache(etpCache['world_available'])
            self.clear_dump_cache(etpCache['world_update'])

        self.clear_dump_cache(etpCache['check_package_update'])
        self.clear_dump_cache(etpCache['filter_satisfied_deps'])
        self.clear_dump_cache(self.atomMatchCacheKey)
        self.clear_dump_cache(etpCache['dep_tree'])
        self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],))
        self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],))

        self.package_match_validator_cache.clear()
        return done

    def unmask_match_by_atom(self, match, dry_run = False):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        atom = dbconn.retrieveAtom(m_id)
        return self.unmask_match_generic(match, atom, dry_run = dry_run)

    def unmask_match_by_keyslot(self, match, dry_run = False):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id)
        return self.unmask_match_generic(match, keyslot, dry_run = dry_run)

    def mask_match_by_atom(self, match, dry_run = False):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        atom = dbconn.retrieveAtom(m_id)
        return self.mask_match_generic(match, atom, dry_run = dry_run)

    def mask_match_by_keyslot(self, match, dry_run = False):
        m_id, m_repo = match
        dbconn = self.openRepositoryDatabase(m_repo)
        keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id)
        return self.mask_match_generic(match, keyslot, dry_run = dry_run)

    def unmask_match_generic(self, match, keyword, dry_run = False):
        self.clear_match_mask(match, dry_run)
        m_file = self.SystemSettings.etpSettingFiles['unmask']
        return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run)

    def mask_match_generic(self, match, keyword, dry_run = False):
        self.clear_match_mask(match, dry_run)
        m_file = self.SystemSettings.etpSettingFiles['mask']
        return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run)

    def _mask_unmask_match_generic(self, keyword, m_file, dry_run = False):
        exist = False
        if not os.path.isfile(m_file):
            if not os.access(os.path.dirname(m_file),os.W_OK):
                return False # cannot write
        elif not os.access(m_file, os.W_OK):
            return False
        elif not dry_run:
            exist = True

        if dry_run:
            return True

        content = []
        if exist:
            f = open(m_file,"r")
            content = [x.strip() for x in f.readlines()]
            f.close()
        content.append(keyword)
        m_file_tmp = m_file+".tmp"
        f = open(m_file_tmp,"w")
        for line in content:
            f.write(line+"\n")
        f.flush()
        f.close()
        shutil.move(m_file_tmp,m_file)
        return True

    def clear_match_mask(self, match, dry_run = False):
        masking_list = [self.SystemSettings.etpSettingFiles['mask'],self.SystemSettings.etpSettingFiles['unmask']]
        return self._clear_match_generic(match, masking_list = masking_list, dry_run = dry_run)

    def _clear_match_generic(self, match, masking_list = [], dry_run = False):

        self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match)
        self.SystemSettings['live_packagemasking']['mask_matches'].discard(match)

        if dry_run: return

        for mask_file in masking_list:
            if not (os.path.isfile(mask_file) and os.access(mask_file,os.W_OK)): continue
            f = open(mask_file,"r")
            newf = self.entropyTools.open_buffer()
            line = f.readline()
            while line:
                line = line.strip()
                if line.startswith("#"):
                    newf.write(line+"\n")
                    line = f.readline()
                    continue
                elif not line:
                    newf.write("\n")
                    line = f.readline()
                    continue
                mymatch = self.atomMatch(line, packagesFilter = False)
                if mymatch == match:
                    line = f.readline()
                    continue
                newf.write(line+"\n")
                line = f.readline()
            f.close()
            tmpfile = mask_file+".w_tmp"
            f = open(tmpfile,"w")
            f.write(newf.getvalue())
            f.flush()
            f.close()
            newf.close()
            shutil.move(tmpfile,mask_file)

    def add_user_package_set(self, set_name, set_atoms):

        def _ensure_package_sets_dir():
            sets_dir = etpConst['confsetsdir']
            if not os.path.isdir(sets_dir):
                if os.path.lexists(sets_dir):
                    os.remove(sets_dir)
                os.makedirs(sets_dir,0755)
                const_setup_perms(sets_dir, etpConst['entropygid'])

        try:
            set_name = str(set_name)
        except (UnicodeEncodeError,UnicodeDecodeError,):
            raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),))

        if set_name.startswith(etpConst['packagesetprefix']):
            raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],))
        set_match, rc = self.packageSetMatch(set_name)
        if rc: return -1,_("Name already taken")

        _ensure_package_sets_dir()
        set_file = os.path.join(etpConst['confsetsdir'],set_name)
        if os.path.isfile(set_file) and os.access(set_file,os.W_OK):
            try:
                os.remove(set_file)
            except OSError:
                return -2,_("Cannot remove the old element")
        if not os.access(os.path.dirname(set_file),os.W_OK):
            return -3,_("Cannot create the element")

        f = open(set_file,"w")
        for x in set_atoms: f.write("%s\n" % (x,))
        f.flush()
        f.close()
        self.SystemSettings['system_package_sets'][set_name] = set(set_atoms)
        return 0,_("All fine")

    def remove_user_package_set(self, set_name):

        try:
            set_name = str(set_name)
        except (UnicodeEncodeError,UnicodeDecodeError,):
            raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),))

        if set_name.startswith(etpConst['packagesetprefix']):
            raise exceptionTools.InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],))

        set_match, rc = self.packageSetMatch(set_name)
        if not rc: return -1,_("Already removed")
        set_id, set_x, set_y = set_match

        if set_id != etpConst['userpackagesetsid']:
            return -2,_("Not defined by user")
        set_file = os.path.join(etpConst['confsetsdir'],set_name)
        if os.path.isfile(set_file) and os.access(set_file,os.W_OK):
            os.remove(set_file)
            if set_name in self.SystemSettings['system_package_sets']:
                del self.SystemSettings['system_package_sets'][set_name]
            return 0,_("All fine")
        return -3,_("Set not found or unable to remove")

    # every tbz2 file that would be installed must pass from here
    def add_tbz2_to_repos(self, tbz2file):
        atoms_contained = []
        basefile = os.path.basename(tbz2file)
        if os.path.isdir(etpConst['entropyunpackdir']+"/"+basefile[:-5]):
            shutil.rmtree(etpConst['entropyunpackdir']+"/"+basefile[:-5])
        os.makedirs(etpConst['entropyunpackdir']+"/"+basefile[:-5])
        dbfile = self.entropyTools.extractEdb(tbz2file, dbpath = etpConst['entropyunpackdir']+"/"+basefile[:-5]+"/packages.db")
        if dbfile == None:
            return -1,atoms_contained
        etpSys['dirstoclean'].add(os.path.dirname(dbfile))
        # add dbfile
        repodata = {}
        repodata['repoid'] = basefile
        repodata['description'] = "Dynamic database from "+basefile
        repodata['packages'] = []
        repodata['dbpath'] = os.path.dirname(dbfile)
        repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added
        repodata['smartpackage'] = False # extra info added

        mydbconn = self.openGenericDatabase(dbfile)
        # read all idpackages
        try:
            myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files
        except:
            return -2,atoms_contained
        if len(myidpackages) > 1:
            repodata[basefile]['smartpackage'] = True
        for myidpackage in myidpackages:
            compiled_arch = mydbconn.retrieveDownloadURL(myidpackage)
            if compiled_arch.find("/"+etpSys['arch']+"/") == -1:
                return -3,atoms_contained
            atoms_contained.append((int(myidpackage),basefile))

        self.addRepository(repodata)
        self.validate_repositories()
        if basefile not in self.validRepositories:
            self.removeRepository(basefile)
            return -4,atoms_contained
        mydbconn.closeDB()
        del mydbconn
        return 0,atoms_contained

    # This is the function that should be used by third party applications
    # to retrieve a list of available updates, along with conflicts (removalQueue) and obsoletes
    # (removed)
    def retrieveWorldQueue(self, empty_deps = False, branch = etpConst['branch']):
        update, remove, fine = self.calculate_world_updates(empty_deps = empty_deps, branch = branch)
        del fine
        data = {}
        data['removed'] = list(remove)
        data['runQueue'] = []
        data['removalQueue'] = []
        status = -1
        if update:
            # calculate install+removal queues
            install, removal, status = self.retrieveInstallQueue(update, empty_deps, deep_deps = False)
            # update data['removed']
            data['removed'] = [x for x in data['removed'] if x not in removal]
            data['runQueue'] += install
            data['removalQueue'] += removal
        return data,status

    def validatePackageRemoval(self, idpackage):

        pkgatom = self.clientDbconn.retrieveAtom(idpackage)
        pkgkey = self.entropyTools.dep_getkey(pkgatom)

        if self.is_installed_idpackage_in_system_mask(idpackage):
            idpackages = self.SystemSettings['repos_system_mask_installed_keys'].get(pkgkey)
            if not idpackages: return False
            if len(idpackages) > 1:
                return True
            return False # sorry!

        # did we store the bastard in the db?
        system_pkg = self.clientDbconn.isSystemPackage(idpackage)
        if not system_pkg: return True
        # check if the package is slotted and exist more than one installed first
        sysresults = self.clientDbconn.atomMatch(pkgkey, multiMatch = True)
        if sysresults[1] == 0:
            if len(sysresults[0]) < 2: return False
            return True
        return False


    def retrieveRemovalQueue(self, idpackages, deep = False):
        queue = []
        if not idpackages:
            return queue
        treeview, status = self.generate_depends_tree(idpackages, deep = deep)
        if status == 0:
            for x in range(len(treeview))[::-1]: queue.extend(treeview[x])
        return queue

    def retrieveInstallQueue(self, matched_atoms, empty_deps, deep_deps, quiet = False):

        install = []
        removal = []
        treepackages, result = self.get_required_packages(matched_atoms, empty_deps, deep_deps, quiet = quiet)

        if result == -2:
            return treepackages,removal,result

        # format
        removal = treepackages.pop(0, set())
        for x in sorted(treepackages.keys()): install.extend(treepackages[x])

        # filter out packages that are in actionQueue comparing key + slot
        if install and removal:
            myremmatch = {}
            for x in removal:
                atom = self.clientDbconn.retrieveAtom(x)
                # XXX check if users removed idpackage while this whole instance is running
                if atom == None: continue
                myremmatch[(self.entropyTools.dep_getkey(atom),self.clientDbconn.retrieveSlot(x),)] = x
            for pkg_id, pkg_repo in install:
                dbconn = self.openRepositoryDatabase(pkg_repo)
                testtuple = (self.entropyTools.dep_getkey(dbconn.retrieveAtom(pkg_id)),dbconn.retrieveSlot(pkg_id))
                removal.discard(myremmatch.get(testtuple))

        return install, sorted(removal), 0

    # this function searches into client database for a package matching provided key + slot
    # and returns its idpackage or -1 if none found
    def retrieveInstalledIdPackage(self, pkgkey, pkgslot):
        match = self.clientDbconn.atomMatch(pkgkey, matchSlot = pkgslot)
        if match[1] == 0:
            return match[0]
        return -1

    '''
        Package interface :: begin
    '''

    def check_needed_package_download(self, filepath, checksum = None):
        # is the file available
        if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath):
            if checksum is None:
                return 0
            else:
                # check digest
                md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum)
                if (md5res):
                    return 0
                else:
                    return -2
        else:
            return -1

    def fetch_file(self, url, branch, digest = None, resume = True, fetch_file_abort_function = None, filepath = None):
        # remove old
        filename = os.path.basename(url)
        if not filepath:
            filepath = etpConst['packagesbindir']+"/"+branch+"/"+filename
        filepath_dir = os.path.dirname(filepath)
        if not os.path.isdir(filepath_dir):
            os.makedirs(filepath_dir,0755)

        # load class
        fetchConn = self.urlFetcher(url, filepath, resume = resume, abort_check_func = fetch_file_abort_function)
        fetchConn.progress = self.progress

        # start to download
        data_transfer = 0
        resumed = False
        try:
            fetchChecksum = fetchConn.download()
            data_transfer = fetchConn.datatransfer
            resumed = fetchConn.resumed
        except KeyboardInterrupt:
            return -4, data_transfer, resumed
        except NameError:
            raise
        except:
            return -1, data_transfer, resumed
        if fetchChecksum == "-3":
            return -3, data_transfer, resumed

        del fetchConn
        if digest:
            if fetchChecksum != digest:
                # not properly downloaded
                return -2, data_transfer, resumed
            else:
                return 0, data_transfer, resumed
        return 0, data_transfer, resumed

    def add_failing_mirror(self, mirrorname,increment = 1):
        item = self.mirrorDownloadFailures.get(mirrorname)
        if item == None:
            self.mirrorDownloadFailures[mirrorname] = increment
        else:
            self.mirrorDownloadFailures[mirrorname] += increment # add a failure
        return self.mirrorDownloadFailures[mirrorname]

    def get_failing_mirror_status(self, mirrorname):
        item = self.mirrorDownloadFailures.get(mirrorname)
        if item == None:
            return 0
        else:
            return item

    def fetch_file_on_mirrors(self, repository, branch, filename, digest = False, verified = False, fetch_abort_function = None):

        uris = etpRepositories[repository]['packages'][::-1]
        remaining = set(uris[:])

        if verified: # file is already in place, match_checksum set infoDict['verified'] to True
            return 0

        mirrorcount = 0
        for uri in uris:

            if not remaining:
                # tried all the mirrors, quitting for error
                return 3

            mirrorcount += 1
            mirrorCountText = "( mirror #"+str(mirrorcount)+" ) "
            url = uri+"/"+filename

            # check if uri is sane
            if self.get_failing_mirror_status(uri) >= 30:
                # ohohoh!
                self.mirrorDownloadFailures[uri] = 30 # set to 30 for convenience
                mytxt = mirrorCountText
                mytxt += blue(" %s: ") % (_("Mirror"),)
                mytxt += red(self.entropyTools.spliturl(url)[1])
                mytxt += " - %s." % (_("maximum failure threshold reached"),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )

                if self.get_failing_mirror_status(uri) == 30:
                    self.add_failing_mirror(uri,45) # put to 75 then decrement by 4 so we won't reach 30 anytime soon ahahaha
                else:
                    # now decrement each time this point is reached, if will be back < 30, then equo will try to use it again
                    if self.get_failing_mirror_status(uri) > 31:
                        self.add_failing_mirror(uri,-4)
                    else:
                        # put to 0 - reenable mirror, welcome back uri!
                        self.mirrorDownloadFailures[uri] = 0

                if uri in remaining:
                    remaining.remove(uri)
                continue

            do_resume = True
            while 1:
                try:
                    mytxt = mirrorCountText
                    mytxt += blue("%s: ") % (_("Downloading from"),)
                    mytxt += red(self.entropyTools.spliturl(url)[1])
                    # now fetch the new one
                    self.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "warning",
                        header = red("   ## ")
                    )
                    rc, data_transfer, resumed = self.fetch_file(
                        url,
                        branch,
                        digest,
                        do_resume,
                        fetch_file_abort_function = fetch_abort_function
                    )
                    if rc == 0:
                        mytxt = mirrorCountText
                        mytxt += blue("%s: ") % (_("Successfully downloaded from"),)
                        mytxt += red(self.entropyTools.spliturl(url)[1])
                        mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),)
                        self.updateProgress(
                            mytxt,
                            importance = 1,
                            type = "info",
                            header = red("   ## ")
                        )

                        return 0
                    elif resumed:
                        do_resume = False
                        continue
                    else:
                        error_message = mirrorCountText
                        error_message += blue("%s: %s") % (
                            _("Error downloading from"),
                            red(self.entropyTools.spliturl(url)[1]),
                        )
                        # something bad happened
                        if rc == -1:
                            error_message += " - %s." % (_("file not available on this mirror"),)
                        elif rc == -2:
                            self.add_failing_mirror(uri,1)
                            error_message += " - %s." % (_("wrong checksum"),)
                        elif rc == -3:
                            #self.add_failing_mirror(uri,2)
                            error_message += " - not found."
                        elif rc == -4:
                            error_message += " - %s." % (_("discarded download"),)
                        else:
                            self.add_failing_mirror(uri, 5)
                            error_message += " - %s." % (_("unknown reason"),)
                        self.updateProgress(
                                            error_message,
                                            importance = 1,
                                            type = "warning",
                                            header = red("   ## ")
                                        )
                        if rc == -4: # user discarded fetch
                            return 1
                        if uri in remaining:
                            remaining.remove(uri)
                        break
                except KeyboardInterrupt:
                    break
                except:
                    raise
        return 0

    def quickpkg(self, atomstring, savedir = None):
        if savedir == None:
            savedir = etpConst['packagestmpdir']
            if not os.path.isdir(etpConst['packagestmpdir']):
                os.makedirs(etpConst['packagestmpdir'])
        # match package
        match = self.clientDbconn.atomMatch(atomstring)
        if match[0] == -1:
            return -1,None,None
        atom = self.clientDbconn.atomMatch(match[0])
        pkgdata = self.clientDbconn.getPackageData(match[0])
        resultfile = self.quickpkg_handler(pkgdata = pkgdata, dirpath = savedir)
        if resultfile == None:
            return -1,atom,None
        else:
            return 0,atom,resultfile

    def quickpkg_handler(
                                self,
                                pkgdata,
                                dirpath,
                                edb = True,
                                portdbPath = None,
                                fake = False,
                                compression = "bz2",
                                shiftpath = ""
                        ):

        import stat
        import tarfile

        if compression not in ("bz2","","gz"):
            compression = "bz2"

        # getting package info
        pkgtag = ''
        pkgrev = "~"+str(pkgdata['revision'])
        if pkgdata['versiontag']: pkgtag = "#"+pkgdata['versiontag']
        pkgname = pkgdata['name']+"-"+pkgdata['version']+pkgrev+pkgtag # + version + tag
        pkgcat = pkgdata['category']
        #pkgfile = pkgname+etpConst['packagesext']
        dirpath += "/"+pkgname+etpConst['packagesext']
        if os.path.isfile(dirpath):
            os.remove(dirpath)
        tar = tarfile.open(dirpath,"w:"+compression)

        if not fake:

            contents = sorted([x for x in pkgdata['content']])
            id_strings = {}

            # collect files
            for path in contents:
                # convert back to filesystem str
                encoded_path = path
                path = path.encode('raw_unicode_escape')
                path = shiftpath+path
                try:
                    exist = os.lstat(path)
                except OSError:
                    continue # skip file
                arcname = path[len(shiftpath):] # remove shiftpath
                if arcname.startswith("/"):
                    arcname = arcname[1:] # remove trailing /
                ftype = pkgdata['content'][encoded_path]
                if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype
                if 'dir' == ftype and \
                    not stat.S_ISDIR(exist.st_mode) and \
                    os.path.isdir(path): # workaround for directory symlink issues
                    path = os.path.realpath(path)

                tarinfo = tar.gettarinfo(path, arcname)
                tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
                tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))

                if stat.S_ISREG(exist.st_mode):
                    tarinfo.type = tarfile.REGTYPE
                    f = open(path)
                    try:
                        tar.addfile(tarinfo, f)
                    finally:
                        f.close()
                else:
                    tar.addfile(tarinfo)

        tar.close()

        # appending xpak metadata
        if etpConst['gentoo-compat']:
            import etpXpak
            Spm = self.Spm()

            gentoo_name = self.entropyTools.remove_tag(pkgname)
            gentoo_name = self.entropyTools.remove_entropy_revision(gentoo_name)
            if portdbPath == None:
                dbdir = Spm.get_vdb_path()+"/"+pkgcat+"/"+gentoo_name+"/"
            else:
                dbdir = portdbPath+"/"+pkgcat+"/"+gentoo_name+"/"
            if os.path.isdir(dbdir):
                tbz2 = etpXpak.tbz2(dirpath)
                tbz2.recompose(dbdir)

        if edb:
            self.inject_entropy_database_into_package(dirpath, pkgdata)

        if os.path.isfile(dirpath):
            return dirpath
        return None

    def inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None):
        dbpath = self.get_tmp_dbpath()
        dbconn = self.openGenericDatabase(dbpath)
        dbconn.initializeDatabase()
        dbconn.addPackage(data, revision = data['revision'])
        if treeupdates_actions != None:
            dbconn.bumpTreeUpdatesActions(treeupdates_actions)
        dbconn.commitChanges()
        dbconn.closeDB()
        self.entropyTools.aggregateEdb(tbz2file = package_filename, dbfile = dbpath)
        return dbpath

    def get_tmp_dbpath(self):
        dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber())
        while os.path.isfile(dbpath):
            dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber())
        return dbpath

    def Package(self):
        conn = PackageInterface(EquoInstance = self)
        return conn

    '''
        Package interface :: end
    '''

    '''
        Source Package Manager Interface :: begin
    '''
    def Spm(self):
        myroot = etpConst['systemroot']
        cached = self.spmCache.get(myroot)
        if cached != None:
            return cached
        conn = SpmInterface(self)
        self.spmCache[myroot] = conn.intf
        return conn.intf

    def _extract_pkg_metadata_generate_extraction_dict(self):
        data = {
            'chost': {
                'path': etpConst['spm']['xpak_entries']['chost'],
                'critical': True,
            },
            'description': {
                'path': etpConst['spm']['xpak_entries']['description'],
                'critical': False,
            },
            'homepage': {
                'path': etpConst['spm']['xpak_entries']['homepage'],
                'critical': False,
            },
            'slot': {
                'path': etpConst['spm']['xpak_entries']['slot'],
                'critical': False,
            },
            'cflags': {
                'path': etpConst['spm']['xpak_entries']['cflags'],
                'critical': False,
            },
            'cxxflags': {
                'path': etpConst['spm']['xpak_entries']['cxxflags'],
                'critical': False,
            },
            'category': {
                'path': etpConst['spm']['xpak_entries']['category'],
                'critical': True,
            },
            'rdepend': {
                'path': etpConst['spm']['xpak_entries']['rdepend'],
                'critical': False,
            },
            'pdepend': {
                'path': etpConst['spm']['xpak_entries']['pdepend'],
                'critical': False,
            },
            'depend': {
                'path': etpConst['spm']['xpak_entries']['depend'],
                'critical': False,
            },
            'use': {
                'path': etpConst['spm']['xpak_entries']['use'],
                'critical': False,
            },
            'iuse': {
                'path': etpConst['spm']['xpak_entries']['iuse'],
                'critical': False,
            },
            'license': {
                'path': etpConst['spm']['xpak_entries']['license'],
                'critical': False,
            },
            'provide': {
                'path': etpConst['spm']['xpak_entries']['provide'],
                'critical': False,
            },
            'sources': {
                'path': etpConst['spm']['xpak_entries']['src_uri'],
                'critical': False,
            },
            'eclasses': {
                'path': etpConst['spm']['xpak_entries']['inherited'],
                'critical': False,
            },
            'counter': {
                'path': etpConst['spm']['xpak_entries']['counter'],
                'critical': False,
            },
            'keywords': {
                'path': etpConst['spm']['xpak_entries']['keywords'],
                'critical': False,
            },
        }
        return data

    def _extract_pkg_metadata_content(self, content_file, package_path):

        pkg_content = {}

        if os.path.isfile(content_file):
            f = open(content_file,"r")
            content = f.readlines()
            f.close()
            outcontent = set()
            for line in content:
                line = line.strip().split()
                try:
                    datatype = line[0]
                    datafile = line[1:]
                    if datatype == 'obj':
                        datafile = datafile[:-2]
                        datafile = ' '.join(datafile)
                    elif datatype == 'dir':
                        datafile = ' '.join(datafile)
                    elif datatype == 'sym':
                        datafile = datafile[:-3]
                        datafile = ' '.join(datafile)
                    else:
                        myexc = "InvalidData: %s %s. %s." % (
                            datafile,
                            _("not supported"),
                            _("Probably Portage API has changed"),
                        )
                        raise exceptionTools.InvalidData(myexc)
                    outcontent.add((datafile,datatype))
                except:
                    pass

            _outcontent = set()
            for i in outcontent:
                i = list(i)
                datatype = i[1]
                _outcontent.add((i[0],i[1]))
            outcontent = sorted(list(_outcontent))
            for i in outcontent:
                pkg_content[i[0]] = i[1]

        else:

            # CONTENTS is not generated when a package is emerged with portage and the option -B
            # we have to unpack the tbz2 and generate content dict
            mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject"
            if os.path.isdir(mytempdir):
                shutil.rmtree(mytempdir)
            if not os.path.isdir(mytempdir):
                os.makedirs(mytempdir)

            self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True)
            for currentdir, subdirs, files in os.walk(mytempdir):
                pkg_content[currentdir[len(mytempdir):]] = "dir"
                for item in files:
                    item = currentdir+"/"+item
                    if os.path.islink(item):
                        pkg_content[item[len(mytempdir):]] = "sym"
                    else:
                        pkg_content[item[len(mytempdir):]] = "obj"

            # now remove
            shutil.rmtree(mytempdir,True)
            try:
                os.rmdir(mytempdir)
            except:
                pass

        return pkg_content

    def _extract_pkg_metadata_needed(self, needed_file):

        pkg_needed = set()
        lines = []

        try:
            f = open(needed_file,"r")
            lines = [x.strip() for x in f.readlines() if x.strip()]
            f.close()
        except IOError:
            return lines

        for line in lines:
            needed = line.split()
            if len(needed) == 2:
                ownlib = needed[0]
                ownelf = -1
                if os.access(ownlib,os.R_OK):
                    ownelf = self.entropyTools.read_elf_class(ownlib)
                for lib in needed[1].split(","):
                    #if lib.find(".so") != -1:
                    pkg_needed.add((lib,ownelf))

        return list(pkg_needed)

    def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False):

        pkg_messages = []

        if os.path.isdir(log_dir):

            elogfiles = os.listdir(log_dir)
            myelogfile = "%s:%s-%s" % (category, name, version,)
            foundfiles = [x for x in elogfiles if x.startswith(myelogfile)]
            if foundfiles:
                elogfile = foundfiles[0]
                if len(foundfiles) > 1:
                    # get the latest
                    mtimes = []
                    for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item))
                    mtimes = sorted(mtimes)
                    elogfile = mtimes[-1][1]
                messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile))
                for message in messages:
                    message = message.replace("emerge","install")
                    pkg_messages.append(message)

        elif not silent:

            mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),)
            self.updateProgress(
                red(log_dir)+mytxt,
                importance = 1,
                type = "warning",
                header = brown(" * ")
            )

        return pkg_messages

    def _extract_pkg_metadata_license_data(self, licenses_dir, license_string):

        pkg_licensedata = {}
        if licenses_dir and os.path.isdir(licenses_dir):
            licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())]
            for mylicense in licdata:
                licfile = os.path.join(licenses_dir,mylicense)
                if os.access(licfile,os.R_OK):
                    if self.entropyTools.istextfile(licfile):
                        f = open(licfile)
                        pkg_licensedata[mylicense] = f.read()
                        f.close()

        return pkg_licensedata

    def _extract_pkg_metadata_mirror_links(self, Spm, sources_list):

        # =mirror://openoffice|link1|link2|link3
        pkg_links = []
        for i in sources_list:
            if i.startswith("mirror://"):
                # parse what mirror I need
                mirrorURI = i.split("/")[2]
                mirrorlist = Spm.get_third_party_mirrors(mirrorURI)
                pkg_links.append([mirrorURI,mirrorlist])
                # mirrorURI = openoffice and mirrorlist = [link1, link2, link3]

        return pkg_links

    def _extract_pkg_metadata_ebuild_entropy_tag(self, ebuild):
        search_tag = etpConst['spm']['ebuild_pkg_tag_var']
        ebuild_tag = ''
        f = open(ebuild,"r")
        tags = [x.strip() for x in f.readlines() if x.strip() and x.strip().startswith(search_tag)]
        f.close()
        if not tags: return ebuild_tag
        tag = tags[-1]
        tag = tag.split("=")[-1].strip('"').strip("'").strip()
        return tag

    # This function extracts all the info from a .tbz2 file and returns them
    def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False):

        data = {}
        info_package = bold(os.path.basename(package))+": "

        if not silent:
            self.updateProgress(
                red(info_package+_("Extracting package metadata")+" ..."),
                importance = 0,
                type = "info",
                header = brown(" * "),
                back = True
            )

        filepath = package
        tbz2File = package
        package = package.split(etpConst['packagesext'])[0]
        package = self.entropyTools.remove_entropy_revision(package)
        package = self.entropyTools.remove_tag(package)
        # remove category
        if package.find(":") != -1:
            package = ':'.join(package.split(":")[1:])

        package = package.split("-")
        pkgname = ""
        pkglen = len(package)
        if package[pkglen-1].startswith("r"):
            pkgver = package[pkglen-2]+"-"+package[pkglen-1]
            pkglen -= 2
        else:
            pkgver = package[-1]
            pkglen -= 1
        for i in range(pkglen):
            if i == pkglen-1:
                pkgname += package[i]
            else:
                pkgname += package[i]+"-"
        pkgname = pkgname.split("/")[-1]

        # Fill Package name and version
        data['name'] = pkgname
        data['version'] = pkgver
        data['digest'] = self.entropyTools.md5sum(tbz2File)
        data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File))
        data['size'] = str(self.entropyTools.get_file_size(tbz2File))

        tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/"
        if not os.path.isdir(tbz2TmpDir):
            if os.path.lexists(tbz2TmpDir):
                os.remove(tbz2TmpDir)
            os.makedirs(tbz2TmpDir)
        self.entropyTools.extractXpak(tbz2File,tbz2TmpDir)

        data['injected'] = False
        if inject: data['injected'] = True
        data['branch'] = etpBranch

        portage_entries = self._extract_pkg_metadata_generate_extraction_dict()
        for item in portage_entries:
            value = ''
            try:
                f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r")
                value = f.readline().strip()
                f.close()
            except IOError:
                if portage_entries[item]['critical']:
                    raise
            data[item] = value

        # setup vars
        data['eclasses'] = data['eclasses'].split()
        try:
            data['counter'] = int(data['counter'])
        except ValueError:
            data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database
        data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()]
        if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords
        needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed'])
        data['needed'] = self._extract_pkg_metadata_needed(needed_file)
        content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents'])
        data['content'] = self._extract_pkg_metadata_content(content_file, filepath)
        data['disksize'] = self.entropyTools.sum_file_sizes(data['content'])

        # [][][] Kernel dependent packages hook [][][]
        data['versiontag'] = ''
        kernelstuff = False
        kernelstuff_kernel = False
        for item in data['content']:
            if item.startswith("/lib/modules/"):
                kernelstuff = True
                # get the version of the modules
                kmodver = item.split("/lib/modules/")[1]
                kmodver = kmodver.split("/")[0]

                lp = kmodver.split("-")[-1]
                if lp.startswith("r"):
                    kname = kmodver.split("-")[-2]
                    kver = kmodver.split("-")[0]+"-"+kmodver.split("-")[-1]
                else:
                    kname = kmodver.split("-")[-1]
                    kver = kmodver.split("-")[0]
                break
        # validate the results above
        if kernelstuff:
            matchatom = "linux-%s-%s" % (kname,kver,)
            if (matchatom == data['name']+"-"+data['version']):
                kernelstuff_kernel = True

            data['versiontag'] = kmodver
            if not kernelstuff_kernel:
                data['slot'] = kmodver # if you change this behaviour,
                                       # you must change "reagent update"
                                       # and "equo database gentoosync" consequentially

        file_ext = etpConst['spm']['ebuild_file_extension']
        ebuilds_in_path = [x for x in os.listdir(tbz2TmpDir) if x.endswith(".%s" % (file_ext,))]
        if not data['versiontag'] and ebuilds_in_path:
            # has the user specified a custom package tag inside the ebuild
            ebuild_path = os.path.join(tbz2TmpDir,ebuilds_in_path[0])
            data['versiontag'] = self._extract_pkg_metadata_ebuild_entropy_tag(ebuild_path)


        data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/"
        data['download'] += self.entropyTools.create_package_filename(data['category'], data['name'], data['version'], data['versiontag'])


        data['trigger'] = ""
        if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']):
            f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb")
            data['trigger'] = f.read()
            f.close()

        Spm = self.Spm()

        portage_metadata = Spm.calculate_dependencies(
            data['iuse'], data['use'], data['license'], data['depend'],
            data['rdepend'], data['pdepend'], data['provide'], data['sources']
        )

        data['provide'] = portage_metadata['PROVIDE'].split()
        data['license'] = portage_metadata['LICENSE']
        data['useflags'] = []
        for x in data['use'].split():
            if x.startswith("+"):
                x = x[1:]
            elif x.startswith("-"):
                x = x[1:]
            if (x in portage_metadata['USE']) or (x in portage_metadata['USE_MASK']):
                data['useflags'].append(x)
            else:
                data['useflags'].append("-"+x)
        data['sources'] = portage_metadata['SRC_URI'].split()
        data['dependencies'] = {}
        for x in portage_metadata['RDEPEND'].split():
            if x.startswith("!") or (x in ("(","||",")","")):
                continue
            data['dependencies'][x] = 0
        for x in portage_metadata['PDEPEND'].split():
            if x.startswith("!") or (x in ("(","||",")","")):
                continue
            data['dependencies'][x] = etpConst['spm']['pdepend_id']
        data['conflicts'] = [x[1:] for x in portage_metadata['RDEPEND'].split()+portage_metadata['PDEPEND'].split() if x.startswith("!") and not x in ("(","||",")","")]

        if (kernelstuff) and (not kernelstuff_kernel):
            # add kname to the dependency
            data['dependencies']["=sys-kernel/linux-"+kname+"-"+kver] = 0

        # Conflicting tagged packages support
        key = data['category']+"/"+data['name']
        confl_data = self.SystemSettings['conflicting_tagged_packages'].get(key)
        if confl_data != None:
            for conflict in confl_data: data['conflicts'].append(conflict)

        # Get License text if possible
        licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses')
        data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license'])
        data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources'])

        # write only if it's a systempackage
        data['systempackage'] = False
        system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()]
        if data['category']+"/"+data['name'] in system_packages:
            data['systempackage'] = True

        # write only if it's a systempackage
        protect, mask = Spm.get_config_protect_and_mask()
        data['config_protect'] = protect
        data['config_protect_mask'] = mask

        log_dir = etpConst['logdir']+"/elog"
        if not os.path.isdir(log_dir): os.makedirs(log_dir)
        data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent)
        data['etpapi'] = etpConst['etpapi']

        # removing temporary directory
        shutil.rmtree(tbz2TmpDir,True)
        if os.path.isdir(tbz2TmpDir):
            try: os.remove(tbz2TmpDir)
            except OSError: pass

        if not silent:
            self.updateProgress(
                red(info_package+_("Package extraction complete")), importance = 0,
                type = "info", header = brown(" * "), back = True
            )
        return data

    '''
        Source Package Manager Interface :: end
    '''

    def Triggers(self, *args, **kwargs):
        conn = TriggerInterface(self, *args, **kwargs)
        return conn

    def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):
        conn = RepoInterface(EquoInstance = self, reponames = reponames, forceUpdate = forceUpdate, noEquoCheck = noEquoCheck, fetchSecurity = fetchSecurity)
        return conn

    def FileUpdatesInterfaceLoader(self):
        conn = FileUpdatesInterface(EquoInstance = self)
        return conn


'''
    Real package actions (install/remove) interface
'''
class PackageInterface:

    def __init__(self, EquoInstance):

        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid Equo instance or subclass is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
        self.Entropy = EquoInstance
        self.infoDict = {}
        self.prepared = False
        self.matched_atom = ()
        self.valid_actions = ("source","fetch","remove","remove_conflict","install")
        self.action = None
        self.fetch_abort_function = None
        self.xterm_title = ''

    def kill(self):
        self.infoDict.clear()
        self.matched_atom = ()
        self.valid_actions = ()
        self.action = None
        self.prepared = False
        self.fetch_abort_function = None

    def error_on_prepared(self):
        if self.prepared:
            mytxt = _("Already prepared")
            raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,))

    def error_on_not_prepared(self):
        if not self.prepared:
            mytxt = _("Not yet prepared")
            raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,))

    def check_action_validity(self, action):
        if action not in self.valid_actions:
            mytxt = _("Action must be in")
            raise exceptionTools.InvalidData("InvalidData: %s %s" % (mytxt,self.valid_actions,))

    def match_checksum(self):
        self.error_on_not_prepared()
        dlcount = 0
        match = False
        while dlcount <= 5:
            self.Entropy.updateProgress(
                blue(_("Checking package checksum...")),
                importance = 0,
                type = "info",
                header = red("   ## "),
                back = True
            )
            dlcheck = self.Entropy.check_needed_package_download(self.infoDict['download'], checksum = self.infoDict['checksum'])
            if dlcheck == 0:
                self.Entropy.updateProgress(
                    blue(_("Package checksum matches.")),
                    importance = 0,
                    type = "info",
                    header = red("   ## ")
                )
                self.infoDict['verified'] = True
                match = True
                break # file downloaded successfully
            else:
                dlcount += 1
                self.Entropy.updateProgress(
                    blue(_("Package checksum does not match. Redownloading... attempt #%s") % (dlcount,)),
                    importance = 0,
                    type = "info",
                    header = red("   ## "),
                    back = True
                )
                fetch = self.Entropy.fetch_file_on_mirrors(
                            self.infoDict['repository'],
                            self.Entropy.get_branch_from_download_relative_uri(self.infoDict['download']),
                            self.infoDict['download'],
                            self.infoDict['checksum'],
                            fetch_abort_function = self.fetch_abort_function
                        )
                if fetch != 0:
                    self.Entropy.updateProgress(
                        blue(_("Cannot properly fetch package! Quitting.")),
                        importance = 0,
                        type = "info",
                        header = red("   ## ")
                    )
                    return fetch
                self.infoDict['verified'] = True
                match = True
                break
        if (not match):
            mytxt = _("Cannot properly fetch package or checksum does not match. Try download latest repositories.")
            self.Entropy.updateProgress(
                blue(mytxt),
                importance = 0,
                type = "info",
                header = red("   ## ")
            )
            return 1
        return 0

    '''
    @description: unpack the given package file into the unpack dir
    @input infoDict: dictionary containing package information
    @output: 0 = all fine, >0 = error!
    '''
    def __unpack_package(self):

        self.error_on_not_prepared()

        if not self.infoDict['merge_from']:
            self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Unpacking package: "+str(self.infoDict['atom']))
        else:
            self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Merging package: "+str(self.infoDict['atom']))

        if os.path.isdir(self.infoDict['unpackdir']):
            shutil.rmtree(self.infoDict['unpackdir'].encode('raw_unicode_escape'))
        elif os.path.isfile(self.infoDict['unpackdir']):
            os.remove(self.infoDict['unpackdir'].encode('raw_unicode_escape'))
        os.makedirs(self.infoDict['imagedir'])

        if not os.path.isfile(self.infoDict['pkgpath']) and not self.infoDict['merge_from']:
            if os.path.isdir(self.infoDict['pkgpath']):
                shutil.rmtree(self.infoDict['pkgpath'])
            if os.path.islink(self.infoDict['pkgpath']):
                os.remove(self.infoDict['pkgpath'])
            self.infoDict['verified'] = False
            rc = self.fetch_step()
            if rc != 0: return rc

        if not self.infoDict['merge_from']:
            unpack_tries = 3
            while 1:
                unpack_tries -= 1
                try:
                    rc = self.Entropy.entropyTools.spawnFunction(
                        self.Entropy.entropyTools.uncompressTarBz2,
                        self.infoDict['pkgpath'],
                        self.infoDict['imagedir'],
                        catchEmpty = True
                    )
                except EOFError:
                    self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"EOFError on "+self.infoDict['pkgpath'])
                    rc = 1
                except (UnicodeEncodeError,UnicodeDecodeError,):
                    # this will make devs to actually catch the right exception and prepare a fix
                    self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Raising Unicode Error for "+self.infoDict['pkgpath'])
                    rc = self.Entropy.entropyTools.uncompressTarBz2(
                        self.infoDict['pkgpath'],self.infoDict['imagedir'],
                        catchEmpty = True
                    )
                if rc == 0:
                    break
                if unpack_tries <= 0:
                    return rc
                # otherwise, try to download it again
                self.infoDict['verified'] = False
                f_rc = self.fetch_step()
                if f_rc != 0: return f_rc
        else:
            pid = os.fork()
            if pid > 0:
                os.waitpid(pid, 0)
            else:
                self.__fill_image_dir(self.infoDict['merge_from'],self.infoDict['imagedir'])
                os._exit(0)

        # unpack xpak ?
        if etpConst['gentoo-compat']:
            if os.path.isdir(self.infoDict['xpakpath']):
                shutil.rmtree(self.infoDict['xpakpath'])
            try:
                os.rmdir(self.infoDict['xpakpath'])
            except OSError:
                pass

            # create data dir where we'll unpack the xpak
            os.makedirs(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'],0755)
            #os.mkdir(self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath'])
            xpakPath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakfilename']

            if not self.infoDict['merge_from']:
                if (self.infoDict['smartpackage']):
                    # we need to get the .xpak from database
                    xdbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository'])
                    xpakdata = xdbconn.retrieveXpakMetadata(self.infoDict['idpackage'])
                    if xpakdata:
                        # save into a file
                        f = open(xpakPath,"wb")
                        f.write(xpakdata)
                        f.flush()
                        f.close()
                        self.infoDict['xpakstatus'] = self.Entropy.entropyTools.unpackXpak(
                            xpakPath,
                            self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
                        )
                    else:
                        self.infoDict['xpakstatus'] = None
                    del xpakdata
                else:
                    self.infoDict['xpakstatus'] = self.Entropy.entropyTools.extractXpak(
                        self.infoDict['pkgpath'],
                        self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
                    )
            else:
                # link xpakdir to self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
                tolink_dir = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
                if os.path.isdir(tolink_dir):
                    shutil.rmtree(tolink_dir,True)
                # now link
                os.symlink(self.infoDict['xpakdir'],tolink_dir)

            # create fake portage ${D} linking it to imagedir
            portage_db_fakedir = os.path.join(
                self.infoDict['unpackdir'],
                "portage/"+self.infoDict['category'] + "/" + self.infoDict['name'] + "-" + self.infoDict['version']
            )

            os.makedirs(portage_db_fakedir,0755)
            # now link it to self.infoDict['imagedir']
            os.symlink(self.infoDict['imagedir'],os.path.join(portage_db_fakedir,"image"))

        return 0

    def __remove_package(self):

        # clear on-disk cache
        self.__clear_cache()

        self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: %s" % (self.infoDict['removeatom'],))

        # remove from database
        if self.infoDict['removeidpackage'] != -1:
            mytxt = "%s: " % (_("Removing from Entropy"),)
            self.Entropy.updateProgress(
                blue(mytxt) + red(self.infoDict['removeatom']),
                importance = 1,
                type = "info",
                header = red("   ## ")
            )
            self.__remove_package_from_database()

        # Handle gentoo database
        if (etpConst['gentoo-compat']):
            gentooAtom = self.Entropy.entropyTools.remove_tag(self.infoDict['removeatom'])
            self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing from Portage: "+str(gentooAtom))
            self.__remove_package_from_gentoo_database(gentooAtom)
            del gentooAtom

        self.__remove_content_from_system()
        return 0

    def __remove_content_from_system(self):

        # load CONFIG_PROTECT and its mask
        # client database at this point has been surely opened,
        # so our dicts are already filled
        protect = etpConst['dbconfigprotect']
        mask = etpConst['dbconfigprotectmask']
        sys_root = etpConst['systemroot']
        col_protect = etpConst['collisionprotect']

        # remove files from system
        directories = set()
        for item in self.infoDict['removecontent']:
            # collision check
            if col_protect > 0:

                if self.Entropy.clientDbconn.isFileAvailable(item) and os.path.isfile(sys_root+item):
                    # in this way we filter out directories
                    mytxt = red(_("Collision found during removal of")) + " " + sys_root+item + " - "
                    mytxt += red(_("cannot overwrite"))
                    self.Entropy.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "warning",
                        header = red("   ## ")
                    )
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "Collision found during remove of "+sys_root+item+" - cannot overwrite"
                    )
                    continue

            protected = False
            if (not self.infoDict['removeconfig']) and (not self.infoDict['diffremoval']):
                protected_item_test = sys_root+item
                protected, x, do_continue = self._handle_config_protect(protect, mask, None, protected_item_test, do_allocation_check = False)
                if do_continue: protected = True

            if protected:
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_VERBOSE,
                    "[remove] Protecting config file: "+sys_root+item
                )
                mytxt = "[%s] %s: %s" % (
                    red(_("remove")),
                    brown(_("Protecting config file")),
                    sys_root+item,
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )
            else:
                try:
                    os.lstat(sys_root+item)
                except OSError:
                    continue # skip file, does not exist
                except UnicodeEncodeError:
                    mytxt = brown(_("This package contains a badly encoded file !!!"))
                    self.Entropy.updateProgress(
                        red("QA: ")+mytxt,
                        importance = 1,
                        type = "warning",
                        header = darkred("   ## ")
                    )
                    continue # file has a really bad encoding

                if os.path.isdir(sys_root+item) and os.path.islink(sys_root+item):
                    # S_ISDIR returns False for directory symlinks, so using os.path.isdir
                    # valid directory symlink
                    directories.add((sys_root+item,"link"))
                elif os.path.isdir(sys_root+item):
                    # plain directory
                    directories.add((sys_root+item,"dir"))
                else: # files, symlinks or not
                    # just a file or symlink or broken directory symlink (remove now)
                    try:
                        os.remove(sys_root+item)
                        # add its parent directory
                        dirfile = os.path.dirname(sys_root+item)
                        if os.path.isdir(dirfile) and os.path.islink(dirfile):
                            directories.add((dirfile,"link"))
                        elif os.path.isdir(dirfile):
                            directories.add((dirfile,"dir"))
                    except OSError:
                        pass

        # now handle directories
        directories = sorted(list(directories), reverse = True)
        while 1:
            taint = False
            for directory, dirtype in directories:
                mydir = "%s%s" % (sys_root,directory,)
                if dirtype == "link":
                    try:
                        mylist = os.listdir(mydir)
                        if not mylist:
                            try:
                                os.remove(mydir)
                                taint = True
                            except OSError:
                                pass
                    except OSError:
                        pass
                elif dirtype == "dir":
                    try:
                        mylist = os.listdir(mydir)
                        if not mylist:
                            try:
                                os.rmdir(mydir)
                                taint = True
                            except OSError:
                                pass
                    except OSError:
                        pass

            if not taint:
                break
        del directories


    '''
    @description: remove package entry from Gentoo database
    @input gentoo package atom (cat/name+ver):
    @output: 0 = all fine, <0 = error!
    '''
    def __remove_package_from_gentoo_database(self, atom):

        # handle gentoo-compat
        try:
            Spm = self.Entropy.Spm()
        except:
            return -1 # no Spm support ??

        portDbDir = Spm.get_vdb_path()
        removePath = portDbDir+atom
        key = self.Entropy.entropyTools.dep_getkey(atom)
        others_installed = Spm.search_keys(key)
        slot = self.infoDict['slot']
        tag = self.infoDict['versiontag']
        if (tag == slot) and tag: slot = "0"
        if os.path.isdir(removePath):
            shutil.rmtree(removePath,True)
        elif others_installed:
            for myatom in others_installed:
                myslot = Spm.get_installed_package_slot(myatom)
                if myslot != slot:
                    continue
                shutil.rmtree(portDbDir+myatom,True)

        if not others_installed:
            world_file = Spm.get_world_file()
            world_file_tmp = world_file+".entropy.tmp"
            if os.access(world_file,os.W_OK) and os.path.isfile(world_file):
                new = open(world_file_tmp,"w")
                old = open(world_file,"r")
                line = old.readline()
                while line:
                    if line.find(key) != -1:
                        line = old.readline()
                        continue
                    if line.find(key+":"+slot) != -1:
                        line = old.readline()
                        continue
                    new.write(line)
                    line = old.readline()
                new.flush()
                new.close()
                old.close()
                shutil.move(world_file_tmp,world_file)

        return 0

    '''
    @description: function that runs at the end of the package installation process, just removes data left by other steps
    @output: 0 = all fine, >0 = error!
    '''
    def __cleanup_package(self, unpack_dir):
        # remove unpack dir
        shutil.rmtree(unpack_dir,True)
        try:
            os.rmdir(unpack_dir)
        except OSError:
            pass
        return 0

    def __remove_package_from_database(self):
        self.error_on_not_prepared()
        self.Entropy.clientDbconn.removePackage(self.infoDict['removeidpackage'])
        return 0

    def __clear_cache(self):
        self.Entropy.clear_dump_cache(etpCache['advisories'])
        self.Entropy.clear_dump_cache(etpCache['filter_satisfied_deps'])
        self.Entropy.clear_dump_cache(etpCache['depends_tree'])
        self.Entropy.clear_dump_cache(etpCache['check_package_update'])
        self.Entropy.clear_dump_cache(etpCache['dep_tree'])
        self.Entropy.clear_dump_cache(etpCache['dbMatch']+etpConst['clientdbid']+"/")
        self.Entropy.clear_dump_cache(etpCache['dbSearch']+etpConst['clientdbid']+"/")

        self.__update_available_cache()
        try:
            self.__update_world_cache()
        except:
            self.Entropy.clear_dump_cache(etpCache['world_update'])

    def __update_world_cache(self):
        if self.Entropy.xcache and (self.action in ("install","remove",)):
            wc_dir = os.path.dirname(os.path.join(etpConst['dumpstoragedir'],etpCache['world_update']))
            wc_filename = os.path.basename(etpCache['world_update'])
            wc_cache_files = [os.path.join(wc_dir,x) for x in os.listdir(wc_dir) if x.startswith(wc_filename)]
            for cache_file in wc_cache_files:

                try:
                    data = self.Entropy.dumpTools.loadobj(cache_file, completePath = True)
                    (update, remove, fine) = data['r']
                    empty_deps = data['empty_deps']
                except:
                    self.Entropy.clear_dump_cache(etpCache['world_update'])
                    return

                if empty_deps:
                    continue

                if self.action == "install":
                    if self.matched_atom in update:
                        update.remove(self.matched_atom)
                        self.Entropy.dumpTools.dumpobj(
                            cache_file,
                            {'r':(update, remove, fine),'empty_deps': empty_deps},
                            completePath = True
                        )
                else:
                    key, slot = self.Entropy.clientDbconn.retrieveKeySlot(self.infoDict['removeidpackage'])
                    matches = self.Entropy.atomMatch(key, matchSlot = slot, multiMatch = True, multiRepo = True)
                    if matches[1] != 0:
                        # hell why! better to rip all off
                        self.Entropy.clear_dump_cache(etpCache['world_update'])
                        return
                    taint = False
                    for match in matches[0]:
                        if match in update:
                            taint = True
                            update.remove(match)
                        if match in remove:
                            taint = True
                            remove.remove(match)
                    if taint:
                        self.Entropy.dumpTools.dumpobj(
                            cache_file,
                            {'r':(update, remove, fine),'empty_deps': empty_deps},
                            completePath = True
                        )

        elif (not self.Entropy.xcache) or (self.action in ("install",)):
            self.Entropy.clear_dump_cache(etpCache['world_update'])

    def __update_available_cache(self):

        # update world available cache
        if self.Entropy.xcache and (self.action in ("remove","install")):

            disk_cache = self.Entropy.Cacher.pop(etpCache['world_available'])
            if disk_cache != None:
                c_hash = self.Entropy.get_available_packages_chash(etpConst['branch'])
                try:
                    if disk_cache['chash'] == c_hash:

                        # remove and old install
                        if self.infoDict['removeidpackage'] != -1:
                            taint = False
                            key = self.Entropy.entropyTools.dep_getkey(self.infoDict['removeatom'])
                            slot = self.infoDict['slot']
                            matches = self.Entropy.atomMatch(key, matchSlot = slot, multiRepo = True, multiMatch = True)
                            if matches[1] == 0:
                                for mymatch in matches[0]:
                                    if mymatch not in disk_cache['available']:
                                        disk_cache['available'].append(mymatch)
                                        taint = True
                            if taint:
                                mydata = {}
                                mylist = []
                                for myidpackage,myrepo in disk_cache['available']:
                                    mydbc = self.Entropy.openRepositoryDatabase(myrepo)
                                    mydata[mydbc.retrieveAtom(myidpackage)] = (myidpackage,myrepo)
                                mykeys = sorted(mydata.keys())
                                for mykey in mykeys:
                                    mylist.append(mydata[mykey])
                                disk_cache['available'] = mylist

                        # install, doing here because matches[0] could contain self.matched_atoms
                        if self.matched_atom in disk_cache['available']:
                            disk_cache['available'].remove(self.matched_atom)

                        self.Entropy.Cacher.push(etpCache['world_available'],disk_cache.copy())

                except KeyError:
                    self.Entropy.Cacher.push(etpCache['world_available'],{})

        elif not self.Entropy.xcache:
            self.Entropy.clear_dump_cache(etpCache['world_available'])


    '''
    @description: install unpacked files, update database and also update gentoo db if requested
    @output: 0 = all fine, >0 = error!
    '''
    def __install_package(self):

        # clear on-disk cache
        self.__clear_cache()

        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "Installing package: %s" % (self.infoDict['atom'],)
        )

        # copy files over - install
        rc = self.__move_image_to_system()
        if rc != 0:
            return rc

        # inject into database
        mytxt = "%s: %s" % (blue(_("Updating database")),red(self.infoDict['atom']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        newidpackage = self._install_package_into_database()

        # remove old files and gentoo stuff
        if (self.infoDict['removeidpackage'] != -1):
            # doing a diff removal
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "Remove old package: %s" % (self.infoDict['removeatom'],)
            )
            self.infoDict['removeidpackage'] = -1 # disabling database removal

            if etpConst['gentoo-compat']:
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "Removing Entropy and Gentoo database entry for %s" % (self.infoDict['removeatom'],)
                )
            else:
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "Removing Entropy (only) database entry for %s" % (self.infoDict['removeatom'],)
                )

            self.Entropy.updateProgress(
                                    blue(_("Cleaning old package files...")),
                                    importance = 1,
                                    type = "info",
                                    header = red("   ## ")
                                )
            self.__remove_package()

        rc = 0
        if etpConst['gentoo-compat']:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "Installing new Gentoo database entry: %s" % (self.infoDict['atom'],)
            )
            rc = self._install_package_into_gentoo_database(newidpackage)

        return rc

    '''
    @description: inject the database information into the Gentoo database
    @output: 0 = all fine, !=0 = error!
    '''
    def _install_package_into_gentoo_database(self, newidpackage):

        # handle gentoo-compat
        try:
            Spm = self.Entropy.Spm()
        except:
            return -1 # no Portage support
        portDbDir = Spm.get_vdb_path()
        if os.path.isdir(portDbDir):

            # extract xpak from unpackDir+etpConst['packagecontentdir']+"/"+package
            key = self.infoDict['category']+"/"+self.infoDict['name']
            atomsfound = set()
            dbdirs = os.listdir(portDbDir)
            if self.infoDict['category'] in dbdirs:
                catdirs = os.listdir(portDbDir+"/"+self.infoDict['category'])
                dirsfound = set([self.infoDict['category']+"/"+x for x in catdirs if key == self.Entropy.entropyTools.dep_getkey(self.infoDict['category']+"/"+x)])
                atomsfound.update(dirsfound)

            ### REMOVE
            # parse slot and match and remove
            if atomsfound:
                pkgToRemove = ''
                for atom in atomsfound:
                    atomslot = Spm.get_installed_package_slot(atom)
                    # get slot from gentoo db
                    if atomslot == self.infoDict['slot']:
                        pkgToRemove = atom
                        break
                if (pkgToRemove):
                    removePath = portDbDir+pkgToRemove
                    shutil.rmtree(removePath,True)
                    try:
                        os.rmdir(removePath)
                    except OSError:
                        pass
            del atomsfound

            # we now install it
            if ((self.infoDict['xpakstatus'] != None) and \
                    os.path.isdir( self.infoDict['xpakpath'] + "/" + etpConst['entropyxpakdatarelativepath'])) or \
                    self.infoDict['merge_from']:

                if self.infoDict['merge_from']:
                    copypath = self.infoDict['xpakdir']
                    if not os.path.isdir(copypath):
                        return 0
                else:
                    copypath = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']

                if not os.path.isdir(portDbDir+self.infoDict['category']):
                    os.makedirs(portDbDir+self.infoDict['category'],0755)
                destination = portDbDir+self.infoDict['category']+"/"+self.infoDict['name']+"-"+self.infoDict['version']
                if os.path.isdir(destination):
                    shutil.rmtree(destination)

                try:
                    shutil.copytree(copypath,destination)
                except (IOError,), e:
                    mytxt = "%s: %s: %s: %s" % (red(_("QA")),brown(_("Cannot update Portage database to destination")),purple(destination),e,)
                    self.Entropy.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "warning",
                        header = darkred("   ## ")
                    )

                # test if /var/cache/edb/counter is fine
                if os.path.isfile(etpConst['edbcounter']):
                    try:
                        f = open(etpConst['edbcounter'],"r")
                        counter = int(f.readline().strip())
                        f.close()
                    except:
                        # need file recreation, parse gentoo tree
                        counter = Spm.refill_counter()
                else:
                    counter = Spm.refill_counter()

                # write new counter to file
                if os.path.isdir(destination):
                    counter += 1
                    f = open(destination+"/"+etpConst['spm']['xpak_entries']['counter'],"w")
                    f.write(str(counter))
                    f.flush()
                    f.close()
                    f = open(etpConst['edbcounter'],"w")
                    f.write(str(counter))
                    f.flush()
                    f.close()
                    # update counter inside clientDatabase
                    self.Entropy.clientDbconn.insertCounter(newidpackage,counter)
                else:
                    mytxt = brown(_("Cannot update Portage counter, destination %s does not exist.") % (destination,))
                    self.Entropy.updateProgress(
                        red("QA: ")+mytxt,
                        importance = 1,
                        type = "warning",
                        header = darkred("   ## ")
                    )

            # add to Portage world
            # key: key
            # slot: self.infoDict['slot']
            myslot = self.infoDict['slot']
            if (self.infoDict['versiontag'] == self.infoDict['slot']) and self.infoDict['versiontag']:
                # usually kernel packages
                myslot = "0"
            keyslot = key+":"+myslot
            world_file = Spm.get_world_file()
            world_atoms = set()

            if os.access(world_file,os.R_OK) and os.path.isfile(world_file):
                f = open(world_file,"r")
                world_atoms = set([x.strip() for x in f.readlines() if x.strip()])
                f.close()
            else:
                mytxt = brown(_("Cannot update Portage world file, destination %s does not exist.") % (world_file,))
                self.Entropy.updateProgress(
                    red("QA: ")+mytxt,
                    importance = 1,
                    type = "warning",
                    header = darkred("   ## ")
                )
                return 0

            try:
                if keyslot not in world_atoms and \
                    os.access(os.path.dirname(world_file),os.W_OK) and \
                    self.Entropy.entropyTools.istextfile(world_file):
                        world_atoms.discard(key)
                        world_atoms.add(keyslot)
                        world_atoms = sorted(list(world_atoms))
                        world_file_tmp = world_file+".entropy_inst"
                        f = open(world_file_tmp,"w")
                        for item in world_atoms:
                            f.write(item+"\n")
                        f.flush()
                        f.close()
                        shutil.move(world_file_tmp,world_file)
            except (UnicodeDecodeError,UnicodeEncodeError), e:
                self.Entropy.entropyTools.printTraceback(f = self.Entropy.clientLog)
                mytxt = brown(_("Cannot update Portage world file, destination %s is corrupted.") % (world_file,))
                self.Entropy.updateProgress(
                    red("QA: ")+mytxt+": "+unicode(e),
                    importance = 1,
                    type = "warning",
                    header = darkred("   ## ")
                )

        return 0

    '''
    @description: injects package info into the installed packages database
    @output: 0 = all fine, >0 = error!
    '''
    def _install_package_into_database(self):

        # fetch info
        dbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository'])
        data = dbconn.getPackageData(self.infoDict['idpackage'], content_insert_formatted = True)
        # open client db
        # always set data['injected'] to False
        # installed packages database SHOULD never have more than one package for scope (key+slot)
        data['injected'] = False
        data['counter'] = -1 # gentoo counter will be set in self._install_package_into_gentoo_database()

        idpackage, rev, x = self.Entropy.clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision'], formattedContent = True)

        # update datecreation
        ctime = self.Entropy.entropyTools.getCurrentUnixTime()
        self.Entropy.clientDbconn.setDateCreation(idpackage, str(ctime))

        # add idpk to the installedtable
        self.Entropy.clientDbconn.removePackageFromInstalledTable(idpackage)
        self.Entropy.clientDbconn.addPackageToInstalledTable(idpackage,self.infoDict['repository'])

        # clear depends table, this will make clientdb dependstable to be regenerated during the next request (retrieveDepends)
        self.Entropy.clientDbconn.clearDependsTable()
        return idpackage

    def __fill_image_dir(self, mergeFrom, imageDir):

        dbconn = self.Entropy.openRepositoryDatabase(self.infoDict['repository'])
        package_content = dbconn.retrieveContent(self.infoDict['idpackage'], extended = True, formatted = True)
        contents = sorted([x for x in package_content])

        # collect files
        for path in contents:
            # convert back to filesystem str
            encoded_path = path
            path = os.path.join(mergeFrom,encoded_path[1:])
            topath = os.path.join(imageDir,encoded_path[1:])
            path = path.encode('raw_unicode_escape')
            topath = topath.encode('raw_unicode_escape')

            try:
                exist = os.lstat(path)
            except OSError:
                continue # skip file
            ftype = package_content[encoded_path]
            if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype
            if 'dir' == ftype and \
                not stat.S_ISDIR(exist.st_mode) and \
                os.path.isdir(path): # workaround for directory symlink issues
                path = os.path.realpath(path)

            copystat = False
            # if our directory is a symlink instead, then copy the symlink
            if os.path.islink(path):
                tolink = os.readlink(path)
                if os.path.islink(topath):
                    os.remove(topath)
                os.symlink(tolink,topath)
            elif os.path.isdir(path):
                if not os.path.isdir(topath):
                    os.makedirs(topath)
                    copystat = True
            elif os.path.isfile(path):
                if os.path.isfile(topath):
                    os.remove(topath) # should never happen
                shutil.copy2(path,topath)
                copystat = True

            if copystat:
                user = os.stat(path)[stat.ST_UID]
                group = os.stat(path)[stat.ST_GID]
                os.chown(topath,user,group)
                shutil.copystat(path,topath)


    def __move_image_to_system(self):

        # load CONFIG_PROTECT and its mask
        protect = etpRepositories[self.infoDict['repository']]['configprotect']
        mask = etpRepositories[self.infoDict['repository']]['configprotectmask']
        sys_root = etpConst['systemroot']
        col_protect = etpConst['collisionprotect']
        items_installed = set()

        # setup imageDir properly
        imageDir = self.infoDict['imagedir']

        # merge data into system
        for currentdir,subdirs,files in os.walk(imageDir.encode('utf-8')):
            # create subdirs
            for subdir in subdirs:

                imagepathDir = "%s/%s" % (currentdir,subdir,)
                rootdir = "%s%s" % (sys_root,imagepathDir[len(imageDir):],)

                # handle broken symlinks
                if os.path.islink(rootdir) and not os.path.exists(rootdir):# broken symlink
                    os.remove(rootdir)

                # if our directory is a file on the live system
                elif os.path.isfile(rootdir): # really weird...!
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "WARNING!!! %s is a file when it should be a directory !! Removing in 20 seconds..." % (rootdir,)
                    )
                    mytxt = darkred(_("%s is a file when should be a directory !! Removing in 20 seconds...") % (rootdir,))
                    self.Entropy.updateProgress(
                        red("QA: ")+mytxt,
                        importance = 1,
                        type = "warning",
                        header = red(" !!! ")
                    )
                    self.Entropy.entropyTools.ebeep(10)
                    time.sleep(20)
                    os.remove(rootdir)

                # if our directory is a symlink instead, then copy the symlink
                if os.path.islink(imagepathDir) and not os.path.isdir(rootdir):
                    # for security we skip live items that are dirs
                    tolink = os.readlink(imagepathDir)
                    if os.path.islink(rootdir):
                        os.remove(rootdir)
                    os.symlink(tolink,rootdir)
                elif (not os.path.isdir(rootdir)) and (not os.access(rootdir,os.R_OK)):
                    try:
                        # we should really force a simple mkdir first of all
                        os.mkdir(rootdir)
                    except OSError:
                        os.makedirs(rootdir)

                if not os.path.islink(rootdir) and os.access(rootdir,os.W_OK):
                    # symlink don't need permissions, also until os.walk ends they might be broken
                    # XXX also, added os.access() check because there might be directories/files unwriteable
                    # what to do otherwise?
                    user = os.stat(imagepathDir)[stat.ST_UID]
                    group = os.stat(imagepathDir)[stat.ST_GID]
                    os.chown(rootdir,user,group)
                    shutil.copystat(imagepathDir,rootdir)

                items_installed.add(os.path.join(os.path.realpath(os.path.dirname(rootdir)),os.path.basename(rootdir)))

            for item in files:

                fromfile = "%s/%s" % (currentdir,item,)
                tofile = "%s%s" % (sys_root,fromfile[len(imageDir):],)

                if col_protect > 1:
                    todbfile = fromfile[len(imageDir):]
                    myrc = self._handle_install_collision_protect(tofile, todbfile)
                    if not myrc:
                        continue

                protected, tofile, do_continue = self._handle_config_protect(protect, mask, fromfile, tofile)
                if do_continue:
                    continue

                try:

                    if os.path.realpath(fromfile) == os.path.realpath(tofile) and os.path.islink(tofile):
                        # there is a serious issue here, better removing tofile, happened to someone:
                        try: # try to cope...
                            os.remove(tofile)
                        except OSError:
                            pass

                    # if our file is a dir on the live system
                    if os.path.isdir(tofile) and not os.path.islink(tofile): # really weird...!
                        self.Entropy.clientLog.log(
                            ETP_LOGPRI_INFO,
                            ETP_LOGLEVEL_NORMAL,
                            "WARNING!!! %s is a directory when it should be a file !! Removing in 20 seconds..." % (tofile,)
                        )
                        mytxt = _("%s is a directory when it should be a file !! Removing in 20 seconds...") % (tofile,)
                        self.Entropy.updateProgress(
                            red("QA: ")+darkred(mytxt),
                            importance = 1,
                            type = "warning",
                            header = red(" !!! ")
                        )
                        self.Entropy.entropyTools.ebeep(10)
                        time.sleep(20)
                        try:
                            shutil.rmtree(tofile, True)
                            os.rmdir(tofile)
                        except:
                            pass
                        try: # if it was a link
                            os.remove(tofile)
                        except OSError:
                            pass

                    # this also handles symlinks
                    # XXX
                    # XXX moving file using the raw format like portage does
                    # XXX
                    try:
                        shutil.move(fromfile,tofile)
                    except shutil.Error, e:
                        self.Entropy.clientLog.log(
                            ETP_LOGPRI_INFO,
                            ETP_LOGLEVEL_NORMAL,
                            "WARNING!!! Error during file move to system: %s" % (e,)
                        )
                        mytxt = "%s: %s, %s" % (_("File move error"),e,_("please report"),)
                        self.Entropy.updateProgress(
                            red("QA: ")+darkred(mytxt),
                            importance = 1,
                            type = "warning",
                            header = red(" !!! ")
                        )

                except IOError, e:
                    if e.errno == 2:
                        # better to pass away, sometimes gentoo packages are fucked up and contain broken things
                        pass
                    else:
                        rc = subprocess.call(["mv",fromfile,tofile])
                        if rc != 0: return 4

                items_installed.add(os.path.join(os.path.realpath(os.path.dirname(tofile)),os.path.basename(tofile)))
                if protected:
                    # add to disk cache
                    self.Entropy.FileUpdates.add_to_cache(tofile, quiet = True)

        # this is useful to avoid the removal of installed files by __remove_package just because
        # there's a difference in the directory path, perhaps, which is not handled correctly by
        # EntropyDatabaseInterface.contentDiff for obvious reasons (think about stuff in /usr/lib and /usr/lib64,
        # where the latter is just a symlink to the former)
        if self.infoDict.get('removecontent'):
            my_remove_content = set([x for x in self.infoDict['removecontent'] if os.path.join(os.path.realpath(os.path.dirname("%s%s" % (sys_root,x,))),os.path.basename(x)) in items_installed])
            self.infoDict['removecontent'] -= my_remove_content

        return 0

    def _handle_config_protect(self, protect, mask, fromfile, tofile, do_allocation_check = True):

        protected = False
        tofile_before_protect = tofile
        do_continue = False

        try:
            encoded_protect = [x.encode('raw_unicode_escape') for x in protect]
            if tofile in encoded_protect:
                protected = True
            elif os.path.dirname(tofile) in encoded_protect:
                protected = True
            else:
                tofile_testdir = os.path.dirname(tofile)
                old_tofile_testdir = None
                while tofile_testdir != old_tofile_testdir:
                    if tofile_testdir in encoded_protect:
                        protected = True
                        break
                    old_tofile_testdir = tofile_testdir
                    tofile_testdir = os.path.dirname(tofile_testdir)

            if protected: # check if perhaps, file is masked, so unprotected
                newmask = [x.encode('raw_unicode_escape') for x in mask]
                if tofile in newmask:
                    protected = False
                elif os.path.dirname(tofile) in newmask:
                    protected = False

            if not os.path.lexists(tofile):
                protected = False # file doesn't exist

            # check if it's a text file
            if (protected) and os.path.isfile(tofile):
                protected = self.Entropy.entropyTools.istextfile(tofile)
            else:
                protected = False # it's not a file

            # request new tofile then
            if protected:
                if tofile not in etpConst['configprotectskip']:
                    prot_status = True
                    if do_allocation_check:
                        tofile, prot_status = self.Entropy.entropyTools.allocateMaskedFile(tofile, fromfile)
                    if not prot_status:
                        protected = False
                    else:
                        oldtofile = tofile
                        if oldtofile.find("._cfg") != -1:
                            oldtofile = os.path.dirname(oldtofile)+"/"+os.path.basename(oldtofile)[10:]
                        self.Entropy.clientLog.log(
                            ETP_LOGPRI_INFO,
                            ETP_LOGLEVEL_NORMAL,
                            "Protecting config file: %s" % (oldtofile,)
                        )
                        mytxt = red("%s: %s") % (_("Protecting config file"),oldtofile,)
                        self.Entropy.updateProgress(
                            mytxt,
                            importance = 1,
                            type = "warning",
                            header = darkred("   ## ")
                        )
                else:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "Skipping config file installation/removal, as stated in equo.conf: %s" % (tofile,)
                    )
                    mytxt = "%s: %s" % (_("Skipping file installation/removal"),tofile,)
                    self.Entropy.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "warning",
                        header = darkred("   ## ")
                    )
                    do_continue = True

        except Exception, e:
            self.Entropy.entropyTools.printTraceback()
            protected = False # safely revert to false
            tofile = tofile_before_protect
            mytxt = darkred("%s: %s") % (_("Cannot check CONFIG PROTECTION. Error"),e,)
            self.Entropy.updateProgress(
                red("QA: ")+mytxt,
                importance = 1,
                type = "warning",
                header = darkred("   ## ")
            )

        return protected, tofile, do_continue


    def _handle_install_collision_protect(self, tofile, todbfile):
        avail = self.Entropy.clientDbconn.isFileAvailable(todbfile, get_id = True)
        if (self.infoDict['removeidpackage'] not in avail) and avail:
            mytxt = darkred(_("Collision found during install for"))
            mytxt += " %s - %s" % (blue(tofile),darkred(_("cannot overwrite")),)
            self.Entropy.updateProgress(
                red("QA: ")+mytxt,
                importance = 1,
                type = "warning",
                header = darkred("   ## ")
            )
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "WARNING!!! Collision found during install for %s - cannot overwrite" % (tofile,)
            )
            return False
        return True

    def sources_fetch_step(self):
        self.error_on_not_prepared()
        down_data = self.infoDict['download']
        rc = 0
        for key in sorted(down_data.keys()):
            # first fine wins
            for url in down_data[key]:
                dest_file = os.path.join(self.infoDict['unpackdir'],os.path.basename(url))
                rc = self._fetch_source(url, dest_file)
                if not rc: break
            if rc: break

        return rc

    def _fetch_source(self, url, dest_file):
        try:
            mytxt = "%s: %s" % (blue(_("Downloading")),brown(url),)
            # now fetch the new one
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red("   ## ")
            )

            rc, data_transfer, resumed = self.Entropy.fetch_file(
                url,
                None,
                None,
                False,
                fetch_file_abort_function = self.fetch_abort_function,
                filepath = dest_file
            )
            if rc == 0:
                mytxt = blue("%s: ") % (_("Successfully downloaded from"),)
                mytxt += red(self.Entropy.entropyTools.spliturl(url)[1])
                mytxt += " %s %s/%s" % (_("at"),self.Entropy.entropyTools.bytesIntoHuman(data_transfer),_("second"),)
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = red("   ## ")
                )
                self.Entropy.updateProgress(
                    "%s: %s" % (blue(_("Local path")),brown(dest_file),),
                    importance = 1,
                    type = "info",
                    header = red("      # ")
                )

                return 0
            else:
                error_message = blue("%s: %s") % (
                    _("Error downloading from"),
                    red(self.Entropy.entropyTools.spliturl(url)[1]),
                )
                # something bad happened
                if rc == -1:
                    error_message += " - %s." % (_("file not available on this mirror"),)
                elif rc == -3:
                    error_message += " - not found."
                elif rc == -4:
                    error_message += " - %s." % (_("discarded download"),)
                else:
                    error_message += " - %s: %s" % (_("unknown reason"),rc,)
                self.Entropy.updateProgress(
                                    error_message,
                                    importance = 1,
                                    type = "warning",
                                    header = red("   ## ")
                                )
                if rc == -4: # user discarded fetch
                    return 1
        except KeyboardInterrupt:
            return 1

    def fetch_step(self):
        self.error_on_not_prepared()
        mytxt = "%s: %s" % (blue(_("Downloading archive")),red(os.path.basename(self.infoDict['download'])),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   ## ")
        )

        rc = self.Entropy.fetch_file_on_mirrors(
            self.infoDict['repository'],
            self.Entropy.get_branch_from_download_relative_uri(self.infoDict['download']),
            self.infoDict['download'],
            self.infoDict['checksum'],
            self.infoDict['verified'],
            fetch_abort_function = self.fetch_abort_function
        )
        if rc != 0:
            mytxt = "%s. %s: %s" % (
                red(_("Package cannot be fetched. Try to update repositories and retry")),
                blue(_("Error")),
                rc,
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = darkred("   ## ")
            )
            return rc
        return 0

    def fetch_not_available_step(self):
        self.Entropy.updateProgress(
            blue(_("Fetch for the chosen package is not available, unknown error.")),
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        return 0

    def vanished_step(self):
        self.Entropy.updateProgress(
            blue(_("Installed package in queue vanished, skipping.")),
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        return 0

    def checksum_step(self):
        self.error_on_not_prepared()
        rc = self.match_checksum()
        return rc

    def unpack_step(self):
        self.error_on_not_prepared()

        if not self.infoDict['merge_from']:
            mytxt = "%s: %s" % (blue(_("Unpacking package")),red(os.path.basename(self.infoDict['download'])),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red("   ## ")
        )
        else:
            mytxt = "%s: %s" % (blue(_("Merging package")),red(os.path.basename(self.infoDict['atom'])),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red("   ## ")
            )
        rc = self.__unpack_package()
        if rc != 0:
            if rc == 512:
                errormsg = "%s. %s. %s: 512" % (
                    red(_("You are running out of disk space")),
                    red(_("I bet, you're probably Michele")),
                    blue(_("Error")),
                )
            else:
                errormsg = "%s. %s. %s: %s" % (
                    red(_("An error occured while trying to unpack the package")),
                    red(_("Check if your system is healthy")),
                    blue(_("Error")),
                    rc,
                )
            self.Entropy.updateProgress(
                errormsg,
                importance = 1,
                type = "error",
                header = red("   ## ")
            )
        return rc

    def install_step(self):
        self.error_on_not_prepared()
        mytxt = "%s: %s" % (blue(_("Installing package")),red(self.infoDict['atom']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        rc = self.__install_package()
        if rc != 0:
            mytxt = "%s. %s. %s: %s" % (
                red(_("An error occured while trying to install the package")),
                red(_("Check if your system is healthy")),
                blue(_("Error")),
                rc,
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = red("   ## ")
            )
        return rc

    def remove_step(self):
        self.error_on_not_prepared()
        mytxt = "%s: %s" % (blue(_("Removing data")),red(self.infoDict['removeatom']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        rc = self.__remove_package()
        if rc != 0:
            mytxt = "%s. %s. %s: %s" % (
                red(_("An error occured while trying to remove the package")),
                red(_("Check if you have enough disk space on your hard disk")),
                blue(_("Error")),
                rc,
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = red("   ## ")
            )
        return rc

    def cleanup_step(self):
        self.error_on_not_prepared()
        mytxt = "%s: %s" % (blue(_("Cleaning")),red(self.infoDict['atom']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   ## ")
        )
        task = self.Entropy.entropyTools.parallelTask(self.__cleanup_package, self.infoDict['unpackdir'])
        task.parallel_wait()
        task.start()
        # we don't care if cleanupPackage fails since it's not critical
        return 0

    def logmessages_step(self):
        for msg in self.infoDict['messages']:
            self.Entropy.clientLog.write(">>>  "+msg)
        return 0

    def messages_step(self):
        self.error_on_not_prepared()
        # get messages
        if self.infoDict['messages']:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "Message from %s:" % (self.infoDict['atom'],)
            )
            mytxt = "%s:" % (darkgreen(_("Compilation messages")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "warning",
                header = brown("   ## ")
            )
        for msg in self.infoDict['messages']:
            self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,msg)
            self.Entropy.updateProgress(
                msg,
                importance = 0,
                type = "warning",
                header = brown("   ## ")
            )
        if self.infoDict['messages']:
            self.Entropy.clientLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"End message.")

    def postinstall_step(self):
        self.error_on_not_prepared()
        pkgdata = self.infoDict['triggers'].get('install')
        if pkgdata:
            Trigger = self.Entropy.Triggers('postinstall',pkgdata, self.action)
            Trigger.prepare()
            Trigger.run()
            Trigger.kill()
            del Trigger
        del pkgdata
        return 0

    def preinstall_step(self):
        self.error_on_not_prepared()
        pkgdata = self.infoDict['triggers'].get('install')
        if pkgdata:

            Trigger = self.Entropy.Triggers('preinstall',pkgdata, self.action)
            Trigger.prepare()
            if (self.infoDict.get("diffremoval") != None): # diffremoval is true only when the remove action is triggered by installPackages()
                if self.infoDict['diffremoval']:
                    remdata = self.infoDict['triggers'].get('remove')
                    if remdata:
                        rTrigger = self.Entropy.Triggers('preremove',remdata, self.action)
                        rTrigger.prepare()
                        Trigger.triggers = [x for x in Trigger.triggers if x not in rTrigger.triggers]
                        rTrigger.kill()
                        del rTrigger
                    del remdata
            Trigger.run()
            Trigger.kill()
            del Trigger

        del pkgdata
        return 0

    def preremove_step(self):
        self.error_on_not_prepared()
        remdata = self.infoDict['triggers'].get('remove')
        if remdata:
            Trigger = self.Entropy.Triggers('preremove',remdata, self.action)
            Trigger.prepare()
            Trigger.run()
            Trigger.kill()
            del Trigger
        del remdata
        return 0

    def postremove_step(self):
        self.error_on_not_prepared()
        remdata = self.infoDict['triggers'].get('remove')
        if remdata:

            Trigger = self.Entropy.Triggers('postremove',remdata, self.action)
            Trigger.prepare()
            if self.infoDict['diffremoval'] and (self.infoDict.get("atom") != None):
                # diffremoval is true only when the remove action is triggered by installPackages()
                pkgdata = self.infoDict['triggers'].get('install')
                if pkgdata:
                    iTrigger = self.Entropy.Triggers('postinstall',pkgdata, self.action)
                    iTrigger.prepare()
                    Trigger.triggers = [x for x in Trigger.triggers if x not in iTrigger.triggers]
                    iTrigger.kill()
                    del iTrigger
                del pkgdata
            Trigger.run()
            Trigger.kill()
            del Trigger

        del remdata
        return 0

    def removeconflict_step(self):

        for idpackage in self.infoDict['conflicts']:
            if not self.Entropy.clientDbconn.isIDPackageAvailable(idpackage):
                continue
            Package = self.Entropy.Package()
            Package.prepare((idpackage,),"remove_conflict", self.infoDict['remove_metaopts'])
            rc = Package.run(xterm_header = self.xterm_title)
            Package.kill()
            if rc != 0:
                return rc

        return 0

    def run_stepper(self, xterm_header):
        if xterm_header == None:
            xterm_header = ""

        if self.infoDict.has_key('remove_installed_vanished'):
            self.xterm_title += ' Installed package vanished'
            self.Entropy.setTitle(self.xterm_title)
            rc = self.vanished_step()
            return rc

        if self.infoDict.has_key('fetch_not_available'):
            self.xterm_title += ' Fetch not available'
            self.Entropy.setTitle(self.xterm_title)
            rc = self.fetch_not_available_step()
            return rc

        def do_fetch():
            mytxt = _("Fetching")
            self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['download']),)
            self.Entropy.setTitle(self.xterm_title)
            return self.fetch_step()

        def do_sources_fetch():
            mytxt = _("Fetching sources")
            self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['atom']),)
            self.Entropy.setTitle(self.xterm_title)
            return self.sources_fetch_step()

        def do_checksum():
            mytxt = _("Verifying")
            self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['download']),)
            self.Entropy.setTitle(self.xterm_title)
            return self.checksum_step()

        def do_unpack():
            if not self.infoDict['merge_from']:
                mytxt = _("Unpacking")
                self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['download']),)
            else:
                mytxt = _("Merging")
                self.xterm_title += ' %s: %s' % (mytxt,os.path.basename(self.infoDict['atom']),)
            self.Entropy.setTitle(self.xterm_title)
            return self.unpack_step()

        def do_remove_conflicts():
            return self.removeconflict_step()

        def do_install():
            mytxt = _("Installing")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['atom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.install_step()

        def do_remove():
            mytxt = _("Removing")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['removeatom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.remove_step()

        def do_showmessages():
            return self.messages_step()

        def do_logmessages():
            return self.logmessages_step()

        def do_cleanup():
            mytxt = _("Cleaning")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['atom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.cleanup_step()

        def do_postinstall():
            mytxt = _("Postinstall")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['atom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.postinstall_step()

        def do_preinstall():
            mytxt = _("Preinstall")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['atom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.preinstall_step()

        def do_preremove():
            mytxt = _("Preremove")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['removeatom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.preremove_step()

        def do_postremove():
            mytxt = _("Postremove")
            self.xterm_title += ' %s: %s' % (mytxt,self.infoDict['removeatom'],)
            self.Entropy.setTitle(self.xterm_title)
            return self.postremove_step()

        steps_data = {
            "fetch": do_fetch,
            "sources_fetch": do_sources_fetch,
            "checksum": do_checksum,
            "unpack": do_unpack,
            "remove_conflicts": do_remove_conflicts,
            "install": do_install,
            "remove": do_remove,
            "showmessages": do_showmessages,
            "logmessages": do_logmessages,
            "cleanup": do_cleanup,
            "postinstall": do_postinstall,
            "preinstall": do_preinstall,
            "postremove": do_postremove,
            "preremove": do_preremove,
        }

        rc = 0
        for step in self.infoDict['steps']:
            self.xterm_title = xterm_header
            rc = steps_data.get(step)()
            if rc != 0: break
        return rc


    '''
        @description: execute the requested steps
        @input xterm_header: purely optional
    '''
    def run(self, xterm_header = None):
        self.error_on_not_prepared()

        gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock)
        if gave_up:
            return 20

        locked = self.Entropy.application_lock_check()
        if locked:
            self.Entropy._resources_run_remove_lock()
            return 21

        # lock
        self.Entropy._resources_run_create_lock()

        try:
            rc = self.run_stepper(xterm_header)
        except:
            self.Entropy._resources_run_remove_lock()
            raise

        # remove lock
        self.Entropy._resources_run_remove_lock()

        if rc != 0:
            self.Entropy.updateProgress(
                blue(_("An error occured. Action aborted.")),
                importance = 2,
                type = "error",
                header = darkred("   ## ")
            )
        return rc

    '''
       Install/Removal process preparation function
       - will generate all the metadata needed to run the action steps, creating infoDict automatically
       @input matched_atom(tuple): is what is returned by EquoInstance.atomMatch:
            (idpackage,repoid):
            (2000,u'sabayonlinux.org')
            NOTE: in case of remove action, matched_atom must be:
            (idpackage,)
        @input action(string): is an action to take, which must be one in self.valid_actions
    '''
    def prepare(self, matched_atom, action, metaopts = {}):

        self.error_on_prepared()
        self.check_action_validity(action)

        self.action = action
        self.matched_atom = matched_atom
        self.metaopts = metaopts
        # generate metadata dictionary
        self.generate_metadata()

    def generate_metadata(self):
        self.error_on_prepared()
        self.check_action_validity(self.action)

        if self.action == "fetch":
            self.__generate_fetch_metadata()
        elif self.action in ("remove","remove_conflict"):
            self.__generate_remove_metadata()
        elif self.action == "install":
            self.__generate_install_metadata()
        elif self.action == "source":
            self.__generate_fetch_metadata(sources = True)
        self.prepared = True

    def __generate_remove_metadata(self):

        self.infoDict.clear()
        idpackage = self.matched_atom[0]

        if not self.Entropy.clientDbconn.isIDPackageAvailable(idpackage):
            self.infoDict['remove_installed_vanished'] = True
            return 0

        self.infoDict['triggers'] = {}
        self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(idpackage)
        self.infoDict['slot'] = self.Entropy.clientDbconn.retrieveSlot(idpackage)
        self.infoDict['versiontag'] = self.Entropy.clientDbconn.retrieveVersionTag(idpackage)
        self.infoDict['removeidpackage'] = idpackage
        self.infoDict['diffremoval'] = False
        removeConfig = False
        if self.metaopts.has_key('removeconfig'):
            removeConfig = self.metaopts.get('removeconfig')
        self.infoDict['removeconfig'] = removeConfig
        self.infoDict['removecontent'] = self.Entropy.clientDbconn.retrieveContent(idpackage)
        self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getTriggerInfo(idpackage)
        self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent']
        self.infoDict['steps'] = []
        self.infoDict['steps'].append("preremove")
        self.infoDict['steps'].append("remove")
        self.infoDict['steps'].append("postremove")

        return 0

    def __generate_install_metadata(self):
        self.infoDict.clear()

        idpackage, repository = self.matched_atom
        self.infoDict['idpackage'] = idpackage
        self.infoDict['repository'] = repository

        # fetch abort function
        if self.metaopts.has_key('fetch_abort_function'):
            self.fetch_abort_function = self.metaopts.pop('fetch_abort_function')

        # get package atom
        dbconn = self.Entropy.openRepositoryDatabase(repository)
        self.infoDict['triggers'] = {}
        self.infoDict['atom'] = dbconn.retrieveAtom(idpackage)
        self.infoDict['slot'] = dbconn.retrieveSlot(idpackage)
        self.infoDict['version'], self.infoDict['versiontag'], self.infoDict['revision'] = dbconn.getVersioningData(idpackage)
        self.infoDict['category'] = dbconn.retrieveCategory(idpackage)
        self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage)
        self.infoDict['name'] = dbconn.retrieveName(idpackage)
        self.infoDict['messages'] = dbconn.retrieveMessages(idpackage)
        self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage)
        self.infoDict['accept_license'] = dbconn.retrieveLicensedataKeys(idpackage)
        self.infoDict['conflicts'] = self.Entropy.get_match_conflicts(self.matched_atom)

        # fill action queue
        self.infoDict['removeidpackage'] = -1
        removeConfig = False
        if self.metaopts.has_key('removeconfig'):
            removeConfig = self.metaopts.get('removeconfig')

        self.infoDict['remove_metaopts'] = {
            'removeconfig': True,
        }
        if self.metaopts.has_key('remove_metaopts'):
            self.infoDict['remove_metaopts'] = self.metaopts.get('remove_metaopts')

        self.infoDict['merge_from'] = None
        mf = self.metaopts.get('merge_from')
        if mf != None:
            self.infoDict['merge_from'] = unicode(mf)
        self.infoDict['removeconfig'] = removeConfig

        self.infoDict['removeidpackage'] = self.Entropy.retrieveInstalledIdPackage(
                                                self.Entropy.entropyTools.dep_getkey(self.infoDict['atom']),
                                                self.infoDict['slot']
                                            )

        if self.infoDict['removeidpackage'] != -1:
            avail = self.Entropy.clientDbconn.isIDPackageAvailable(self.infoDict['removeidpackage'])
            if avail:
                self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage'])
            else:
                self.infoDict['removeidpackage'] = -1

        # smartpackage ?
        self.infoDict['smartpackage'] = False
        # set unpack dir and image dir
        if self.infoDict['repository'].endswith(etpConst['packagesext']):
            # do arch check
            compiled_arch = dbconn.retrieveDownloadURL(idpackage)
            if compiled_arch.find("/"+etpSys['arch']+"/") == -1:
                self.infoDict.clear()
                self.prepared = False
                return -1
            self.infoDict['smartpackage'] = etpRepositories[self.infoDict['repository']]['smartpackage']
            self.infoDict['pkgpath'] = etpRepositories[self.infoDict['repository']]['pkgpath']
        else:
            self.infoDict['pkgpath'] = etpConst['entropyworkdir']+"/"+self.infoDict['download']
        self.infoDict['unpackdir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']
        self.infoDict['imagedir'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyimagerelativepath']

        # gentoo xpak data
        if etpConst['gentoo-compat']:
            self.infoDict['xpakpath'] = etpConst['entropyunpackdir']+"/"+self.infoDict['download']+"/"+etpConst['entropyxpakrelativepath']
            if not self.infoDict['merge_from']:
                self.infoDict['xpakstatus'] = None
                self.infoDict['xpakdir'] = self.infoDict['xpakpath']+"/"+etpConst['entropyxpakdatarelativepath']
            else:
                self.infoDict['xpakstatus'] = True
                portdbdir = 'var/db/pkg' # XXX hard coded ?
                portdbdir = os.path.join(self.infoDict['merge_from'],portdbdir)
                portdbdir = os.path.join(portdbdir,self.infoDict['category'])
                portdbdir = os.path.join(portdbdir,self.infoDict['name']+"-"+self.infoDict['version'])
                self.infoDict['xpakdir'] = portdbdir

        # compare both versions and if they match, disable removeidpackage
        if self.infoDict['removeidpackage'] != -1:
            installedVer, installedTag, installedRev = self.Entropy.clientDbconn.getVersioningData(self.infoDict['removeidpackage'])
            pkgcmp = self.Entropy.entropyTools.entropyCompareVersions(
                (self.infoDict['version'], self.infoDict['versiontag'], self.infoDict['revision'],),
                (installedVer, installedTag, installedRev,)
            )
            if pkgcmp == 0:
                self.infoDict['removeidpackage'] = -1
            else:
                # differential remove list
                self.infoDict['diffremoval'] = True
                self.infoDict['removeatom'] = self.Entropy.clientDbconn.retrieveAtom(self.infoDict['removeidpackage'])
                self.infoDict['removecontent'] = self.Entropy.clientDbconn.contentDiff(
                        self.infoDict['removeidpackage'],
                        dbconn,
                        idpackage
                )
                self.infoDict['triggers']['remove'] = self.Entropy.clientDbconn.getTriggerInfo(
                        self.infoDict['removeidpackage']
                )
                self.infoDict['triggers']['remove']['removecontent'] = self.infoDict['removecontent']

        # set steps
        self.infoDict['steps'] = []
        if self.infoDict['conflicts']:
            self.infoDict['steps'].append("remove_conflicts")
        # install
        if (self.infoDict['removeidpackage'] != -1):
            self.infoDict['steps'].append("preremove")
        self.infoDict['steps'].append("unpack")
        self.infoDict['steps'].append("preinstall")
        self.infoDict['steps'].append("install")
        if (self.infoDict['removeidpackage'] != -1):
            self.infoDict['steps'].append("postremove")
        self.infoDict['steps'].append("postinstall")
        if not etpConst['gentoo-compat']: # otherwise gentoo triggers will show that
            self.infoDict['steps'].append("showmessages")
        else:
            self.infoDict['steps'].append("logmessages")
        self.infoDict['steps'].append("cleanup")

        self.infoDict['triggers']['install'] = dbconn.getTriggerInfo(idpackage)
        self.infoDict['triggers']['install']['accept_license'] = self.infoDict['accept_license']
        self.infoDict['triggers']['install']['unpackdir'] = self.infoDict['unpackdir']
        if etpConst['gentoo-compat']:
            #self.infoDict['triggers']['install']['xpakpath'] = self.infoDict['xpakpath']
            self.infoDict['triggers']['install']['xpakdir'] = self.infoDict['xpakdir']

        return 0

    def __generate_fetch_metadata(self, sources = False):
        self.infoDict.clear()

        idpackage, repository = self.matched_atom
        dochecksum = True

        # fetch abort function
        if self.metaopts.has_key('fetch_abort_function'):
            self.fetch_abort_function = self.metaopts.pop('fetch_abort_function')

        if self.metaopts.has_key('dochecksum'):
            dochecksum = self.metaopts.get('dochecksum')
        self.infoDict['repository'] = repository
        self.infoDict['idpackage'] = idpackage
        dbconn = self.Entropy.openRepositoryDatabase(repository)
        self.infoDict['atom'] = dbconn.retrieveAtom(idpackage)
        if sources:
            self.infoDict['download'] = dbconn.retrieveSources(idpackage, extended = True)
        else:
            self.infoDict['checksum'] = dbconn.retrieveDigest(idpackage)
            self.infoDict['download'] = dbconn.retrieveDownloadURL(idpackage)

        if not self.infoDict['download']:
            self.infoDict['fetch_not_available'] = True
            return 0

        self.infoDict['verified'] = False
        self.infoDict['steps'] = []
        if not repository.endswith(etpConst['packagesext']) and not sources:
            if self.Entropy.check_needed_package_download(self.infoDict['download'], None) < 0:
                self.infoDict['steps'].append("fetch")
            if dochecksum:
                self.infoDict['steps'].append("checksum")
        elif sources:
            self.infoDict['steps'].append("sources_fetch")

        if sources:
            # create sources destination directory
            unpack_dir = etpConst['entropyunpackdir']+"/sources/"+self.infoDict['atom']
            self.infoDict['unpackdir'] = unpack_dir
            if os.path.lexists(unpack_dir):
                if os.path.isfile(unpack_dir):
                    os.remove(unpack_dir)
                elif os.path.isdir(unpack_dir):
                    shutil.rmtree(unpack_dir,True)
            if not os.path.lexists(unpack_dir):
                os.makedirs(unpack_dir,0775)
            const_setup_perms(unpack_dir,etpConst['entropygid'])

        else:
            # if file exists, first checksum then fetch
            if os.path.isfile(os.path.join(etpConst['entropyworkdir'],self.infoDict['download'])):
                # check size first
                repo_size = dbconn.retrieveSize(idpackage)
                f = open(os.path.join(etpConst['entropyworkdir'],self.infoDict['download']),"r")
                f.seek(0,2)
                disk_size = f.tell()
                f.close()
                if repo_size == disk_size:
                    self.infoDict['steps'].reverse()
        return 0

class FileUpdatesInterface:

    def __init__(self, EquoInstance):
        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid Equo instance or subclass is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
        self.Entropy = EquoInstance
        self.scandata = None

    def merge_file(self, key):
        self.scanfs(dcache = True)
        self.do_backup(key)
        if os.access(etpConst['systemroot'] + self.scandata[key]['source'], os.R_OK):
            shutil.move(
                etpConst['systemroot'] + self.scandata[key]['source'],
                etpConst['systemroot'] + self.scandata[key]['destination']
            )
        self.remove_from_cache(key)

    def remove_file(self, key):
        self.scanfs(dcache = True)
        try:
            os.remove(etpConst['systemroot'] + self.scandata[key]['source'])
        except OSError:
            pass
        self.remove_from_cache(key)

    def do_backup(self, key):
        self.scanfs(dcache = True)
        if etpConst['filesbackup'] and os.path.isfile(etpConst['systemroot']+self.scandata[key]['destination']):
            bcount = 0
            backupfile = etpConst['systemroot'] + \
                os.path.dirname(self.scandata[key]['destination']) + \
                "/._equo_backup." + unicode(bcount) + "_" + \
                os.path.basename(self.scandata[key]['destination'])
            while os.path.lexists(backupfile):
                bcount += 1
                backupfile = etpConst['systemroot'] + \
                os.path.dirname(self.scandata[key]['destination']) + \
                "/._equo_backup." + unicode(bcount) + "_" + \
                os.path.basename(self.scandata[key]['destination'])
            try:
                shutil.copy2(etpConst['systemroot'] + self.scandata[key]['destination'],backupfile)
            except IOError:
                pass

    '''
    @description: scan for files that need to be merged
    @output: dictionary using filename as key
    '''
    def scanfs(self, dcache = True, quiet = False):

        if dcache:

            if self.scandata != None:
                return self.scandata

            # can we load cache?
            try:
                z = self.load_cache()
                if z != None:
                    self.scandata = z
                    return self.scandata
            except:
                pass

        # open client database to fill etpConst['dbconfigprotect']
        scandata = {}
        counter = 0
        name_cache = set()
        for path in etpConst['dbconfigprotect']:
            # it's a file?
            scanfile = False
            if os.path.isfile(path):
                # find inside basename
                path = os.path.dirname(path)
                scanfile = True

            for currentdir,subdirs,files in os.walk(path):
                for item in files:

                    if scanfile:
                        if path != item:
                            continue

                    filepath = os.path.join(currentdir,item)
                    if item.startswith("._cfg"):

                        # further check then
                        number = item[5:9]
                        try:
                            int(number)
                        except ValueError:
                            continue # not a valid etc-update file
                        if item[9] != "_": # no valid format provided
                            continue

                        if filepath in name_cache:
                            continue # skip, already done
                        name_cache.add(filepath)

                        mydict = self.generate_dict(filepath)
                        if mydict['automerge']:
                            if not quiet:
                                mytxt = _("Automerging file")
                                self.Entropy.updateProgress(
                                    darkred("%s: %s") % (
                                        mytxt,
                                        darkgreen(etpConst['systemroot'] + mydict['source']),
                                    ),
                                    importance = 0,
                                    type = "info"
                                )
                            if os.path.isfile(etpConst['systemroot']+mydict['source']):
                                try:
                                    shutil.move(    etpConst['systemroot']+mydict['source'],
                                                    etpConst['systemroot']+mydict['destination']
                                    )
                                except IOError, e:
                                    if not quiet:
                                        mytxt = "%s :: %s: %s. %s: %s" % (
                                            red(_("I/O Error")),
                                            red(_("Cannot automerge file")),
                                            brown(etpConst['systemroot'] + mydict['source']),
                                            blue("Error"),
                                            e,
                                        )
                                        self.Entropy.updateProgress(
                                            mytxt,
                                            importance = 1,
                                            type = "warning"
                                        )
                            continue
                        else:
                            counter += 1
                            scandata[counter] = mydict.copy()

                        if not quiet:
                            try:
                                self.Entropy.updateProgress(
                                    "("+blue(str(counter))+") " + red(" file: ") + \
                                    os.path.dirname(filepath) + "/" + os.path.basename(filepath)[10:],
                                    importance = 1,
                                    type = "info"
                                )
                            except:
                                pass # possible encoding issues
        # store data
        self.Entropy.Cacher.push(etpCache['configfiles'],scandata.copy())
        self.scandata = scandata.copy()
        return scandata

    def load_cache(self):
        sd = self.Entropy.Cacher.pop(etpCache['configfiles'])
        if not isinstance(sd,dict):
            raise exceptionTools.CacheCorruptionError("CacheCorruptionError")
        # quick test if data is reliable
        try:
            name_cache = set()

            for x in sd:
                mysource = sd[x]['source']
                # filter dupies
                if mysource in name_cache:
                    sd.pop(x)
                    continue
                if not os.path.isfile(etpConst['systemroot']+mysource):
                    raise exceptionTools.CacheCorruptionError("CacheCorruptionError")
                name_cache.add(mysource)

            return sd
        except (KeyError,EOFError,IOError,):
            raise exceptionTools.CacheCorruptionError("CacheCorruptionError")

    '''
    @description: prints information about config files that should be updated
    @attention: please be sure that filepath is properly formatted before using this function
    '''
    def add_to_cache(self, filepath, quiet = False):
        self.scanfs(dcache = True, quiet = quiet)
        keys = self.scandata.keys()
        try:
            for key in keys:
                if self.scandata[key]['source'] == filepath[len(etpConst['systemroot']):]:
                    del self.scandata[key]
        except:
            pass
        # get next counter
        if keys:
            keys = sorted(keys)
            index = keys[-1]
        else:
            index = 0
        index += 1
        mydata = self.generate_dict(filepath)
        self.scandata[index] = mydata.copy()
        self.Entropy.Cacher.push(etpCache['configfiles'],self.scandata.copy())

    def remove_from_cache(self, key):
        self.scanfs(dcache = True)
        try:
            del self.scandata[key]
        except:
            pass
        self.Entropy.Cacher.push(etpCache['configfiles'],self.scandata.copy())
        return self.scandata

    def generate_dict(self, filepath):

        item = os.path.basename(filepath)
        currentdir = os.path.dirname(filepath)
        tofile = item[10:]
        number = item[5:9]
        try:
            int(number)
        except:
            mytxt = _("Invalid config file number")
            raise exceptionTools.InvalidDataType("InvalidDataType: %s '0000->9999'." % (mytxt,))
        tofilepath = currentdir+"/"+tofile
        mydict = {}
        mydict['revision'] = number
        mydict['destination'] = tofilepath[len(etpConst['systemroot']):]
        mydict['source'] = filepath[len(etpConst['systemroot']):]
        mydict['automerge'] = False
        if not os.path.isfile(tofilepath):
            mydict['automerge'] = True
        if (not mydict['automerge']):
            # is it trivial?
            try:
                if not os.path.lexists(filepath): # if file does not even exist
                    return mydict
                if os.path.islink(filepath):
                    # if it's broken, skip diff and automerge
                    if not os.path.exists(filepath):
                        return mydict
                result = commands.getoutput('diff -Nua "%s" "%s" | grep "^[+-][^+-]" | grep -v \'# .Header:.*\'' % (filepath,tofilepath,))
                if not result:
                    mydict['automerge'] = True
            except:
                pass
            # another test
            if (not mydict['automerge']):
                try:
                    if not os.path.lexists(filepath): # if file does not even exist
                        return mydict
                    if os.path.islink(filepath):
                        # if it's broken, skip diff and automerge
                        if not os.path.exists(filepath):
                            return mydict
                    result = subprocess.call('diff -Bbua "%s" "%s" | egrep \'^[+-]\' | egrep -v \'^[+-][\t ]*#|^--- |^\+\+\+ \' | egrep -qv \'^[-+][\t ]*$\'' % (filepath,tofilepath,), shell = True)
                    if result == 1:
                        mydict['automerge'] = True
                except:
                    pass
        return mydict

#
# repository control class, that's it
#
class RepoInterface:

    import dumpTools
    import entropyTools
    import socket
    def __init__(self, EquoInstance, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):

        self.LockScanner = None
        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid Equo instance or subclass is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        self.supported_download_items = (
            "db","rev","ck",
            "lock","mask","system_mask","dbdump", "conflicting_tagged",
            "dbdumpck","lic_whitelist","make.conf",
            "package.mask","package.unmask","package.keywords","profile.link",
            "package.use","server.cert","ca.cert",
            "notice_board"
        )
        self.big_socket_timeout = 25
        self.Entropy = EquoInstance
        self.dbapi2 = dbapi2
        self.reponames = reponames
        self.forceUpdate = forceUpdate
        self.syncErrors = False
        self.dbupdated = False
        self.newEquo = False
        self.fetchSecurity = fetchSecurity
        self.noEquoCheck = noEquoCheck
        self.alreadyUpdated = 0
        self.notAvailable = 0
        self.valid_eapis = [1,2,3]
        self.reset_dbformat_eapi(None)
        self.current_repository_got_locked = False
        self.updated_repos = set()

        # check etpRepositories
        if not etpRepositories:
            mytxt = _("No repositories specified in %s") % (etpConst['repositoriesconf'],)
            raise exceptionTools.MissingParameter("MissingParameter: %s" % (mytxt,))

        # Test network connectivity
        conntest = self.entropyTools.get_remote_data(etpConst['conntestlink'])
        if not conntest:
            mytxt = _("Cannot connect to %s") % (etpConst['conntestlink'],)
            raise exceptionTools.OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,))

        if not self.reponames:
            self.reponames.extend(etpRepositories.keys()[:])

    def __del__(self):
        if self.LockScanner != None:
            self.LockScanner.kill()

    def get_eapi3_connection(self, repository):
        # get database url
        dburl = etpRepositories[repository]['plain_database']
        if dburl.startswith("file://"):
            return None
        try:
            dburl = dburl.split("/")[2]
        except IndexError:
            return None
        port = etpRepositories[repository]['service_port']
        try:
            eapi3_socket = SystemSocketClientInterface(
                self.Entropy, RepositorySocketClientCommands, output_header = "\t"
            )
            eapi3_socket.socket_timeout = self.big_socket_timeout
            eapi3_socket.connect(dburl, port)
            return eapi3_socket
        except (exceptionTools.ConnectionError,self.socket.error,):
            return None

    def check_eapi3_availability(self, repository):
        conn = self.get_eapi3_connection(repository)
        if conn == None: return False
        try:
            conn.disconnect()
        except (self.socket.error,AttributeError,):
            return False
        return True

    def reset_dbformat_eapi(self, repository):

        self.dbformat_eapi = 2
        if repository != None:
            eapi_avail = self.check_eapi3_availability(repository)
            if eapi_avail:
                self.dbformat_eapi = 3

        # FIXME, find a way to do that without needing sqlite3 exec.
        if not os.access("/usr/bin/sqlite3",os.X_OK) or self.entropyTools.islive():
            self.dbformat_eapi = 1
        else:
            rc = subprocess.call("/usr/bin/sqlite3 -version &> /dev/null", shell = True)
            if rc != 0: self.dbformat_eapi = 1

        eapi_env = os.getenv("FORCE_EAPI")
        if eapi_env != None:
            try:
                myeapi = int(eapi_env)
            except (ValueError,TypeError,):
                return
            if myeapi in self.valid_eapis:
                self.dbformat_eapi = myeapi


    def __validate_repository_id(self, repoid):
        if repoid not in self.reponames:
            mytxt = _("Repository is not listed in self.reponames")
            raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,))

    def __validate_compression_method(self, repo):

        self.__validate_repository_id(repo)

        cmethod = etpConst['etpdatabasecompressclasses'].get(etpRepositories[repo]['dbcformat'])
        if cmethod == None:
            mytxt = _("Wrong database compression method")
            raise exceptionTools.InvalidDataType("InvalidDataType: %s" % (mytxt,))

        return cmethod

    def __ensure_repository_path(self, repo):

        self.__validate_repository_id(repo)

        # create dir if it doesn't exist
        if not os.path.isdir(etpRepositories[repo]['dbpath']):
            os.makedirs(etpRepositories[repo]['dbpath'],0775)

        const_setup_perms(etpConst['etpdatabaseclientdir'],etpConst['entropygid'])

    def _construct_paths(self, item, repo, cmethod):

        if item not in self.supported_download_items:
            mytxt = _("Supported items: %s") % (self.supported_download_items,)
            raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,))
        if (item in ("db","dbdump", "dbdumpck",)) and (cmethod == None):
                mytxt = _("For %s, cmethod can't be None") % (item,)
                raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,))

        if item == "db":
            url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[2]]
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[2]]
        elif item == "dbdump":
            url = etpRepositories[repo]['database'] +   "/" + etpConst[cmethod[3]]
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[3]]
        elif item == "rev":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaserevisionfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaserevisionfile']
        elif item == "ck":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasehashfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasehashfile']
        elif item == "dbdumpck":
            url = etpRepositories[repo]['database'] + "/" + etpConst[cmethod[4]]
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst[cmethod[4]]
        elif item == "mask":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasemaskfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasemaskfile']
        elif item == "system_mask":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasesytemmaskfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasesytemmaskfile']
        elif item == "conflicting_tagged":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaseconflictingtaggedfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaseconflictingtaggedfile']
        elif item == "make.conf":
            myfile = os.path.basename(etpConst['spm']['global_make_conf'])
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "package.mask":
            myfile = os.path.basename(etpConst['spm']['global_package_mask'])
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "package.unmask":
            myfile = os.path.basename(etpConst['spm']['global_package_unmask'])
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "package.keywords":
            myfile = os.path.basename(etpConst['spm']['global_package_keywords'])
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "package.use":
            myfile = os.path.basename(etpConst['spm']['global_package_use'])
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "profile.link":
            myfile = etpConst['spm']['global_make_profile_link_name']
            url = etpRepositories[repo]['database'] + "/" + myfile
            filepath = etpRepositories[repo]['dbpath'] + "/" + myfile
        elif item == "lic_whitelist":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaselicwhitelistfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaselicwhitelistfile']
        elif item == "lock":
            url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabasedownloadlockfile']
            filepath = "/dev/null"
        elif item == "server.cert":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabasecacertfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabasecacertfile']
        elif item == "ca.cert":
            url = etpRepositories[repo]['database'] + "/" + etpConst['etpdatabaseservercertfile']
            filepath = etpRepositories[repo]['dbpath'] + "/" + etpConst['etpdatabaseservercertfile']
        elif item == "notice_board":
            url = etpRepositories[repo]['notice_board']
            filepath = etpRepositories[repo]['dbpath'] + "/" + os.path.basename(etpRepositories[repo]['notice_board'])

        return url, filepath

    def __remove_repository_files(self, repo, cmethod):

        dbfilenameid = cmethod[2]
        self.__validate_repository_id(repo)

        if self.dbformat_eapi == 1:
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile']):
                os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'])
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid]):
                os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[dbfilenameid])
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
                os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
        elif self.dbformat_eapi == 2:
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+cmethod[4]):
                os.remove(etpRepositories[repo]['dbpath']+"/"+cmethod[4])
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]):
                os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]])
            if os.path.isfile(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile']):
                os.remove(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'])
        else:
            mytxt = _("self.dbformat_eapi must be in (1,2)")
            raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,))

    def __unpack_downloaded_database(self, repo, cmethod):

        self.__validate_repository_id(repo)
        rc = 0
        path = None

        if self.dbformat_eapi == 1:
            myfile = etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[2]]
            try:
                path = eval("self.entropyTools."+cmethod[1])(myfile)
            except EOFError:
                rc = 1
            if os.path.isfile(myfile):
                os.remove(myfile)
        elif self.dbformat_eapi == 2:
            myfile = etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[3]]
            try:
                path = eval("self.entropyTools."+cmethod[1])(myfile)
            except EOFError:
                rc = 1
            if os.path.isfile(myfile):
                os.remove(myfile)
        else:
            mytxt = _("self.dbformat_eapi must be in (1,2)")
            raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,))

        if rc == 0:
            self.Entropy.setup_default_file_perms(path)

        return rc

    def __verify_database_checksum(self, repo, cmethod = None):

        self.__validate_repository_id(repo)

        if self.dbformat_eapi == 1:
            dbfile = etpConst['etpdatabasefile']
            try:
                f = open(etpRepositories[repo]['dbpath']+"/"+etpConst['etpdatabasehashfile'],"r")
                md5hash = f.readline().strip()
                md5hash = md5hash.split()[0]
                f.close()
            except:
                return -1
        elif self.dbformat_eapi == 2:
            dbfile = etpConst[cmethod[3]]
            try:
                f = open(etpRepositories[repo]['dbpath']+"/"+etpConst[cmethod[4]],"r")
                md5hash = f.readline().strip()
                md5hash = md5hash.split()[0]
                f.close()
            except:
                return -1
        else:
            mytxt = _("self.dbformat_eapi must be in (1,2)")
            raise exceptionTools.InvalidData('InvalidData: %s' % (mytxt,))

        rc = self.entropyTools.compareMd5(etpRepositories[repo]['dbpath']+"/"+dbfile,md5hash)
        return rc

    # @returns -1 if the file is not available
    # @returns int>0 if the revision has been retrieved
    def get_online_repository_revision(self, repo):

        self.__validate_repository_id(repo)

        url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaserevisionfile']
        status = self.entropyTools.get_remote_data(url)
        if (status):
            status = status[0].strip()
            try:
                status = int(status)
            except ValueError:
                status = -1
            return status
        else:
            return -1

    def get_online_eapi3_lock(self, repo):
        self.__validate_repository_id(repo)
        url = etpRepositories[repo]['database']+"/"+etpConst['etpdatabaseeapi3lockfile']
        data = self.entropyTools.get_remote_data(url)
        if not data:
            return False
        return True

    def is_repository_eapi3_locked(self, repo):
        self.__validate_repository_id(repo)
        return self.get_online_eapi3_lock(repo)

    def is_repository_updatable(self, repo):

        self.__validate_repository_id(repo)

        onlinestatus = self.get_online_repository_revision(repo)
        if (onlinestatus != -1):
            localstatus = self.Entropy.get_repository_revision(repo)
            if (localstatus == onlinestatus) and (not self.forceUpdate):
                return False
        return True

    def is_repository_unlocked(self, repo):

        self.__validate_repository_id(repo)

        rc = self.download_item("lock", repo, disallow_redirect = True)
        if rc: # cannot download database
            self.syncErrors = True
            return False
        return True

    def clear_repository_cache(self, repo):
        self.__validate_repository_id(repo)
        self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repo,))
        self.Entropy.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repo,))

    # this function can be reimplemented
    def download_item(self, item, repo, cmethod = None, lock_status_func = None, disallow_redirect = True):

        self.__validate_repository_id(repo)
        url, filepath = self._construct_paths(item, repo, cmethod)

        # to avoid having permissions issues
        # it's better to remove the file before,
        # otherwise new permissions won't be written
        if os.path.isfile(filepath):
            os.remove(filepath)

        fetchConn = self.Entropy.urlFetcher(
            url,
            filepath,
            resume = False,
            abort_check_func = lock_status_func,
            disallow_redirect = disallow_redirect
        )
        fetchConn.progress = self.Entropy.progress

        rc = fetchConn.download()
        del fetchConn
        if rc in ("-1","-2","-3"):
            return False
        self.Entropy.setup_default_file_perms(filepath)
        return True

    def check_downloaded_database(self, repo, cmethod):
        dbfilename = etpConst['etpdatabasefile']
        if self.dbformat_eapi == 2:
            dbfilename = etpConst[cmethod[3]]
        # verify checksum
        mytxt = "%s %s %s" % (red(_("Checking downloaded database")),darkgreen(dbfilename),red("..."))
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            back = True,
            type = "info",
            header = "\t"
        )
        db_status = self.__verify_database_checksum(repo, cmethod)
        if db_status == -1:
            mytxt = "%s. %s !" % (red(_("Cannot open digest")),red(_("Cannot verify database integrity")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )
        elif db_status:
            mytxt = "%s: %s" % (red(_("Downloaded database status")),bold(_("OK")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = "\t"
            )
        else:
            mytxt = "%s: %s" % (red(_("Downloaded database status")),darkred(_("ERROR")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = "\t"
            )
            mytxt = "%s. %s" % (red(_("An error occured while checking database integrity")),red(_("Giving up")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = "\t"
            )
            return 1
        return 0


    def show_repository_information(self, repo, count_info):

        self.Entropy.updateProgress(
            bold("%s") % ( etpRepositories[repo]['description'] ),
            importance = 2,
            type = "info",
            count = count_info,
            header = blue("  # ")
        )
        mytxt = "%s: %s" % (red(_("Database URL")),darkgreen(etpRepositories[repo]['database']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = blue("  # ")
        )
        mytxt = "%s: %s" % (red(_("Database local path")),darkgreen(etpRepositories[repo]['dbpath']),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            type = "info",
            header = blue("  # ")
        )
        mytxt = "%s: %s" % (red(_("Database EAPI")),darkgreen(str(self.dbformat_eapi)),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            type = "info",
            header = blue("  # ")
        )

    def get_eapi3_local_database(self, repo):

        dbfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasefile'])
        mydbconn = None
        try:
            mydbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False)
            mydbconn.validateDatabase()
        except (
            self.Entropy.dbapi2.OperationalError,
            self.Entropy.dbapi2.IntegrityError,
            exceptionTools.SystemDatabaseError,
            IOError,
            OSError,):
                mydbconn = None
        return mydbconn

    def get_eapi3_database_differences(self, eapi3_interface, repo, idpackages, session):

        data = eapi3_interface.CmdInterface.differential_packages_comparison(
            session, idpackages, repo, etpConst['currentarch'], etpConst['product']
        )
        if isinstance(data,bool): # then it's probably == False
            return False,False,False
        elif not isinstance(data,dict):
            return None,None,None
        elif not data.has_key('added') or \
            not data.has_key('removed') or \
            not data.has_key('checksum'):
                return None,None,None
        return data['added'],data['removed'],data['checksum']

    def get_eapi3_database_treeupdates(self, eapi3_interface, repo, session):
        self.socket.setdefaulttimeout(self.big_socket_timeout)
        data = eapi3_interface.CmdInterface.get_repository_treeupdates(
            session, repo, etpConst['currentarch'], etpConst['product']
        )
        if not isinstance(data,dict): return None,None
        return data.get('digest'), data.get('actions')

    def get_eapi3_package_sets(self, eapi3_interface, repo, session):
        self.socket.setdefaulttimeout(self.big_socket_timeout)
        data = eapi3_interface.CmdInterface.get_package_sets(
            session, repo, etpConst['currentarch'], etpConst['product']
        )
        if not isinstance(data,dict): return {}
        return data

    def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12):

        def prepare_exit(mysock, session = None):
            try:
                if session != None:
                    mysock.close_session(session)
                mysock.disconnect()
            except (self.socket.error,):
                pass

        eapi3_interface = self.get_eapi3_connection(repo)
        if eapi3_interface == None: return False

        session = eapi3_interface.open_session()

        try:
            mydbconn = self.get_eapi3_local_database(repo)
            myidpackages = mydbconn.listAllIdpackages()
        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
            prepare_exit(eapi3_interface, session)
            return False

        added_ids, removed_ids, checksum = self.get_eapi3_database_differences(
            eapi3_interface, repo,
            myidpackages, session
        )
        if (None in (added_ids,removed_ids,checksum)) or \
            (not added_ids and not removed_ids and self.forceUpdate):
                mydbconn.closeDB()
                prepare_exit(eapi3_interface, session)
                return False

        elif not checksum: # {added_ids, removed_ids, checksum} == False
            mydbconn.closeDB()
            prepare_exit(eapi3_interface, session)
            mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("remote database suddenly locked")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = blue("  # "),
            )
            return None

        # is it worth it?
        if len(added_ids) > threshold:
            mytxt = "%s: %s (%s: %s/%s)" % (
                blue(_("EAPI3 Service")), darkred(_("skipping differential sync")),
                brown(_("threshold")), blue(str(len(added_ids))), darkred(str(threshold)),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = blue("  # "),
            )
            mydbconn.closeDB()
            prepare_exit(eapi3_interface, session)
            return False

        count = 0
        added_segments = []
        mytmp = set()

        for idpackage in added_ids:
            count += 1
            mytmp.add(idpackage)
            if count % chunk_size == 0:
                added_segments.append(list(mytmp))
                mytmp.clear()
        if mytmp: added_segments.append(list(mytmp))
        del mytmp

        # fetch and store
        count = 0
        maxcount = len(added_segments)
        for segment in added_segments:

            count += 1
            mytxt = "%s %s" % (blue(_("Fetching segments")), "...",)
            self.Entropy.updateProgress(
                mytxt, importance = 0, type = "info",
                header = "\t", back = True, count = (count,maxcount,)
            )
            fetch_count = 0
            max_fetch_count = 5

            error_caught = False
            error_rc = None

            while 1:

                # anti loop protection
                if fetch_count > max_fetch_count:
                    mydbconn.closeDB()
                    prepare_exit(eapi3_interface, session)
                    error_caught = True
                    error_rc = False
                    break

                fetch_count += 1
                pkgdata = eapi3_interface.CmdInterface.get_package_information(
                    session, segment, repo, etpConst['currentarch'], etpConst['product']
                )
                if pkgdata == None:
                    mytxt = "%s: %s" % ( blue(_("Fetch error on segment")), darkred(str(segment)),)
                    self.Entropy.updateProgress(
                        mytxt, importance = 1, type = "warning",
                        header = "\t", count = (count,maxcount,)
                    )
                    continue
                elif not pkgdata: # pkgdata == False
                    mytxt = "%s: %s" % (
                        blue(_("Service status")),
                        darkred("remote database suddenly locked"),
                    )
                    self.Entropy.updateProgress(
                        mytxt, importance = 1, type = "info",
                        header = "\t", count = (count,maxcount,)
                    )
                    mydbconn.closeDB()
                    prepare_exit(eapi3_interface, session)
                    error_caught = True
                    break
                elif isinstance(pkgdata,tuple):
                    mytxt = "%s: %s, %s. %s" % ( blue(_("Service status")), pkgdata[0], pkgdata[1], darkred("Error processing the command"),)
                    self.Entropy.updateProgress(
                        mytxt, importance = 1, type = "info",
                        header = "\t", count = (count,maxcount,)
                    )
                    mydbconn.closeDB()
                    prepare_exit(eapi3_interface, session)
                    error_caught = True
                    break

                try:
                    for idpackage in pkgdata:
                        self.dumpTools.dumpobj(
                            "%s%s" % (etpCache['eapi3_fetch'],idpackage,),
                            pkgdata[idpackage],
                            ignoreExceptions = False
                        )
                except (IOError,EOFError,OSError,), e:
                    mytxt = "%s: %s: %s." % ( blue(_("Local status")), darkred("Error storing data"), e,)
                    self.Entropy.updateProgress(
                        mytxt, importance = 1, type = "info",
                        header = "\t", count = (count,maxcount,)
                    )
                    mydbconn.closeDB()
                    prepare_exit(eapi3_interface, session)
                    error_caught = True
                    break

                break

            if error_caught: return error_rc

        del added_segments

        # get treeupdates stuff
        dbdigest, treeupdates_actions = self.get_eapi3_database_treeupdates(eapi3_interface, repo, session)
        if dbdigest == None:
            mydbconn.closeDB()
            prepare_exit(eapi3_interface, session)
            mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("treeupdates data not available")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = blue("  # "),
            )
            return None

        try:
            mydbconn.setRepositoryUpdatesDigest(repo, dbdigest)
            mydbconn.bumpTreeUpdatesActions(treeupdates_actions)
        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
            mydbconn.closeDB()
            prepare_exit(eapi3_interface, session)
            mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update treeupdates data")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = blue("  # "),
            )
            return None


        # get updated package sets
        repo_sets = self.get_eapi3_package_sets(eapi3_interface, repo, session)
        try:
            mydbconn.clearPackageSets()
            mydbconn.insertPackageSets(repo_sets)
        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
            mydbconn.closeDB()
            prepare_exit(eapi3_interface, session)
            mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update package sets data")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = blue("  # "),
            )
            return None

        # I don't need you anymore
        # disconnect socket
        prepare_exit(eapi3_interface, session)

        # now that we have all stored, add
        count = 0
        maxcount = len(added_ids)
        for idpackage in added_ids:
            count += 1
            mydata = self.Entropy.Cacher.pop("%s%s" % (etpCache['eapi3_fetch'],idpackage,))
            if mydata == None:
                mytxt = "%s: %s" % (
                    blue(_("Fetch error on segment while adding")),
                    darkred(str(segment)),
                )
                self.Entropy.updateProgress(
                    mytxt, importance = 1, type = "warning",
                    header = "\t", count = (count,maxcount,)
                )
                mydbconn.closeDB()
                return False

            mytxt = "%s %s" % (blue(_("Injecting package")), darkgreen(mydata['atom']),)
            self.Entropy.updateProgress(
                mytxt, importance = 0, type = "info",
                header = "\t", back = True, count = (count,maxcount,)
            )
            mydbconn.addPackage(
                mydata, revision = mydata['revision'],
                idpackage = idpackage, do_remove = False,
                do_commit = False, formatted_content = True
            )

        self.Entropy.updateProgress(
            blue(_("Packages injection complete")), importance = 0,
            type = "info", header = "\t",
        )

        # now remove
        maxcount = len(removed_ids)
        count = 0
        for idpackage in removed_ids:
            myatom = mydbconn.retrieveAtom(idpackage)
            count += 1
            mytxt = "%s: %s" % (blue(_("Removing package")), darkred(str(myatom)),)
            self.Entropy.updateProgress(
                mytxt, importance = 0, type = "info",
                header = "\t", back = True, count = (count,maxcount,)
            )
            mydbconn.removePackage(idpackage, do_cleanup = False, do_commit = False)

        self.Entropy.updateProgress(
            blue(_("Packages removal complete")),
            importance = 0, type = "info",
            header = "\t",
        )

        #mydbconn.doCleanups()
        mydbconn.commitChanges()
        # now verify if both checksums match
        result = False
        mychecksum = mydbconn.database_checksum(do_order = True, strict = False, strings = True)
        if checksum == mychecksum:
            result = True
        else:
            mytxt = "%s: %s: %s | %s: %s" % (
                blue(_("Database checksum doesn't match remote.")),
                darkgreen(_("local")), mychecksum,
                darkred(_("remote")), checksum,
            )
            self.Entropy.updateProgress(
                mytxt, importance = 0,
                type = "info", header = "\t",
            )

        mydbconn.closeDB()
        return result

    def run_sync(self):

        self.dbupdated = False
        repocount = 0
        repolength = len(self.reponames)
        for repo in self.reponames:

            repocount += 1
            self.reset_dbformat_eapi(repo)
            self.show_repository_information(repo, (repocount,repolength))

            if not self.forceUpdate:
                updated = self.handle_repository_update(repo)
                if updated:
                    self.Entropy.cycleDone()
                    self.alreadyUpdated += 1
                    continue

            locked = self.handle_repository_lock(repo)
            if locked:
                self.notAvailable += 1
                self.Entropy.cycleDone()
                continue

            # clear database interface cache belonging to this repository
            self.clear_repository_cache(repo)
            self.__ensure_repository_path(repo)

            # dealing with EAPI
            # setting some vars
            do_skip = False
            skip_this_repo = False
            db_down_status = False
            do_db_update_transfer = False
            rc = 0
            # some variables
            dumpfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasedump'])
            dbfile = os.path.join(etpRepositories[repo]['dbpath'],etpConst['etpdatabasefile'])
            dbfile_old = dbfile+".sync"

            while 1:

                if do_skip:
                    break

                if self.dbformat_eapi < 3:

                    cmethod = self.__validate_compression_method(repo)
                    down_status = self.handle_database_download(repo, cmethod)
                    if not down_status:
                        self.Entropy.cycleDone()
                        self.notAvailable += 1
                        do_skip = True
                        skip_this_repo = True
                        continue
                    db_down_status = self.handle_database_checksum_download(repo, cmethod)
                    break

                elif self.dbformat_eapi == 3 and not (os.path.isfile(dbfile) and os.access(dbfile,os.W_OK)):

                    do_db_update_transfer = None
                    self.dbformat_eapi -= 1
                    continue

                elif self.dbformat_eapi == 3:

                    status = False
                    try:
                        status = self.handle_eapi3_database_sync(repo)
                    except self.socket.error, e:
                        mytxt = "%s: %s" % (
                            blue(_("EAPI3 Service error")),
                            darkred(unicode(e)),
                        )
                        self.Entropy.updateProgress(
                            mytxt,
                            importance = 0,
                            type = "info",
                            header = blue("  # "),
                        )
                        status = False
                    if status == None: # remote db not available anymore ?
                        time.sleep(5)
                        locked = self.handle_repository_lock(repo)
                        if locked:
                            self.Entropy.cycleDone()
                            self.notAvailable += 1
                            do_skip = True
                            skip_this_repo = True
                        else: # ah, well... dunno then...
                            do_db_update_transfer = None
                            self.dbformat_eapi -= 1
                        continue
                    elif not status: # (status == False)
                        # set to none and completely skip database alignment
                        do_db_update_transfer = None
                        self.dbformat_eapi -= 1
                        continue

                    break

            if skip_this_repo:
                continue

            if self.dbformat_eapi in (1,2,):

                if self.dbformat_eapi == 2 and db_down_status:
                    rc = self.check_downloaded_database(repo, cmethod)
                    if rc != 0:
                        # delete all
                        self.__remove_repository_files(repo, cmethod)
                        self.syncErrors = True
                        self.Entropy.cycleDone()
                        continue

                if isinstance(do_db_update_transfer,bool) and not do_db_update_transfer:
                    if os.path.isfile(dbfile):
                        try:
                            shutil.move(dbfile,dbfile_old)
                            do_db_update_transfer = True
                        except:
                            pass

                # unpack database
                unpack_status = self.handle_downloaded_database_unpack(repo, cmethod)
                if not unpack_status:
                    # delete all
                    self.__remove_repository_files(repo, cmethod)
                    self.syncErrors = True
                    self.Entropy.cycleDone()
                    continue

                if self.dbformat_eapi == 1 and db_down_status:
                    rc = self.check_downloaded_database(repo, cmethod)
                    if rc != 0:
                        # delete all
                        self.__remove_repository_files(repo, cmethod)
                        self.syncErrors = True
                        self.Entropy.cycleDone()
                        if os.path.isfile(dbfile_old):
                            os.remove(dbfile_old)
                        continue

                # re-validate
                if not os.path.isfile(dbfile):
                    do_db_update_transfer = False
                elif os.path.isfile(dbfile) and not do_db_update_transfer and (self.dbformat_eapi != 1):
                    os.remove(dbfile)

                if self.dbformat_eapi == 2:
                    rc = self.do_eapi2_inject_downloaded_dump(dumpfile, dbfile, cmethod)

                if do_db_update_transfer:
                    self.do_eapi1_eapi2_databases_alignment(dbfile, dbfile_old)
                if self.dbformat_eapi == 2:
                    # remove the dump
                    os.remove(dumpfile)

            if rc != 0:
                # delete all
                self.__remove_repository_files(repo, cmethod)
                self.syncErrors = True
                self.Entropy.cycleDone()
                if os.path.isfile(dbfile_old):
                    os.remove(dbfile_old)
                continue

            if os.path.isfile(dbfile) and os.access(dbfile,os.W_OK):
                try:
                    self.Entropy.setup_default_file_perms(dbfile)
                except OSError: # notification applet
                    pass

            # database is going to be updated
            self.dbupdated = True
            self.do_standard_items_download(repo)
            self.Entropy.update_repository_revision(repo)
            if self.Entropy.indexing:
                self.do_database_indexing(repo)
            if (repo == etpConst['officialrepositoryid']):
                try:
                    self.run_config_files_updates(repo)
                except Exception, e:
                    self.entropyTools.printTraceback()
                    mytxt = "%s: %s" % (
                        blue(_("Configuration files update error, not critical, continuing")),
                        darkred(unicode(e)),
                    )
                    self.Entropy.updateProgress(mytxt, importance = 0, type = "info", header = blue("  # "),)
            self.updated_repos.add(repo)
            self.Entropy.cycleDone()

            # remove garbage
            if os.path.isfile(dbfile_old):
                os.remove(dbfile_old)

        # keep them closed
        self.Entropy.closeAllRepositoryDatabases()
        self.Entropy.validate_repositories()
        self.Entropy.closeAllRepositoryDatabases()

        # clean caches, fetch security
        if self.dbupdated:
            self.Entropy.generate_cache(
                depcache = self.Entropy.xcache,
                configcache = False,
                client_purge = False,
                install_queue = False
            )
            if self.fetchSecurity:
                self.do_update_security_advisories()
            # do treeupdates
            if isinstance(self.Entropy.clientDbconn,EntropyDatabaseInterface):
                for repo in self.reponames:
                    dbc = self.Entropy.openRepositoryDatabase(repo)
                    dbc.clientUpdatePackagesData(self.Entropy.clientDbconn)
                self.Entropy.closeAllRepositoryDatabases()

        if self.syncErrors:
            self.Entropy.updateProgress(
                red(_("Something bad happened. Please have a look.")),
                importance = 1,
                type = "warning",
                header = darkred(" @@ ")
            )
            self.syncErrors = True
            self.Entropy._resources_run_remove_lock()
            return 128

        if not self.noEquoCheck:
            self.check_entropy_updates()

        return 0

    def run_config_files_updates(self, repo):

        # are we root?
        if etpConst['uid'] != 0:
            self.Entropy.updateProgress(
                brown(_("Skipping configuration files update, you are not root.")),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            return

        # make.conf
        self._config_updates_make_conf(repo)
        self._config_updates_make_profile(repo)


    def _config_updates_make_conf(self, repo):

        ## WARNING: it doesn't handle multi-line variables, yet. remember this.
        url, repo_make_conf = self._construct_paths("make.conf", repo, None)
        system_make_conf = etpConst['spm']['global_make_conf']
        make_conf_variables_check = ["CHOST"]

        if os.path.isfile(repo_make_conf) and os.access(repo_make_conf,os.R_OK):

            if not os.path.isfile(system_make_conf):
                self.Entropy.updateProgress(
                    "%s %s. %s." % (red(system_make_conf),blue(_("does not exist")),blue(_("Overwriting")),),
                    importance = 1,
                    type = "info",
                    header = blue(" @@ ")
                )
                if os.path.lexists(system_make_conf):
                    shutil.move(
                        system_make_conf,
                        "%s.backup_%s" % (system_make_conf,self.entropyTools.getRandomNumber(),)
                    )
                shutil.copy2(repo_make_conf,system_make_conf)

            elif os.access(system_make_conf,os.W_OK):

                repo_f = open(repo_make_conf,"r")
                sys_f = open(system_make_conf,"r")
                repo_make_c = [x.strip() for x in repo_f.readlines()]
                sys_make_c = [x.strip() for x in sys_f.readlines()]
                repo_f.close()
                sys_f.close()

                # read repository settings
                repo_data = {}
                for setting in make_conf_variables_check:
                    for line in repo_make_c:
                        if line.startswith(setting+"="):
                            # there can't be bash vars with a space after its name on declaration
                            repo_data[setting] = line
                            # I don't break, because there might be other overlapping settings

                differences = {}
                # update make.conf data in memory
                for setting in repo_data:
                    for idx in range(len(sys_make_c)):
                        line = sys_make_c[idx]
                        if line.startswith(setting+"=") and (line != repo_data[setting]):
                            # there can't be bash vars with a space after its name on declaration
                            self.Entropy.updateProgress(
                                "%s: %s %s. %s." % (
                                    red(system_make_conf), bold(unicode(setting)),
                                    blue(_("variable differs")), red(_("Updating")),
                                ),
                                importance = 1,
                                type = "info",
                                header = blue(" @@ ")
                            )
                            differences[setting] = repo_data[setting]
                            line = repo_data[setting]
                        sys_make_c[idx] = line

                if differences:

                    self.Entropy.updateProgress(
                        "%s: %s." % (red(system_make_conf), blue(_("updating critical variables")),),
                        importance = 1,
                        type = "info",
                        header = blue(" @@ ")
                    )
                    # backup user make.conf
                    shutil.copy2(system_make_conf,"%s.entropy_backup" % (system_make_conf,))

                    self.Entropy.updateProgress(
                        "%s: %s." % (
                            red(system_make_conf), darkgreen("writing changes to disk"),
                        ),
                        importance = 1,
                        type = "info",
                        header = blue(" @@ ")
                    )
                    # write to disk, safely
                    tmp_make_conf = "%s.entropy_write" % (system_make_conf,)
                    f = open(tmp_make_conf,"w")
                    for line in sys_make_c: f.write(line+"\n")
                    f.flush()
                    f.close()
                    shutil.move(tmp_make_conf,system_make_conf)

                # update environment
                for var in differences:
                    try:
                        myval = '='.join(differences[var].strip().split("=")[1:])
                        if myval:
                            if myval[0] in ("'",'"',): myval = myval[1:]
                            if myval[-1] in ("'",'"',): myval = myval[:-1]
                    except IndexError:
                        myval = ''
                    os.environ[var] = myval

    def _config_updates_make_profile(self, repo):
        url, repo_make_profile = self._construct_paths("profile.link", repo, None)
        system_make_profile = etpConst['spm']['global_make_profile']
        if not (os.path.isfile(repo_make_profile) and os.access(repo_make_profile,os.R_OK)):
            return
        f = open(repo_make_profile,"r")
        repo_profile_link_data = f.readline().strip()
        f.close()
        current_profile_link = ''
        if os.path.islink(system_make_profile) and os.access(system_make_profile,os.R_OK):
            current_profile_link = os.readlink(system_make_profile)
        if repo_profile_link_data != current_profile_link:
            self.Entropy.updateProgress(
                "%s: %s %s. %s." % (
                    red(system_make_profile), blue("link"),
                    blue(_("differs")), red(_("Updating")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            if os.path.lexists(repo_make_profile):
                shutil.move(system_make_profile,"%s.entropy_old" % (system_make_profile,))
            # broken symlink?
            if os.path.islink(system_make_profile) and not os.path.isdir(system_make_profile):
                os.remove(system_make_profile)
            os.symlink(repo_profile_link_data,system_make_profile)
            if not self.entropyTools.is_valid_path(system_make_profile):
                # revert change, link does not exist yet
                self.Entropy.updateProgress(
                    "%s: %s %s. %s." % (
                        red(system_make_profile), blue("new link"),
                        blue(_("does not exist")), red(_("Reverting")),
                    ),
                    importance = 1,
                    type = "info",
                    header = blue(" @@ ")
                )
                os.symlink(current_profile_link,system_make_profile)


    def check_entropy_updates(self):
        rc = False
        if not self.noEquoCheck:
            try:
                rc = self.Entropy.check_equo_updates()
            except:
                pass
        if rc:
            self.newEquo = True
            mytxt = "%s: %s. %s." % (
                bold("Equo/Entropy"),
                blue(_("a new release is available")),
                darkred(_("Mind to install it before any other package")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = bold(" !!! ")
            )

    def handle_downloaded_database_unpack(self, repo, cmethod):

        file_to_unpack = etpConst['etpdatabasedump']
        if self.dbformat_eapi == 1:
            file_to_unpack = etpConst['etpdatabasefile']
        mytxt = "%s %s %s" % (red(_("Unpacking database to")),darkgreen(file_to_unpack),red("..."),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            type = "info",
            header = "\t"
        )

        myrc = self.__unpack_downloaded_database(repo, cmethod)
        if myrc != 0:
            mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),red(_("Skipping repository")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )
            return False
        return True


    def handle_database_checksum_download(self, repo, cmethod):

        hashfile = etpConst['etpdatabasehashfile']
        downitem = 'ck'
        if self.dbformat_eapi == 2: # EAPI = 2
            hashfile = etpConst[cmethod[4]]
            downitem = 'dbdumpck'

        mytxt = "%s %s %s" % (red(_("Downloading checksum")),darkgreen(hashfile),red("..."),)
        # download checksum
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            type = "info",
            header = "\t"
        )

        db_down_status = self.download_item(downitem, repo, cmethod, disallow_redirect = True)
        if not db_down_status:
            mytxt = "%s %s !" % (red(_("Cannot fetch checksum")),red(_("Cannot verify database integrity")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )
        return db_down_status

    def load_background_repository_lock_check(self, repo):
        # kill previous
        self.current_repository_got_locked = False
        self.kill_previous_repository_lock_scanner()
        self.LockScanner = self.entropyTools.TimeScheduled( self.repository_lock_scanner, 2, {'repo': repo} )
        self.LockScanner.setName("Lock_Scanner::"+str(abs(hash(os.urandom(20)))))
        self.LockScanner.start()

    def kill_previous_repository_lock_scanner(self):
        if self.LockScanner != None:
            self.LockScanner.kill()

    def repository_lock_scanner(self, data):
        repo = data['repo']
        locked = self.handle_repository_lock(repo)
        if locked:
            self.current_repository_got_locked = True

    def repository_lock_scanner_status(self):
        # raise an exception if repo got suddenly locked
        if self.current_repository_got_locked:
            mytxt = _("Current repository got suddenly locked. Download aborted.")
            raise exceptionTools.RepositoryError('RepositoryError %s' % (mytxt,))

    def handle_database_download(self, repo, cmethod):

        def show_repo_locked_message():
            mytxt = "%s: %s." % (bold(_("Attention")),red(_("remote database got suddenly locked")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )

        # starting to download
        mytxt = "%s ..." % (red(_("Downloading repository database")),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = "\t"
        )

        down_status = False
        if self.dbformat_eapi == 2:
            # start a check in background
            self.load_background_repository_lock_check(repo)
            down_status = self.download_item("dbdump", repo, cmethod, lock_status_func = self.repository_lock_scanner_status, disallow_redirect = True)
            if self.current_repository_got_locked:
                self.kill_previous_repository_lock_scanner()
                show_repo_locked_message()
                return False
        if not down_status: # fallback to old db
            # start a check in background
            self.load_background_repository_lock_check(repo)
            self.dbformat_eapi = 1
            down_status = self.download_item("db", repo, cmethod, lock_status_func = self.repository_lock_scanner_status, disallow_redirect = True)
            if self.current_repository_got_locked:
                self.kill_previous_repository_lock_scanner()
                show_repo_locked_message()
                return False

        if not down_status:
            mytxt = "%s: %s." % (bold(_("Attention")),red(_("database does not exist online")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )

        self.kill_previous_repository_lock_scanner()
        return down_status

    def handle_repository_update(self, repo):
        # check if database is already updated to the latest revision
        update = self.is_repository_updatable(repo)
        if not update:
            mytxt = "%s: %s." % (bold(_("Attention")),red(_("database is already up to date")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = "\t"
            )
            return True
        # also check for eapi3 lock
        if self.dbformat_eapi == 3:
            locked = self.is_repository_eapi3_locked(repo)
            if locked:
                mytxt = "%s: %s." % (bold(_("Attention")),red(_("database will be ready soon")),)
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = "\t"
                )
                return True
        return False

    def handle_repository_lock(self, repo):
        # get database lock
        unlocked = self.is_repository_unlocked(repo)
        if not unlocked:
            mytxt = "%s: %s. %s." % (
                bold(_("Attention")),
                red(_("Repository is being updated")),
                red(_("Try again in a few minutes")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = "\t"
            )
            return True
        return False

    def do_eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):

        dbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False)
        old_dbconn = self.Entropy.openGenericDatabase(dbfile_old, xcache = False, indexing_override = False)
        upd_rc = 0
        try:
            upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t")
        except (dbapi2.OperationalError,dbapi2.IntegrityError,):
            pass
        old_dbconn.closeDB()
        dbconn.closeDB()
        if upd_rc > 0:
            # -1 means no changes, == force used
            # 0 means too much hassle
            shutil.move(dbfile_old,dbfile)
        return upd_rc

    def do_eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):

        # load the dump into database
        mytxt = "%s %s, %s %s" % (
            red(_("Injecting downloaded dump")),
            darkgreen(etpConst[cmethod[3]]),
            red(_("please wait")),
            red("..."),
        )
        self.Entropy.updateProgress(
            mytxt,
            importance = 0,
            type = "info",
            header = "\t"
        )
        dbconn = self.Entropy.openGenericDatabase(dbfile, xcache = False, indexing_override = False)
        rc = dbconn.doDatabaseImport(dumpfile, dbfile)
        dbconn.closeDB()
        return rc


    def do_update_security_advisories(self):
        # update Security Advisories
        try:
            securityConn = self.Entropy.Security()
            securityConn.fetch_advisories()
        except Exception, e:
            self.entropyTools.printTraceback(f = self.Entropy.clientLog)
            mytxt = "%s: %s" % (red(_("Advisories fetch error")),e,)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = darkred(" @@ ")
            )

    def do_standard_items_download(self, repo):

        download_items = [
            (
                "ca.cert",
                etpConst['etpdatabasecacertfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading SSL CA certificate")),
                    darkgreen(etpConst['etpdatabasecacertfile']),
                    red("..."),
                )
            ),
            (
                "server.cert",
                etpConst['etpdatabaseservercertfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading SSL Server certificate")),
                    darkgreen(etpConst['etpdatabaseservercertfile']),
                    red("..."),
                )
            ),
            (
                "mask",
                etpConst['etpdatabasemaskfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading package mask")),
                    darkgreen(etpConst['etpdatabasemaskfile']),
                    red("..."),
                )
            ),
            (
                "system_mask",
                etpConst['etpdatabasesytemmaskfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading packages system mask")),
                    darkgreen(etpConst['etpdatabasesytemmaskfile']),
                    red("..."),
                )
            ),
            (
                "conflicting_tagged",
                etpConst['etpdatabaseconflictingtaggedfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading conflicting tagged packages file")),
                    darkgreen(etpConst['etpdatabaseconflictingtaggedfile']),
                    red("..."),
                )
            ),
            (
                "lic_whitelist",
                etpConst['etpdatabaselicwhitelistfile'],
                True,
                "%s %s %s" % (
                    red(_("Downloading license whitelist")),
                    darkgreen(etpConst['etpdatabaselicwhitelistfile']),
                    red("..."),
                )
            ),
            (
                "rev",
                etpConst['etpdatabasemaskfile'],
                False,
                "%s %s %s" % (
                    red(_("Downloading revision")),
                    darkgreen(etpConst['etpdatabaserevisionfile']),
                    red("..."),
                )
            ),
            (
                "make.conf",
                os.path.basename(etpConst['spm']['global_make_conf']),
                True,
                "%s %s %s" % (
                    red(_("Downloading SPM global configuration")),
                    darkgreen(os.path.basename(etpConst['spm']['global_make_conf'])),
                    red("..."),
                )
            ),
            (
                "package.unmask",
                os.path.basename(etpConst['spm']['global_package_unmask']),
                True,
                "%s %s %s" % (
                    red(_("Downloading SPM package unmasking configuration")),
                    darkgreen(os.path.basename(etpConst['spm']['global_package_unmask'])),
                    red("..."),
                )
            ),
            (
                "package.keywords",
                os.path.basename(etpConst['spm']['global_package_keywords']),
                True,
                "%s %s %s" % (
                    red(_("Downloading SPM package keywording configuration")),
                    darkgreen(os.path.basename(etpConst['spm']['global_package_keywords'])),
                    red("..."),
                )
            ),
            (
                "package.use",
                os.path.basename(etpConst['spm']['global_package_use']),
                True,
                "%s %s %s" % (
                    red(_("Downloading SPM package USE flags configuration")),
                    darkgreen(os.path.basename(etpConst['spm']['global_package_use'])),
                    red("..."),
                )
            ),
            (
                "profile.link",
                etpConst['spm']['global_make_profile_link_name'],
                True,
                "%s %s %s" % (
                    red(_("Downloading SPM Profile configuration")),
                    darkgreen(etpConst['spm']['global_make_profile_link_name']),
                    red("..."),
                )
            ),
            (
                "notice_board",
                os.path.basename(etpRepositories[repo]['local_notice_board']),
                True,
                "%s %s %s" % (
                    red(_("Downloading Notice Board")),
                    darkgreen(os.path.basename(etpRepositories[repo]['local_notice_board'])),
                    red("..."),
                )
            )
        ]

        for item, myfile, ignorable, mytxt in download_items:
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = "\t",
                back = True
            )
            mystatus = self.download_item(item, repo, disallow_redirect = True)
            mytype = 'info'
            if not mystatus:
                if ignorable:
                    message = "%s: %s." % (blue(myfile),red(_("not available, it's ok")))
                else:
                    mytype = 'warning'
                    message = "%s: %s." % (blue(myfile),darkred(_("not available, not much ok!")))
            else:
                message = "%s: %s." % (blue(myfile),darkgreen(_("available, w00t!")))
            self.Entropy.updateProgress(
                message,
                importance = 0,
                type = mytype,
                header = "\t"
            )

        mytxt = "%s: %s" % (
            red(_("Repository revision")),
            bold(str(self.Entropy.get_repository_revision(repo))),
        )
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = "\t"
        )



    def do_database_indexing(self, repo):

        # renice a bit, to avoid eating resources
        old_prio = self.Entropy.set_priority(15)
        mytxt = red("%s ...") % (_("Indexing Repository metadata"),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = "\t",
            back = True
        )
        dbconn = self.Entropy.openRepositoryDatabase(repo)
        dbconn.createAllIndexes()
        # get list of indexes
        repo_indexes = dbconn.listAllIndexes()
        if self.Entropy.clientDbconn != None:
            try: # client db can be absent
                client_indexes = self.Entropy.clientDbconn.listAllIndexes()
                if repo_indexes != client_indexes:
                    self.Entropy.clientDbconn.createAllIndexes()
            except:
                pass
        self.Entropy.set_priority(old_prio)


    def sync(self):

        # close them
        self.Entropy.closeAllRepositoryDatabases()

        # let's dance!
        mytxt = darkgreen("%s ...") % (_("Repositories synchronization"),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = darkred(" @@ ")
        )

        gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock)
        if gave_up:
            return 3

        locked = self.Entropy.application_lock_check()
        if locked:
            self.Entropy._resources_run_remove_lock()
            return 4

        # lock
        self.Entropy._resources_run_create_lock()
        try:
            rc = self.run_sync()
        except:
            self.Entropy._resources_run_remove_lock()
            raise
        if rc: return rc

        # remove lock
        self.Entropy._resources_run_remove_lock()

        if (self.notAvailable >= len(self.reponames)):
            return 2
        elif (self.notAvailable > 0):
            return 1

        return 0

class QAInterface:

    import entropyTools
    def __init__(self, EntropyInterface):
        if not isinstance(EntropyInterface, (EquoInterface, ServerInterface)) and \
            not issubclass(EntropyInterface, (EquoInterface, ServerInterface)):
                mytxt = _("A valid EquoInterface/ServerInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,))
        self.Entropy = EntropyInterface

    def test_depends_linking(self, idpackages, dbconn, repo = etpConst['officialrepositoryid']):

        scan_msg = blue(_("Now searching for broken depends"))
        self.Entropy.updateProgress(
            "[repo:%s] %s..." % (
                        darkgreen(repo),
                        scan_msg,
                    ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        broken = False

        count = 0
        maxcount = len(idpackages)
        for idpackage in idpackages:
            count += 1
            atom = dbconn.retrieveAtom(idpackage)
            scan_msg = "%s, %s:" % (blue(_("scanning for broken depends")),darkgreen(atom),)
            self.Entropy.updateProgress(
                "[repo:%s] %s" % (
                    darkgreen(repo),
                    scan_msg,
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True,
                count = (count,maxcount,)
            )
            mydepends = dbconn.retrieveDepends(idpackage)
            if not mydepends:
                continue
            for mydepend in mydepends:
                myatom = dbconn.retrieveAtom(mydepend)
                self.Entropy.updateProgress(
                    "[repo:%s] %s => %s" % (
                        darkgreen(repo),
                        darkgreen(atom),
                        darkred(myatom),
                    ),
                    importance = 0,
                    type = "info",
                    header = blue(" @@ "),
                    back = True,
                    count = (count,maxcount,)
                )
                mycontent = dbconn.retrieveContent(mydepend)
                mybreakages = self.content_test(mycontent)
                if not mybreakages:
                    continue
                broken = True
                self.Entropy.updateProgress(
                    "[repo:%s] %s %s => %s" % (
                        darkgreen(repo),
                        darkgreen(atom),
                        darkred(myatom),
                        bold(_("broken libraries detected")),
                    ),
                    importance = 1,
                    type = "warning",
                    header = purple(" @@ "),
                    count = (count,maxcount,)
                )
                for mylib in mybreakages:
                    self.Entropy.updateProgress(
                        "%s %s:" % (
                            darkgreen(mylib),
                            red(_("needs")),
                        ),
                        importance = 1,
                        type = "warning",
                        header = brown("   ## ")
                    )
                    for needed in mybreakages[mylib]:
                        self.Entropy.updateProgress(
                            "%s" % (
                                red(needed),
                            ),
                            importance = 1,
                            type = "warning",
                            header = purple("     # ")
                        )
        return broken


    def scan_missing_dependencies(self, idpackages, dbconn, ask = True, self_check = False, repo = etpConst['officialrepositoryid'], black_list = set(), black_list_adder = None):

        taint = False
        scan_msg = blue(_("Now searching for missing RDEPENDs"))
        self.Entropy.updateProgress(
            "[repo:%s] %s..." % (
                        darkgreen(repo),
                        scan_msg,
                    ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )
        scan_msg = blue(_("scanning for missing RDEPENDs"))
        count = 0
        maxcount = len(idpackages)
        for idpackage in idpackages:
            count += 1
            atom = dbconn.retrieveAtom(idpackage)
            if not atom: continue
            self.Entropy.updateProgress(
                "[repo:%s] %s: %s" % (
                            darkgreen(repo),
                            scan_msg,
                            darkgreen(atom),
                        ),
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True,
                count = (count,maxcount,)
            )
            missing_extended, missing = self.get_missing_rdepends(dbconn, idpackage, self_check = self_check)
            missing -= black_list
            for item in missing_extended.keys():
                missing_extended[item] -= black_list
                if not missing_extended[item]:
                    del missing_extended[item]
            if (not missing) or (not missing_extended):
                continue
            self.Entropy.updateProgress(
                "[repo:%s] %s: %s %s:" % (
                            darkgreen(repo),
                            blue("package"),
                            darkgreen(atom),
                            blue(_("is missing the following dependencies")),
                        ),
                importance = 1,
                type = "info",
                header = red(" @@ "),
                count = (count,maxcount,)
            )
            for missing_data in missing_extended:
                self.Entropy.updateProgress(
                        "%s:" % (brown(unicode(missing_data)),),
                        importance = 0,
                        type = "info",
                        header = purple("   ## ")
                )
                for dependency in missing_extended[missing_data]:
                    self.Entropy.updateProgress(
                            "%s" % (darkred(dependency),),
                            importance = 0,
                            type = "info",
                            header = blue("     # ")
                    )
            if ask:
                rc = self.Entropy.askQuestion(_("Do you want to add them?"))
                if rc == "No":
                    continue
                rc = self.Entropy.askQuestion(_("Selectively?"))
                if rc == "Yes":
                    newmissing = set()
                    new_blacklist = set()
                    for dependency in missing:
                        self.Entropy.updateProgress(
                            "[repo:%s|%s] %s" % (
                                    darkgreen(repo),
                                    brown(atom),
                                    blue(dependency),
                            ),
                            importance = 0,
                            type = "info",
                            header = blue(" @@ ")
                        )
                        rc = self.Entropy.askQuestion(_("Want to add?"))
                        if rc == "Yes":
                            newmissing.add(dependency)
                        else:
                            rc = self.Entropy.askQuestion(_("Want to blacklist?"))
                            if rc == "Yes":
                                new_blacklist.add(dependency)
                    if new_blacklist and (black_list_adder != None):
                        black_list_adder(new_blacklist, repo = repo)
                    missing = newmissing
            if missing:
                taint = True
                dbconn.insertDependencies(idpackage,missing)
                dbconn.commitChanges()
                self.Entropy.updateProgress(
                    "[repo:%s] %s: %s" % (
                                darkgreen(repo),
                                darkgreen(atom),
                                blue(_("missing dependencies added")),
                            ),
                    importance = 1,
                    type = "info",
                    header = red(" @@ "),
                    count = (count,maxcount,)
                )

        return taint

    def content_test(self, mycontent):

        def is_contained(needed, content):
            for item in content:
                if os.path.basename(item) == needed:
                    return True
            return False

        mylibs = {}
        for myfile in mycontent:
            myfile = myfile.encode('raw_unicode_escape')
            if not os.access(myfile,os.R_OK):
                continue
            if not os.path.isfile(myfile):
                continue
            if not self.entropyTools.is_elf_file(myfile):
                continue
            mylibs[myfile] = self.entropyTools.read_elf_dynamic_libraries(myfile)

        broken_libs = {}
        for mylib in mylibs:
            for myneeded in mylibs[mylib]:
                # is this inside myself ?
                if is_contained(myneeded, mycontent):
                    continue
                found = self.resolve_dynamic_library(myneeded, mylib)
                if found:
                    continue
                if not broken_libs.has_key(mylib):
                    broken_libs[mylib] = set()
                broken_libs[mylib].add(myneeded)

        return broken_libs

    def resolve_dynamic_library(self, library, requiring_executable):

        def do_resolve(mypaths):
            found_path = None
            for mypath in mypaths:
                mypath = os.path.join(etpConst['systemroot']+mypath,library)
                if not os.access(mypath,os.R_OK):
                    continue
                if os.path.isdir(mypath):
                    continue
                if not self.entropyTools.is_elf_file(mypath):
                    continue
                found_path = mypath
                break
            return found_path

        mypaths = self.entropyTools.collectLinkerPaths()
        found_path = do_resolve(mypaths)

        if not found_path:
            mypaths = self.entropyTools.read_elf_linker_paths(requiring_executable)
            found_path = do_resolve(mypaths)

        return found_path

    def get_missing_rdepends(self, dbconn, idpackage, self_check = False):

        rdepends = {}
        rdepends_plain = set()
        neededs = dbconn.retrieveNeeded(idpackage, extended = True)
        ldpaths = set(self.entropyTools.collectLinkerPaths())
        deps_content = set()
        dependencies = self._get_deep_dependency_list(dbconn, idpackage, atoms = True)
        dependencies_cache = set()

        def update_depscontent(mycontent, dbconn, ldpaths):
            return set( \
                    [   x for x in mycontent if os.path.dirname(x) in ldpaths \
                        and (dbconn.isNeededAvailable(os.path.basename(x)) > 0) ] \
                    )

        def is_in_content(myneeded, content):
            for item in content:
                item = os.path.basename(item)
                if myneeded == item:
                    return True
            return False

        for dependency in dependencies:
            match = dbconn.atomMatch(dependency)
            if match[0] != -1:
                mycontent = dbconn.retrieveContent(match[0])
                deps_content |= update_depscontent(mycontent, dbconn, ldpaths)
                key, slot = dbconn.retrieveKeySlot(match[0])
                dependencies_cache.add((key,slot))

        key, slot = dbconn.retrieveKeySlot(idpackage)
        mycontent = dbconn.retrieveContent(idpackage)
        deps_content |= update_depscontent(mycontent, dbconn, ldpaths)
        dependencies_cache.add((key,slot))

        idpackages_cache = set()
        for needed, elfclass in neededs:
            data_solved = dbconn.resolveNeeded(needed,elfclass)
            data_size = len(data_solved)
            data_solved = set([x for x in data_solved if x[0] not in idpackages_cache])
            if not data_solved or (data_size != len(data_solved)):
                continue

            if self_check:
                if is_in_content(needed,mycontent):
                    continue

            found = False
            for data in data_solved:
                if data[1] in deps_content:
                    found = True
                    break
            if not found:
                for data in data_solved:
                    key, slot = dbconn.retrieveKeySlot(data[0])
                    if (key,slot) not in dependencies_cache:
                        if not dbconn.isSystemPackage(data[0]):
                            if not rdepends.has_key((needed,elfclass)):
                                rdepends[(needed,elfclass)] = set()
                            rdepends[(needed,elfclass)].add(key+":"+slot)
                            rdepends_plain.add(key+":"+slot)
                        idpackages_cache.add(data[0])
        return rdepends, rdepends_plain

    def _get_deep_dependency_list(self, dbconn, idpackage, atoms = False):

        mybuffer = self.entropyTools.lifobuffer()
        matchcache = set()
        depcache = set()
        mydeps = dbconn.retrieveDependencies(idpackage)
        for mydep in mydeps:
            mybuffer.push(mydep)
        mydep = mybuffer.pop()

        while mydep:

            if mydep in depcache:
                mydep = mybuffer.pop()
                continue

            mymatch = dbconn.atomMatch(mydep)
            if atoms:
                matchcache.add(mydep)
            else:
                matchcache.add(mymatch[0])

            if mymatch[0] != -1:
                owndeps = dbconn.retrieveDependencies(mymatch[0])
                for owndep in owndeps: mybuffer.push(owndep)

            depcache.add(mydep)
            mydep = mybuffer.pop()

        if atoms and -1 in matchcache:
            matchcache.discard(-1)

        return matchcache

'''
   Entropy FTP interface
'''
class FtpInterface:

    # this must be run before calling the other functions
    def __init__(self, ftpuri, EntropyInterface, verbose = True):

        if not isinstance(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)) and \
            not issubclass(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)):
                mytxt = _("A valid TextInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,))

        import socket, ftplib, entropyTools
        self.socket, self.ftplib, self.entropyTools = socket, ftplib, entropyTools
        self.Entropy = EntropyInterface
        self.verbose = verbose
        self.oldprogress = 0.0

        # import FTP modules
        self.socket.setdefaulttimeout(60)

        self.ftpuri = ftpuri
        self.ftphost = self.entropyTools.extractFTPHostFromUri(self.ftpuri)

        self.ftpuser, self.ftppassword, self.ftpport, self.ftpdir = self.entropyTools.extract_ftp_data(ftpuri)

        count = 10
        while 1:
            count -= 1
            try:
                self.ftpconn = self.ftplib.FTP(self.ftphost)
                break
            except (self.socket.gaierror,), e:
                raise exceptionTools.ConnectionError('ConnectionError: %s' % (e,))
            except:
                if not count: raise
                continue

        if self.verbose:
            mytxt = _("connecting with user")
            self.Entropy.updateProgress(
                "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpuser),),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
        try:
            self.ftpconn.login(self.ftpuser,self.ftppassword)
        except self.ftplib.error_perm, e:
            raise exceptionTools.FtpError('FtpError: %s' % (e,))
        if self.verbose:
            mytxt = _("switching to")
            self.Entropy.updateProgress(
                "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpdir),),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
        self.setCWD(self.ftpdir, dodir = True)

    def setBasedir(self):
        rc = self.setCWD(self.ftpdir)
        return rc

    # this can be used in case of exceptions
    def reconnectHost(self):
        # import FTP modules
        self.socket.setdefaulttimeout(60)
        counter = 10
        while 1:
            counter -= 1
            try:
                self.ftpconn = self.ftplib.FTP(self.ftphost)
                break
            except:
                if not counter:
                    raise
                continue
        if self.verbose:
            mytxt = _("reconnecting with user")
            self.Entropy.updateProgress(
                "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpuser),),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
        self.ftpconn.login(self.ftpuser,self.ftppassword)
        if self.verbose:
            mytxt = _("switching to")
            self.Entropy.updateProgress(
                "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(self.ftpdir),),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
        self.setCWD(self.currentdir)

    def getHost(self):
        return self.ftphost

    def getPort(self):
        return self.ftpport

    def getDir(self):
        return self.ftpdir

    def getCWD(self):
        pwd = self.ftpconn.pwd()
        return pwd

    def setCWD(self, mydir, dodir = False):
        try:
            return self._setCWD(mydir, dodir)
        except self.ftplib.error_perm, e:
            raise exceptionTools.FtpError('FtpError: %s' % (e,))

    def _setCWD(self, mydir, dodir = False):
        if self.verbose:
            mytxt = _("switching to")
            self.Entropy.updateProgress(
                "[ftp:%s] %s: %s" % (darkgreen(self.ftphost),mytxt,blue(mydir),),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
        try:
            self.ftpconn.cwd(mydir)
        except self.ftplib.error_perm, e:
            if e[0][:3] == '550' and dodir:
                self.recursiveMkdir(mydir)
                self.ftpconn.cwd(mydir)
            else:
                raise
        self.currentdir = self.getCWD()

    def setPASV(self,bool):
        self.ftpconn.set_pasv(bool)

    def setChmod(self,chmodvalue,file):
        return self.ftpconn.voidcmd("SITE CHMOD "+str(chmodvalue)+" "+str(file))

    def getFileMtime(self,path):
        rc = self.ftpconn.sendcmd("mdtm "+path)
        return rc.split()[-1]

    def spawnCommand(self,cmd):
        return self.ftpconn.sendcmd(cmd)

    # list files and directory of a FTP
    # @returns a list
    def listDir(self):
        # directory is: self.ftpdir
        try:
            rc = self.ftpconn.nlst()
            _rc = []
            for i in rc:
                _rc.append(i.split("/")[-1])
            rc = _rc
        except:
            return []
        return rc

    # list if the file is available
    # @returns True or False
    def isFileAvailable(self,filename):
        # directory is: self.ftpdir
        try:
            rc = self.ftpconn.nlst()
            _rc = []
            for i in rc:
                _rc.append(i.split("/")[-1])
            rc = _rc
            for i in rc:
                if i == filename:
                    return True
            return False
        except:
            return False

    def deleteFile(self,file):
        try:
            rc = self.ftpconn.delete(file)
            if rc.startswith("250"):
                return True
            else:
                return False
        except:
            return False

    def recursiveMkdir(self, mypath):
        mydirs = [x for x in mypath.split("/") if x]
        mycurpath = ""
        for mydir in mydirs:
            mycurpath = os.path.join(mycurpath,mydir)
            if not self.isFileAvailable(mycurpath):
                try:
                    self.mkdir(mycurpath)
                except self.ftplib.error_perm, e:
                    if e[0].lower().find("permission denied") != -1:
                        raise
                    elif e[0][:3] != '550':
                        raise

    def mkdir(self,directory):
        return self.ftpconn.mkd(directory)

    # this function also supports callback, because storbinary doesn't
    def advancedStorBinary(self, cmd, fp, callback=None):
        ''' Store a file in binary mode. Our version supports a callback function'''
        self.ftpconn.voidcmd('TYPE I')
        conn = self.ftpconn.transfercmd(cmd)
        while 1:
            buf = fp.readline()
            if not buf: break
            conn.sendall(buf)
            if callback: callback(buf)
        conn.close()

        # that's another workaround
        #return "226"
        try:
            rc = self.ftpconn.voidresp()
        except:
            self.reconnectHost()
            return "226"
        return rc

    def updateProgress(self, buf_len):
        # get the buffer size
        self.mykByteCount += float(buf_len)/1024
        # create percentage
        if self.myFileSize < 1:
            myUploadPercentage = 100.0
        else:
            myUploadPercentage = round((round(self.mykByteCount,1)/self.myFileSize)*100,1)
        currentprogress = myUploadPercentage
        myUploadSize = round(self.mykByteCount,1)
        if (currentprogress > self.oldprogress+0.5) and (myUploadPercentage < 100.1) and (myUploadSize <= self.myFileSize):
            myUploadPercentage = str(myUploadPercentage)+"%"

            # create text
            mytxt = _("Upload status")
            currentText = brown("    <-> %s: " % (mytxt,)) + \
                green(str(myUploadSize)) + "/" + red(str(self.myFileSize)) + " kB " + \
                brown("[") + str(myUploadPercentage) + brown("]")
            print_info(currentText, back = True)
            # XXX too slow, reimplement self.updateProgress and do whatever you want
            #self.Entropy.updateProgress(currentText, importance = 0, type = "info", back = True)
            self.oldprogress = currentprogress

    def uploadFile(self,file,ascii = False):

        self.oldprogress = 0.0

        def uploadFileAndUpdateProgress(buf):
            self.updateProgress(len(buf))

        for i in range(10): # ten tries
            filename = file.split("/")[len(file.split("/"))-1]
            try:
                f = open(file,"r")
                # get file size
                self.myFileSize = round(float(self.entropyTools.get_file_size(file))/1024,1)
                self.mykByteCount = 0

                if self.isFileAvailable(filename+".tmp"):
                    self.deleteFile(filename+".tmp")

                if (ascii):
                    rc = self.ftpconn.storlines("STOR "+filename+".tmp",f)
                else:
                    rc = self.advancedStorBinary("STOR "+filename+".tmp", f, callback = uploadFileAndUpdateProgress )

                # now we can rename the file with its original name
                self.renameFile(filename+".tmp",filename)
                f.close()

                if rc.find("226") != -1: # upload complete
                    return True
                else:
                    return False

            except Exception, e: # connection reset by peer
                import traceback
                traceback.print_exc()
                self.Entropy.updateProgress(" ", importance = 0, type = "info")
                mytxt = red("%s: %s, %s... #%s") % (
                    _("Upload issue"),
                    e,
                    _("retrying"),
                    i+1,
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = "  "
                    )
                self.reconnectHost() # reconnect
                if self.isFileAvailable(filename):
                    self.deleteFile(filename)
                if self.isFileAvailable(filename+".tmp"):
                    self.deleteFile(filename+".tmp")
                pass

    def downloadFile(self, filename, downloaddir, ascii = False):

        self.oldprogress = 0.0

        def downloadFileStoreAndUpdateProgress(buf):
            # writing file buffer
            f.write(buf)
            # update progress
            self.mykByteCount += float(len(buf))/1024
            # create text
            cnt = round(self.mykByteCount,1)
            mytxt = _("Download status")
            currentText = brown("    <-> %s: " % (mytxt,)) + green(str(cnt)) + "/" + \
                red(str(self.myFileSize)) + " kB"
            self.Entropy.updateProgress(
                currentText,
                importance = 0,
                type = "info",
                back = True,
                count = (cnt, self.myFileSize),
                percent = True
            )

        # look if the file exist
        if self.isFileAvailable(filename):
            self.mykByteCount = 0
            # get the file size
            self.myFileSize = self.getFileSizeCompat(filename)
            if (self.myFileSize):
                self.myFileSize = round(float(int(self.myFileSize))/1024,1)
                if (self.myFileSize == 0):
                    self.myFileSize = 1
            else:
                self.myFileSize = 0
            if (not ascii):
                f = open(downloaddir+"/"+filename,"wb")
                rc = self.ftpconn.retrbinary('RETR '+filename, downloadFileStoreAndUpdateProgress, 1024)
            else:
                f = open(downloaddir+"/"+filename,"w")
                rc = self.ftpconn.retrlines('RETR '+filename, f.write)
            f.flush()
            f.close()
            if rc.find("226") != -1: # upload complete
                return True
            else:
                return False
        else:
            return None

    # also used to move files
    def renameFile(self,fromfile,tofile):
        rc = self.ftpconn.rename(fromfile,tofile)
        return rc

    def getFileSize(self,file):
        return self.ftpconn.size(file)

    def getFileSizeCompat(self,file):
        data = self.getRoughList()
        for item in data:
            if item.find(file) != -1:
                return item.split()[4]
        return ""

    def bufferizer(self,buf):
        self.FTPbuffer.append(buf)

    def getRoughList(self):
        self.FTPbuffer = []
        self.ftpconn.dir(self.bufferizer)
        return self.FTPbuffer

    def closeConnection(self):
        try:
            self.ftpconn.quit()
        except (EOFError,AttributeError,self.socket.timeout,self.ftplib.error_reply,):
            # AttributeError is raised when socket gets trashed
            # EOFError is raised when the connection breaks
            # timeout, who cares!
            pass


class rssFeed:

    import entropyTools
    def __init__(self, filename, title, description, maxentries = 100):

        self.feed_title = title
        self.feed_title = self.feed_title.strip()
        self.feed_description = description
        self.feed_language = "en-EN"
        self.feed_editor = etpConst['rss-managing-editor']
        self.feed_copyright = "%s - (C) %s" % (
            etpConst['systemname'],
            self.entropyTools.getYear(),
        )

        self.file = filename
        self.items = {}
        self.itemscounter = 0
        self.maxentries = maxentries
        from xml.dom import minidom
        self.minidom = minidom

        # sanity check
        broken = False
        if os.path.isfile(self.file):
            try:
                self.xmldoc = self.minidom.parse(self.file)
            except:
                broken = True

        if not os.path.isfile(self.file) or broken:
            self.title = self.feed_title
            self.description = self.feed_description
            self.language = self.feed_language
            self.cright = self.feed_copyright
            self.editor = self.feed_editor
            self.link = etpConst['rss-website-url']
            f = open(self.file,"w")
            f.write('')
            f.close()
        else:
            self.rssdoc = self.xmldoc.getElementsByTagName("rss")[0]
            self.channel = self.rssdoc.getElementsByTagName("channel")[0]
            self.title = self.channel.getElementsByTagName("title")[0].firstChild.data.strip()
            self.link = self.channel.getElementsByTagName("link")[0].firstChild.data.strip()
            self.description = self.channel.getElementsByTagName("description")[0].firstChild.data.strip()
            try:
                self.language = self.channel.getElementsByTagName("language")[0].firstChild.data.strip()
            except IndexError:
                self.language = 'en'
            try:
                self.cright = self.channel.getElementsByTagName("copyright")[0].firstChild.data.strip()
            except IndexError:
                self.cright = ''
            try:
                self.editor = self.channel.getElementsByTagName("managingEditor")[0].firstChild.data.strip()
            except IndexError:
                self.editor = ''
            entries = self.channel.getElementsByTagName("item")
            self.itemscounter = len(entries)
            if self.itemscounter > self.maxentries:
                self.itemscounter = self.maxentries
            mycounter = self.itemscounter
            for item in entries:
                if mycounter == 0: # max entries reached
                    break
                mycounter -= 1
                self.items[mycounter] = {}
                self.items[mycounter]['title'] = item.getElementsByTagName("title")[0].firstChild.data.strip()
                description = item.getElementsByTagName("description")[0].firstChild
                if description:
                    self.items[mycounter]['description'] = description.data.strip()
                else:
                    self.items[mycounter]['description'] = ""
                link = item.getElementsByTagName("link")[0].firstChild
                if link:
                    self.items[mycounter]['link'] = link.data.strip()
                else:
                    self.items[mycounter]['link'] = ""
                self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data.strip()
                self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data.strip()
                dcs = item.getElementsByTagName("dc:creator")
                if dcs:
                    self.items[mycounter]['dc:creator'] = dcs[0].firstChild.data.strip()

    def addItem(self, title, link = '', description = '', pubDate = ''):
        self.itemscounter += 1
        self.items[self.itemscounter] = {}
        self.items[self.itemscounter]['title'] = title
        if pubDate:
            self.items[self.itemscounter]['pubDate'] = pubDate
        else:
            self.items[self.itemscounter]['pubDate'] = time.strftime("%a, %d %b %Y %X +0000")
        self.items[self.itemscounter]['description'] = description
        self.items[self.itemscounter]['link'] = link
        if link:
            self.items[self.itemscounter]['guid'] = link
        else:
            myguid = etpConst['systemname'].lower()
            myguid = myguid.replace(" ","")
            self.items[self.itemscounter]['guid'] = myguid+"~"+description+str(self.itemscounter)
        return self.itemscounter

    def removeEntry(self, id):
        if id in self.items:
            del self.items[id]
            self.itemscounter -= 1
        return self.itemscounter

    def getEntries(self):
        return self.items, self.itemscounter

    def writeChanges(self, reverse = True):

        # filter entries to fit in maxentries
        if self.itemscounter > self.maxentries:
            tobefiltered = self.itemscounter - self.maxentries
            for index in range(tobefiltered):
                try:
                    del self.items[index]
                except KeyError:
                    pass

        doc = self.minidom.Document()

        rss = doc.createElement("rss")
        rss.setAttribute("version","2.0")
        rss.setAttribute("xmlns:atom","http://www.w3.org/2005/Atom")

        channel = doc.createElement("channel")

        # title
        title = doc.createElement("title")
        title_text = doc.createTextNode(unicode(self.title))
        title.appendChild(title_text)
        channel.appendChild(title)
        # link
        link = doc.createElement("link")
        link_text = doc.createTextNode(unicode(self.link))
        link.appendChild(link_text)
        channel.appendChild(link)
        # description
        description = doc.createElement("description")
        desc_text = doc.createTextNode(unicode(self.description))
        description.appendChild(desc_text)
        channel.appendChild(description)
        # language
        language = doc.createElement("language")
        lang_text = doc.createTextNode(unicode(self.language))
        language.appendChild(lang_text)
        channel.appendChild(language)
        # copyright
        cright = doc.createElement("copyright")
        cr_text = doc.createTextNode(unicode(self.cright))
        cright.appendChild(cr_text)
        channel.appendChild(cright)
        # managingEditor
        managingEditor = doc.createElement("managingEditor")
        ed_text = doc.createTextNode(unicode(self.editor))
        managingEditor.appendChild(ed_text)
        channel.appendChild(managingEditor)

        keys = self.items.keys()
        if reverse: keys.reverse()
        for key in keys:

            # sanity check, you never know
            if not self.items.has_key(key):
                self.removeEntry(key)
                continue
            k_error = False
            for item in ['title','link','guid','description','pubDate']:
                if not self.items[key].has_key(item):
                    k_error = True
                    break
            if k_error:
                self.removeEntry(key)
                continue

            # item
            item = doc.createElement("item")
            # title
            item_title = doc.createElement("title")
            item_title_text = doc.createTextNode(unicode(self.items[key]['title']))
            item_title.appendChild(item_title_text)
            item.appendChild(item_title)
            # link
            item_link = doc.createElement("link")
            item_link_text = doc.createTextNode(unicode(self.items[key]['link']))
            item_link.appendChild(item_link_text)
            item.appendChild(item_link)
            # guid
            item_guid = doc.createElement("guid")
            item_guid.setAttribute("isPermaLink","true")
            item_guid_text = doc.createTextNode(unicode(self.items[key]['guid']))
            item_guid.appendChild(item_guid_text)
            item.appendChild(item_guid)
            # description
            item_desc = doc.createElement("description")
            item_desc_text = doc.createTextNode(unicode(self.items[key]['description']))
            item_desc.appendChild(item_desc_text)
            item.appendChild(item_desc)
            # pubdate
            item_date = doc.createElement("pubDate")
            item_date_text = doc.createTextNode(unicode(self.items[key]['pubDate']))
            item_date.appendChild(item_date_text)
            item.appendChild(item_date)

            # add item to channel
            channel.appendChild(item)

        # add channel to rss
        rss.appendChild(channel)
        doc.appendChild(rss)
        f = open(self.file,"w")
        f.writelines(doc.toprettyxml(indent="    ").encode('utf-8'))
        f.flush()
        f.close()

class TriggerInterface:

    import entropyTools
    def __init__(self, EquoInstance, phase, pkgdata, package_action = None):

        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid Entropy Instance is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        self.Entropy = EquoInstance
        self.clientLog = self.Entropy.clientLog
        self.validPhases = ("preinstall","postinstall","preremove","postremove")
        self.pkgdata = pkgdata
        self.prepared = False
        self.triggers = set()
        self.gentoo_compat = etpConst['gentoo-compat']
	self.package_action = package_action

        '''
        @ description: Gentoo toolchain variables
        '''
        self.MODULEDB_DIR="/var/lib/module-rebuild/"
        self.INITSERVICES_DIR="/var/lib/init.d/"

        ''' portage stuff '''
        if self.gentoo_compat:
            try:
                Spm = self.Entropy.Spm()
                self.Spm = Spm
            except Exception, e:
                self.entropyTools.printTraceback()
                mytxt = darkred("%s, %s: %s, %s !") % (
                    _("Portage interface can't be loaded"),
                    _("Error"),
                    e,
                    _("please fix"),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = bold(" !!! ")
                )
                self.gentoo_compat = False

        self.phase = phase
        # validate phase
        self.phaseValidation()

    def phaseValidation(self):
        if self.phase not in self.validPhases:
            mytxt = "%s: %s" % (_("Valid phases"),self.validPhases,)
            raise exceptionTools.InvalidData("InvalidData: %s" % (mytxt,))

    def prepare(self):
        self.triggers = eval("self."+self.phase)()
        remove = set()
        for trigger in self.triggers:
            if trigger in etpUi[self.phase+'_triggers_disable']:
                remove.add(trigger)
        self.triggers = [x for x in self.triggers if x not in remove]
        del remove
        self.prepared = True

    def run(self):
        for trigger in self.triggers:
            eval("self.trigger_"+trigger)()

    def kill(self):
        self.prepared = False
        del self.triggers[:]

    def postinstall(self):

        functions = []
        # Gentoo hook
        if self.gentoo_compat:
            functions.append('ebuild_postinstall')

        # equo purge cache
        if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-apps/entropy":
            functions.append("purgecache")

        # binutils configuration
        if self.pkgdata['category']+"/"+self.pkgdata['name'] == "sys-devel/binutils":
            functions.append("binutilsswitch")

        # opengl configuration
        if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")):
            if "ebuild_postinstall" in functions:
                functions.remove("ebuild_postinstall") # disabling gentoo postinstall since we reimplemented it
            functions.append("openglsetup")

        # load linker paths
        ldpaths = self.Entropy.entropyTools.collectLinkerPaths()
        for x in self.pkgdata['content']:

            if (x.startswith("/etc/conf.d") or x.startswith("/etc/init.d")) and ("conftouch" not in functions):
                functions.append('conftouch')

            if x.startswith('/lib/modules/') and ("kernelmod" not in functions):
                if "ebuild_postinstall" in functions:
                    # disabling gentoo postinstall since we reimplemented it
                    functions.remove("ebuild_postinstall")
                functions.append('kernelmod')

            if x.startswith('/boot/kernel-') and ("addbootablekernel" not in functions):
                functions.append('addbootablekernel')

            if x.startswith('/usr/src/') and ("createkernelsym" not in functions):
                functions.append('createkernelsym')

            if x.startswith('/etc/env.d/') and ("env_update" not in functions):
                functions.append('env_update')

            if (os.path.dirname(x) in ldpaths) and ("run_ldconfig" not in functions):
                if x.find(".so") > -1:
                    functions.append('run_ldconfig')

        if self.pkgdata['trigger']:
            functions.append('call_ext_postinstall')

        del ldpaths
        return functions

    def preinstall(self):

        functions = []

        # Gentoo hook
        if self.gentoo_compat:
            functions.append('ebuild_preinstall')

        for x in self.pkgdata['content']:
            if x.startswith("/etc/init.d/") and ("initinform" not in functions):
                functions.append('initinform')
            if x.startswith("/boot") and ("mountboot" not in functions):
                functions.append('mountboot')

        if self.pkgdata['trigger']:
            functions.append('call_ext_preinstall')

        return functions

    def postremove(self):

        functions = []

        # load linker paths
        ldpaths = self.Entropy.entropyTools.collectLinkerPaths()

        for x in self.pkgdata['removecontent']:
            if x.startswith('/boot/kernel-') and ("removebootablekernel" not in functions):
                functions.append('removebootablekernel')
            if x.startswith('/etc/init.d/') and ("initdisable" not in functions):
                functions.append('initdisable')
            if x.endswith('.py') and ("cleanpy" not in functions):
                functions.append('cleanpy')
            if x.startswith('/etc/env.d/') and ("env_update" not in functions):
                functions.append('env_update')
            if (os.path.dirname(x) in ldpaths) and ("run_ldconfig" not in functions):
                if x.find(".so") > -1:
                    functions.append('run_ldconfig')

        if self.pkgdata['trigger']:
            functions.append('call_ext_postremove')

        del ldpaths
        return functions


    def preremove(self):

        functions = []

        # Gentoo hook
        if self.gentoo_compat:
            functions.append('ebuild_preremove')
            functions.append('ebuild_postremove')
            # doing here because we need /var/db/pkg stuff in place and also because doesn't make any difference

        # opengl configuration
        if (self.pkgdata['category'] == "x11-drivers") and (self.pkgdata['name'].startswith("nvidia-") or self.pkgdata['name'].startswith("ati-")):
            if "ebuild_preremove" in functions:
                functions.remove("ebuild_preremove")
            if "ebuild_postremove" in functions:
                # disabling gentoo postinstall since we reimplemented it
                functions.remove("ebuild_postremove")
            if self.package_action not in ["remove_conflict"]:
                functions.append("openglsetup_xorg")

        for x in self.pkgdata['removecontent']:
            if x.startswith("/boot"):
                functions.append('mountboot')
                break

        if self.pkgdata['trigger']:
            functions.append('call_ext_preremove')

        return functions


    '''
        Real triggers
    '''
    def trigger_call_ext_preinstall(self):
        return self.trigger_call_ext_generic()

    def trigger_call_ext_postinstall(self):
        return self.trigger_call_ext_generic()

    def trigger_call_ext_preremove(self):
        return self.trigger_call_ext_generic()

    def trigger_call_ext_postremove(self):
        return self.trigger_call_ext_generic()

    def trigger_call_ext_generic(self):
        try:
            return self.do_trigger_call_ext_generic()
        except Exception, e:
            mykey = self.pkgdata['category']+"/"+self.pkgdata['name']
            self.entropyTools.printTraceback()
            self.entropyTools.printTraceback(f = self.Entropy.clientLog)
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] ATTENTION Cannot run External trigger for "+mykey+"!! "+str(Exception)+": "+str(e)
            )
            mytxt = "%s: %s %s. %s." % (
                bold(_("QA")),
                brown(_("Cannot run External trigger for")),
                bold(mykey),
                brown(_("Please report it")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            return 0

    def do_trigger_call_ext_generic(self):

        # if mute, supress portage output
        if etpUi['mute']:
            oldsystderr = sys.stderr
            oldsysstdout = sys.stdout
            stdfile = open("/dev/null","w")
            sys.stdout = stdfile
            sys.stderr = stdfile

        triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber())
        while os.path.isfile(triggerfile):
            triggerfile = etpConst['entropyunpackdir']+"/trigger-"+str(self.Entropy.entropyTools.getRandomNumber())
        triggerdir = os.path.dirname(triggerfile)
        if not os.path.isdir(triggerdir):
            os.makedirs(triggerdir)
        f = open(triggerfile,"w")
        for x in self.pkgdata['trigger']:
            f.write(x)
        f.close()

        # if mute, restore old stdout/stderr
        if etpUi['mute']:
            sys.stderr = oldsystderr
            sys.stdout = oldsysstdout
            stdfile.close()

        stage = self.phase
        pkgdata = self.pkgdata
        # since I am sick of seeing pychecker reporting this
        # let me do a nasty thing
        x = type(stage),type(pkgdata)
        del x
        my_ext_status = 0
        execfile(triggerfile)
        os.remove(triggerfile)
        return my_ext_status

    def trigger_purgecache(self):
        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "[POST] Purging Entropy cache..."
        )

        mytxt = "%s: %s." % (_("Please remember"),_("It is always better to leave Entropy updates isolated"),)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        mytxt = "%s ..." % (_("Purging Entropy cache"),)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        self.Entropy.purge_cache(False)

    def trigger_conftouch(self):
        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "[POST] Updating {conf.d,init.d} mtime..."
        )
        mytxt = "%s ..." % (_("Updating {conf.d,init.d} mtime"),)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        for item in self.pkgdata['content']:
            if not (item.startswith("/etc/conf.d") or item.startswith("/etc/conf.d")):
                continue
            if not os.path.isfile(item):
                continue
            if not os.access(item,os.W_OK):
                continue
            try:
                f = open(item,"abw")
                f.flush()
                f.close()
            except (OSError,IOError,):
                pass

    def trigger_binutilsswitch(self):
        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "[POST] Configuring Binutils Profile..."
        )
        mytxt = "%s ..." % (_("Configuring Binutils Profile"),)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        # get binutils profile
        pkgsplit = self.Entropy.entropyTools.catpkgsplit(
            self.pkgdata['category'] + "/" + self.pkgdata['name'] + "-" + self.pkgdata['version']
        )
        profile = self.pkgdata['chost']+"-"+pkgsplit[2]
        self.trigger_set_binutils_profile(profile)

    def trigger_kernelmod(self):
        if self.pkgdata['category'] != "sys-kernel":
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Updating moduledb..."
            )
            mytxt = "%s ..." % (_("Updating moduledb"),)
            self.Entropy.updateProgress(
                brown(mytxt),
                importance = 0,
                header = red("   ## ")
            )
            item = 'a:1:'+self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
            self.trigger_update_moduledb(item)
        mytxt = "%s ..." % (_("Running depmod"),)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        # get kernel modules dir name
        name = ''
        for item in self.pkgdata['content']:
            item = etpConst['systemroot']+item
            if item.startswith(etpConst['systemroot']+"/lib/modules/"):
                name = item[len(etpConst['systemroot']):]
                name = name.split("/")[3]
                break
        if name:
            self.trigger_run_depmod(name)

    def trigger_initdisable(self):
        for item in self.pkgdata['removecontent']:
            item = etpConst['systemroot']+item
            if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item):
                myroot = "/"
                if etpConst['systemroot']:
                    myroot = etpConst['systemroot']+"/"
                runlevels_dir = etpConst['systemroot']+"/etc/runlevels"
                runlevels = []
                if os.path.isdir(runlevels_dir) and os.access(runlevels_dir,os.R_OK):
                    runlevels = [x for x in os.listdir(runlevels_dir) \
                        if os.path.isdir(os.path.join(runlevels_dir,x)) \
                        and os.path.isfile(os.path.join(runlevels_dir,x,os.path.basename(item)))
                    ]
                for runlevel in runlevels:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[POST] Removing boot service: %s, runlevel: %s" % (os.path.basename(item),runlevel,)
                    )
                    mytxt = "%s: %s : %s" % (brown(_("Removing boot service")),os.path.basename(item),runlevel,)
                    self.Entropy.updateProgress(
                        mytxt,
                        importance = 0,
                        header = red("   ## ")
                    )
                    cmd = 'ROOT="%s" rc-update del %s %s' % (myroot, os.path.basename(item), runlevel)
                    subprocess.call(cmd, shell = True)

    def trigger_initinform(self):
        for item in self.pkgdata['content']:
            item = etpConst['systemroot']+item
            if item.startswith(etpConst['systemroot']+"/etc/init.d/") and not os.path.isfile(etpConst['systemroot']+item):
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "[PRE] A new service will be installed: %s" % (item,)
                )
                mytxt = "%s: %s" % (brown(_("A new service will be installed")),item,)
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )

    def trigger_openglsetup(self):
        opengl = "xorg-x11"
        if self.pkgdata['name'] == "nvidia-drivers":
            opengl = "nvidia"
        elif self.pkgdata['name'] == "ati-drivers":
            opengl = "ati"
        # is there eselect ?
        eselect = subprocess.call("eselect opengl &> /dev/null", shell = True)
        if eselect == 0:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Reconfiguring OpenGL to %s ..." % (opengl,)
            )
            mytxt = "%s ..." % (brown(_("Reconfiguring OpenGL")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            quietstring = ''
            if etpUi['quiet']: quietstring = " &>/dev/null"
            if etpConst['systemroot']:
                subprocess.call('echo "eselect opengl set --use-old %s" | chroot %s %s' % (opengl,etpConst['systemroot'],quietstring,), shell = True)
            else:
                subprocess.call('eselect opengl set --use-old %s %s' % (opengl,quietstring,), shell = True)
        else:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Eselect NOT found, cannot run OpenGL trigger"
            )
            mytxt = "%s !" % (brown(_("Eselect NOT found, cannot run OpenGL trigger")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ##")
            )

    def trigger_openglsetup_xorg(self):
        eselect = subprocess.call("eselect opengl &> /dev/null", shell = True)
        if eselect == 0:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Reconfiguring OpenGL to fallback xorg-x11 ..."
            )
            mytxt = "%s ..." % (brown(_("Reconfiguring OpenGL")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            quietstring = ''
            if etpUi['quiet']: quietstring = " &>/dev/null"
            if etpConst['systemroot']:
                subprocess.call('echo "eselect opengl set xorg-x11" | chroot %s %s' % (etpConst['systemroot'],quietstring,), shell = True)
            else:
                subprocess.call('eselect opengl set xorg-x11 %s' % (quietstring,), shell = True)
        else:
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Eselect NOT found, cannot run OpenGL trigger"
            )
            mytxt = "%s !" % (brown(_("Eselect NOT found, cannot run OpenGL trigger")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ##")
            )

    # FIXME: this only supports grub (no lilo support)
    def trigger_addbootablekernel(self):
        boot_mount = False
        if os.path.ismount("/boot"):
            boot_mount = True
        kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")]
        if boot_mount:
            kernels = [x[len("/boot"):] for x in kernels]
        for kernel in kernels:
            mykernel = kernel.split('/kernel-')[1]
            initramfs = "/boot/initramfs-"+mykernel
            if initramfs not in self.pkgdata['content']:
                initramfs = ''
            elif boot_mount:
                initramfs = initramfs[len("/boot"):]

            # configure GRUB
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Configuring GRUB bootloader. Adding the new kernel..."
            )
            mytxt = "%s. %s ..." % (
                _("Configuring GRUB bootloader"),
                _("Adding the new kernel"),
            )
            self.Entropy.updateProgress(
                brown(mytxt),
                importance = 0,
                header = red("   ## ")
            )
            self.trigger_configure_boot_grub(kernel,initramfs)

    # FIXME: this only supports grub (no lilo support)
    def trigger_removebootablekernel(self):
        kernels = [x for x in self.pkgdata['content'] if x.startswith("/boot/kernel-")]
        for kernel in kernels:
            initramfs = "/boot/initramfs-"+kernel[13:]
            if initramfs not in self.pkgdata['content']:
                initramfs = ''
            # configure GRUB
            self.Entropy.clientLog.log(
                ETP_LOGPRI_INFO,
                ETP_LOGLEVEL_NORMAL,
                "[POST] Configuring GRUB bootloader. Removing the selected kernel..."
            )
            mytxt = "%s. %s ..." % (
                _("Configuring GRUB bootloader"),
                _("Removing the selected kernel"),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            self.trigger_remove_boot_grub(kernel,initramfs)

    def trigger_mountboot(self):
        # is in fstab?
        if etpConst['systemroot']:
            return
        if os.path.isfile("/etc/fstab"):
            f = open("/etc/fstab","r")
            fstab = f.readlines()
            fstab = self.Entropy.entropyTools.listToUtf8(fstab)
            f.close()
            for line in fstab:
                fsline = line.split()
                if len(fsline) > 1:
                    if fsline[1] == "/boot":
                        if not os.path.ismount("/boot"):
                            # trigger mount /boot
                            rc = subprocess.call("mount /boot &> /dev/null", shell = True)
                            if rc == 0:
                                self.Entropy.clientLog.log(
                                    ETP_LOGPRI_INFO,
                                    ETP_LOGLEVEL_NORMAL,
                                    "[PRE] Mounted /boot successfully"
                                )
                                self.Entropy.updateProgress(
                                    brown(_("Mounted /boot successfully")),
                                    importance = 0,
                                    header = red("   ## ")
                                )
                            elif rc != 8192: # already mounted
                                self.Entropy.clientLog.log(
                                    ETP_LOGPRI_INFO,
                                    ETP_LOGLEVEL_NORMAL,
                                    "[PRE] Cannot mount /boot automatically !!"
                                )
                                self.Entropy.updateProgress(
                                    brown(_("Cannot mount /boot automatically !!")),
                                    importance = 0,
                                    header = red("   ## ")
                                )
                            break

    def trigger_cleanpy(self):
        pyfiles = [x for x in self.pkgdata['content'] if x.endswith(".py")]
        for item in pyfiles:
            item = etpConst['systemroot']+item
            if os.path.isfile(item+"o"):
                try: os.remove(item+"o")
                except OSError: pass
            if os.path.isfile(item+"c"):
                try: os.remove(item+"c")
                except OSError: pass

    def trigger_createkernelsym(self):
        for item in self.pkgdata['content']:
            item = etpConst['systemroot']+item
            if item.startswith(etpConst['systemroot']+"/usr/src/"):
                # extract directory
                try:
                    todir = item[len(etpConst['systemroot']):]
                    todir = todir.split("/")[3]
                except:
                    continue
                if os.path.isdir(etpConst['systemroot']+"/usr/src/"+todir):
                    # link to /usr/src/linux
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[POST] Creating kernel symlink "+etpConst['systemroot']+"/usr/src/linux for /usr/src/"+todir
                    )
                    mytxt = "%s %s %s %s" % (
                        _("Creating kernel symlink"),
                        etpConst['systemroot']+"/usr/src/linux",
                        _("for"),
                        "/usr/src/"+todir,
                    )
                    self.Entropy.updateProgress(
                        brown(mytxt),
                        importance = 0,
                        header = red("   ## ")
                    )
                    if os.path.isfile(etpConst['systemroot']+"/usr/src/linux") or \
                        os.path.islink(etpConst['systemroot']+"/usr/src/linux"):
                            os.remove(etpConst['systemroot']+"/usr/src/linux")
                    if os.path.isdir(etpConst['systemroot']+"/usr/src/linux"):
                        mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber())
                        while os.path.isdir(mydir):
                            mydir = etpConst['systemroot']+"/usr/src/linux."+str(self.Entropy.entropyTools.getRandomNumber())
                        shutil.move(etpConst['systemroot']+"/usr/src/linux",mydir)
                    try:
                        os.symlink(todir,etpConst['systemroot']+"/usr/src/linux")
                    except OSError: # not important in the end
                        pass
                    break

    def trigger_run_ldconfig(self):
        if not etpConst['systemroot']:
            myroot = "/"
        else:
            myroot = etpConst['systemroot']+"/"
        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "[POST] Running ldconfig"
        )
        mytxt = "%s %s" % (_("Regenerating"),"/etc/ld.so.cache",)
        self.Entropy.updateProgress(
            brown(mytxt),
            importance = 0,
            header = red("   ## ")
        )
        subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True)

    def trigger_env_update(self):

        self.Entropy.clientLog.log(
            ETP_LOGPRI_INFO,
            ETP_LOGLEVEL_NORMAL,
            "[POST] Running env-update"
        )
        if os.access(etpConst['systemroot']+"/usr/sbin/env-update",os.X_OK):
            mytxt = "%s ..." % (_("Updating environment"),)
            self.Entropy.updateProgress(
                brown(mytxt),
                importance = 0,
                header = red("   ## ")
            )
            if etpConst['systemroot']:
                subprocess.call("echo 'env-update --no-ldconfig' | chroot %s &> /dev/null" % (etpConst['systemroot'],), shell = True)
            else:
                subprocess.call('env-update --no-ldconfig &> /dev/null', shell = True)

    def trigger_ebuild_postinstall(self):
        stdfile = open("/dev/null","w")
        oldstderr = sys.stderr
        oldstdout = sys.stdout
        sys.stderr = stdfile

        myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(".ebuild")]
        if myebuild:
            myebuild = myebuild[0]
            portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
            self.Entropy.updateProgress(
                brown("Ebuild: pkg_postinst()"),
                importance = 0,
                header = red("   ## ")
            )
            try:

                if not os.path.isfile(self.pkgdata['unpackdir']+"/portage/"+portage_atom+"/temp/environment"):
                    # if environment is not yet created, we need to run pkg_setup()
                    sys.stdout = stdfile
                    rc = self.Spm.spm_doebuild(
                        myebuild,
                        mydo = "setup",
                        tree = "bintree",
                        cpv = portage_atom,
                        portage_tmpdir = self.pkgdata['unpackdir'],
                        licenses = self.pkgdata['accept_license']
                    )
                    if rc == 1:
                        self.Entropy.clientLog.log(
                            ETP_LOGPRI_INFO,
                            ETP_LOGLEVEL_NORMAL,
                            "[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_setup())"
                            " trigger for "+str(portage_atom)+". Something bad happened."
                        )
                    sys.stdout = oldstdout

                rc = self.Spm.spm_doebuild(
                    myebuild,
                    mydo = "postinst",
                    tree = "bintree",
                    cpv = portage_atom,
                    portage_tmpdir = self.pkgdata['unpackdir'],
                    licenses = self.pkgdata['accept_license']
                )
                if rc == 1:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[POST] ATTENTION Cannot properly run Gentoo postinstall (pkg_postinst()) trigger for " + \
                        str(portage_atom) + ". Something bad happened."
                        )

            except Exception, e:
                sys.stdout = oldstdout
                self.entropyTools.printTraceback()
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "[POST] ATTENTION Cannot run Portage trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e)
                )
                mytxt = "%s: %s %s. %s. %s: %s" % (
                    bold(_("QA")),
                    brown(_("Cannot run Portage trigger for")),
                    bold(str(portage_atom)),
                    brown(_("Please report it")),
                    bold(_("Attach this")),
                    darkred(etpConst['spmlogfile']),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
        sys.stderr = oldstderr
        sys.stdout = oldstdout
        stdfile.close()
        return 0

    def trigger_ebuild_preinstall(self):
        stdfile = open("/dev/null","w")
        oldstderr = sys.stderr
        oldstdout = sys.stdout
        sys.stderr = stdfile

        myebuild = [self.pkgdata['xpakdir']+"/"+x for x in os.listdir(self.pkgdata['xpakdir']) if x.endswith(".ebuild")]
        if myebuild:
            myebuild = myebuild[0]
            portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
            self.Entropy.updateProgress(
                brown(" Ebuild: pkg_preinst()"),
                importance = 0,
                header = red("   ##")
            )
            try:
                sys.stdout = stdfile
                rc = self.Spm.spm_doebuild(
                    myebuild,
                    mydo = "setup",
                    tree = "bintree",
                    cpv = portage_atom,
                    portage_tmpdir = self.pkgdata['unpackdir'],
                    licenses = self.pkgdata['accept_license']
                ) # create mysettings["T"]+"/environment"
                if rc == 1:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[PRE] ATTENTION Cannot properly run Portage preinstall (pkg_setup()) trigger for " + \
                        str(portage_atom) + ". Something bad happened."
                    )
                sys.stdout = oldstdout
                rc = self.Spm.spm_doebuild(
                    myebuild,
                    mydo = "preinst",
                    tree = "bintree",
                    cpv = portage_atom,
                    portage_tmpdir = self.pkgdata['unpackdir'],
                    licenses = self.pkgdata['accept_license']
                )
                if rc == 1:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[PRE] ATTENTION Cannot properly run Gentoo preinstall (pkg_preinst()) trigger for " + \
                        str(portage_atom)+". Something bad happened."
                    )
            except Exception, e:
                sys.stdout = oldstdout
                self.entropyTools.printTraceback()
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "[PRE] ATTENTION Cannot run Gentoo preinst trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e)
                )
                mytxt = "%s: %s %s. %s. %s: %s" % (
                    bold(_("QA")),
                    brown(_("Cannot run Portage trigger for")),
                    bold(str(portage_atom)),
                    brown(_("Please report it")),
                    bold(_("Attach this")),
                    darkred(etpConst['spmlogfile']),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
        sys.stderr = oldstderr
        sys.stdout = oldstdout
        stdfile.close()
        return 0

    def trigger_ebuild_preremove(self):
        stdfile = open("/dev/null","w")
        oldstderr = sys.stderr
        sys.stderr = stdfile

        portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
        try:
            myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild"
        except:
            myebuild = ''

        self.myebuild_moved = None
        if os.path.isfile(myebuild):
            try:
                myebuild = self._setup_remove_ebuild_environment(myebuild, portage_atom)
            except EOFError, e:
                sys.stderr = oldstderr
                stdfile.close()
                # stuff on system is broken, ignore it
                self.Entropy.updateProgress(
                    darkred("!!! Ebuild: pkg_prerm() failed, EOFError: ")+str(e)+darkred(" - ignoring"),
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )
                return 0
            except ImportError, e:
                sys.stderr = oldstderr
                stdfile.close()
                # stuff on system is broken, ignore it
                self.Entropy.updateProgress(
                    darkred("!!! Ebuild: pkg_prerm() failed, ImportError: ")+str(e)+darkred(" - ignoring"),
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )
                return 0

        if os.path.isfile(myebuild):

            self.Entropy.updateProgress(
                                    brown(" Ebuild: pkg_prerm()"),
                                    importance = 0,
                                    header = red("   ##")
                                )
            try:
                rc = self.Spm.spm_doebuild(
                    myebuild,
                    mydo = "prerm",
                    tree = "bintree",
                    cpv = portage_atom,
                    portage_tmpdir = etpConst['entropyunpackdir'] + "/" + portage_atom
                )
                if rc == 1:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[PRE] ATTENTION Cannot properly run Portage trigger for " + \
                        str(portage_atom)+". Something bad happened."
                    )
            except Exception, e:
                sys.stderr = oldstderr
                stdfile.close()
                self.entropyTools.printTraceback()
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "[PRE] ATTENTION Cannot run Portage preremove trigger for "+portage_atom+"!! "+str(Exception)+": "+str(e)
                )
                mytxt = "%s: %s %s. %s. %s: %s" % (
                    bold(_("QA")),
                    brown(_("Cannot run Portage trigger for")),
                    bold(str(portage_atom)),
                    brown(_("Please report it")),
                    bold(_("Attach this")),
                    darkred(etpConst['spmlogfile']),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
                return 0

        sys.stderr = oldstderr
        stdfile.close()
        self._remove_overlayed_ebuild()
        return 0

    def trigger_ebuild_postremove(self):
        stdfile = open("/dev/null","w")
        oldstderr = sys.stderr
        sys.stderr = stdfile

        portage_atom = self.pkgdata['category']+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']
        try:
            myebuild = self.Spm.get_vdb_path()+portage_atom+"/"+self.pkgdata['name']+"-"+self.pkgdata['version']+".ebuild"
        except:
            myebuild = ''

        self.myebuild_moved = None
        if os.path.isfile(myebuild):
            try:
                myebuild = self._setup_remove_ebuild_environment(myebuild, portage_atom)
            except EOFError, e:
                sys.stderr = oldstderr
                stdfile.close()
                # stuff on system is broken, ignore it
                self.Entropy.updateProgress(
                    darkred("!!! Ebuild: pkg_postrm() failed, EOFError: ")+str(e)+darkred(" - ignoring"),
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )
                return 0
            except ImportError, e:
                sys.stderr = oldstderr
                stdfile.close()
                # stuff on system is broken, ignore it
                self.Entropy.updateProgress(
                    darkred("!!! Ebuild: pkg_postrm() failed, ImportError: ")+str(e)+darkred(" - ignoring"),
                    importance = 1,
                    type = "warning",
                    header = red("   ## ")
                )
                return 0

        if os.path.isfile(myebuild):
            self.Entropy.updateProgress(
                                    brown(" Ebuild: pkg_postrm()"),
                                    importance = 0,
                                    header = red("   ##")
                                )
            try:
                rc = self.Spm.spm_doebuild(
                    myebuild,
                    mydo = "postrm",
                    tree = "bintree",
                    cpv = portage_atom,
                    portage_tmpdir = etpConst['entropyunpackdir']+"/"+portage_atom
                )
                if rc == 1:
                    self.Entropy.clientLog.log(
                        ETP_LOGPRI_INFO,
                        ETP_LOGLEVEL_NORMAL,
                        "[PRE] ATTENTION Cannot properly run Gentoo postremove trigger for " + \
                        str(portage_atom)+". Something bad happened."
                    )
            except Exception, e:
                sys.stderr = oldstderr
                stdfile.close()
                self.entropyTools.printTraceback()
                self.Entropy.clientLog.log(
                    ETP_LOGPRI_INFO,
                    ETP_LOGLEVEL_NORMAL,
                    "[PRE] ATTENTION Cannot run Gentoo postremove trigger for " + \
                    portage_atom+"!! "+str(Exception)+": "+str(e)
                )
                mytxt = "%s: %s %s. %s. %s: %s" % (
                    bold(_("QA")),
                    brown(_("Cannot run Portage trigger for")),
                    bold(str(portage_atom)),
                    brown(_("Please report it")),
                    bold(_("Attach this")),
                    darkred(etpConst['spmlogfile']),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
                return 0

        sys.stderr = oldstderr
        stdfile.close()
        self._remove_overlayed_ebuild()
        return 0

    def _setup_remove_ebuild_environment(self, myebuild, portage_atom):

        ebuild_dir = os.path.dirname(myebuild)
        ebuild_file = os.path.basename(myebuild)

        # copy the whole directory in a safe place
        dest_dir = os.path.join(etpConst['entropyunpackdir'],"vardb/"+portage_atom)
        if os.path.exists(dest_dir):
            if os.path.isdir(dest_dir):
                shutil.rmtree(dest_dir,True)
            elif os.path.isfile(dest_dir) or os.path.islink(dest_dir):
                os.remove(dest_dir)
        os.makedirs(dest_dir)
        items = os.listdir(ebuild_dir)
        for item in items:
            myfrom = os.path.join(ebuild_dir,item)
            myto = os.path.join(dest_dir,item)
            shutil.copy2(myfrom,myto)

        newmyebuild = os.path.join(dest_dir,ebuild_file)
        if os.path.isfile(newmyebuild):
            myebuild = newmyebuild
            self.myebuild_moved = myebuild
            self._ebuild_env_setup_hook(myebuild)
        return myebuild

    def _ebuild_env_setup_hook(self, myebuild):
        ebuild_path = os.path.dirname(myebuild)
        if not etpConst['systemroot']:
            myroot = "/"
        else:
            myroot = etpConst['systemroot']+"/"

        # we need to fix ROOT= if it's set inside environment
        bz2envfile = os.path.join(ebuild_path,"environment.bz2")
        if os.path.isfile(bz2envfile) and os.path.isdir(myroot):
            import bz2
            envfile = self.Entropy.entropyTools.unpackBzip2(bz2envfile)
            bzf = bz2.BZ2File(bz2envfile,"w")
            f = open(envfile,"r")
            line = f.readline()
            while line:
                if line.startswith("ROOT="):
                    line = "ROOT=%s\n" % (myroot,)
                bzf.write(line)
                line = f.readline()
            f.close()
            bzf.close()
            os.remove(envfile)

    def _remove_overlayed_ebuild(self):
        if not self.myebuild_moved:
            return

        if os.path.isfile(self.myebuild_moved):
            mydir = os.path.dirname(self.myebuild_moved)
            shutil.rmtree(mydir,True)
            mydir = os.path.dirname(mydir)
            content = os.listdir(mydir)
            while not content:
                os.rmdir(mydir)
                mydir = os.path.dirname(mydir)
                content = os.listdir(mydir)

    '''
        Internal ones
    '''

    '''
    @description: set chosen gcc profile
    @output: returns int() as exit status
    '''
    def trigger_set_gcc_profile(self, profile):
        if os.access(etpConst['systemroot']+'/usr/bin/gcc-config',os.X_OK):
            redirect = ""
            if etpUi['quiet']:
                redirect = " &> /dev/null"
            if etpConst['systemroot']:
                subprocess.call("echo '/usr/bin/gcc-config %s' | chroot %s %s" % (profile,etpConst['systemroot'],redirect,), shell = True)
            else:
                subprocess.call('/usr/bin/gcc-config %s %s' % (profile,redirect,), shell = True)
        return 0

    '''
    @description: set chosen binutils profile
    @output: returns int() as exit status
    '''
    def trigger_set_binutils_profile(self, profile):
        if os.access(etpConst['systemroot']+'/usr/bin/binutils-config',os.X_OK):
            redirect = ""
            if etpUi['quiet']:
                redirect = " &> /dev/null"
            if etpConst['systemroot']:
                subprocess.call("echo '/usr/bin/binutils-config %s' | chroot %s %s" % (profile,etpConst['systemroot'],redirect,), shell = True)
            else:
                subprocess.call('/usr/bin/binutils-config %s %s' % (profile,redirect,), shell = True)
        return 0

    '''
    @description: updates moduledb
    @output: returns int() as exit status
    '''
    def trigger_update_moduledb(self, item):
        if os.access(etpConst['systemroot']+'/usr/sbin/module-rebuild',os.X_OK):
            if os.path.isfile(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb'):
                f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"r")
                moduledb = f.readlines()
                moduledb = self.Entropy.entropyTools.listToUtf8(moduledb)
                f.close()
                avail = [x for x in moduledb if x.strip() == item]
                if (not avail):
                    f = open(etpConst['systemroot']+self.MODULEDB_DIR+'moduledb',"aw")
                    f.write(item+"\n")
                    f.flush()
                    f.close()
        return 0

    '''
    @description: insert kernel object into kernel modules db
    @output: returns int() as exit status
    '''
    def trigger_run_depmod(self, name):
        if os.access('/sbin/depmod',os.X_OK):
            if not etpConst['systemroot']:
                myroot = "/"
            else:
                myroot = etpConst['systemroot']+"/"
            subprocess.call('/sbin/depmod -a -b %s -r %s &> /dev/null' % (myroot,name,), shell = True)
        return 0

    def __get_entropy_kernel_grub_line(self, kernel):
        return "title="+etpConst['systemname']+" ("+os.path.basename(kernel)+")\n"

    '''
    @description: append kernel entry to grub.conf
    @output: returns int() as exit status
    '''
    def trigger_configure_boot_grub(self, kernel,initramfs):

        if not os.path.isdir(etpConst['systemroot']+"/boot/grub"):
            os.makedirs(etpConst['systemroot']+"/boot/grub")
        if os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"):
            # open in append
            grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","aw")
            shutil.copy2(etpConst['systemroot']+"/boot/grub/grub.conf",etpConst['systemroot']+"/boot/grub/grub.conf.old.add")
            # get boot dev
            boot_dev = self.trigger_get_grub_boot_dev()
            # test if entry has been already added
            grubtest = open(etpConst['systemroot']+"/boot/grub/grub.conf","r")
            content = grubtest.readlines()
            content = [unicode(x,'raw_unicode_escape') for x in content]
            for line in content:
                if line.find(self.__get_entropy_kernel_grub_line(kernel)) != -1:
                    grubtest.close()
                    return
                # also check if we have the same kernel listed
                if (line.find("kernel") != 1) and (line.find(os.path.basename(kernel)) != -1) and not line.strip().startswith("#"):
                    grubtest.close()
                    return
        else:
            # create
            boot_dev = "(hd0,0)"
            grub = open(etpConst['systemroot']+"/boot/grub/grub.conf","w")
            # write header - guess (hd0,0)... since it is weird having a running system without a bootloader, at least, grub.
            grub_header = '''
default=0
timeout=10
            '''
            grub.write(grub_header)
        cmdline = ' '
        if os.path.isfile("/proc/cmdline"):
            f = open("/proc/cmdline","r")
            cmdline = " "+f.readline().strip()
            params = cmdline.split()
            if "dolvm" not in params: # support new kernels >= 2.6.23
                cmdline += " dolvm "
            f.close()
        grub.write(self.__get_entropy_kernel_grub_line(kernel))
        grub.write("\troot "+boot_dev+"\n")
        grub.write("\tkernel "+kernel+cmdline+"\n")
        if initramfs:
            grub.write("\tinitrd "+initramfs+"\n")
        grub.write("\n")
        grub.flush()
        grub.close()

    def trigger_remove_boot_grub(self, kernel,initramfs):
        if os.path.isdir(etpConst['systemroot']+"/boot/grub") and os.path.isfile(etpConst['systemroot']+"/boot/grub/grub.conf"):
            shutil.copy2(etpConst['systemroot']+"/boot/grub/grub.conf",etpConst['systemroot']+"/boot/grub/grub.conf.old.remove")
            f = open(etpConst['systemroot']+"/boot/grub/grub.conf","r")
            grub_conf = f.readlines()
            f.close()
            content = [unicode(x,'raw_unicode_escape') for x in grub_conf]
            try:
                kernel, initramfs = (unicode(kernel,'raw_unicode_escape'),unicode(initramfs,'raw_unicode_escape'))
            except TypeError:
                pass
            #kernelname = os.path.basename(kernel)
            new_conf = []
            skip = False
            for line in content:

                if (line.find(self.__get_entropy_kernel_grub_line(kernel)) != -1):
                    skip = True
                    continue

                if line.strip().startswith("title"):
                    skip = False

                if not skip or line.strip().startswith("#"):
                    new_conf.append(line)

            f = open(etpConst['systemroot']+"/boot/grub/grub.conf","w")
            for line in new_conf:
                try:
                    f.write(line)
                except UnicodeEncodeError:
                    f.write(line.encode('utf-8'))
            f.flush()
            f.close()

    def trigger_get_grub_boot_dev(self):
        if etpConst['systemroot']:
            return "(hd0,0)"
        import re
        df_avail = subprocess.call("which df &> /dev/null", shell = True)
        if df_avail != 0:
            mytxt = "%s: %s! %s. %s (hd0,0)." % (
                bold(_("QA")),
                brown(_("Cannot find df")),
                brown(_("Cannot properly configure the kernel")),
                brown(_("Defaulting to")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            return "(hd0,0)"
        grub_avail = subprocess.call("which grub &> /dev/null", shell = True)
        if grub_avail != 0:
            mytxt = "%s: %s! %s. %s (hd0,0)." % (
                bold(_("QA")),
                brown(_("Cannot find grub")),
                brown(_("Cannot properly configure the kernel")),
                brown(_("Defaulting to")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            return "(hd0,0)"

        gboot = commands.getoutput("df /boot").split("\n")[-1].split()[0]
        if gboot.startswith("/dev/"):
            # it's ok - handle /dev/md
            if gboot.startswith("/dev/md"):
                md = os.path.basename(gboot)
                if not md.startswith("md"):
                    md = "md"+md
                f = open("/proc/mdstat","r")
                mdstat = f.readlines()
                mdstat = [x for x in mdstat if x.startswith(md)]
                f.close()
                if mdstat:
                    mdstat = mdstat[0].strip().split()
                    mddevs = []
                    for x in mdstat:
                        if x.startswith("sd"):
                            mddevs.append(x[:-3])
                    mddevs = sorted(mddevs)
                    if mddevs:
                        gboot = "/dev/"+mddevs[0]
                    else:
                        gboot = "/dev/sda1"
                else:
                    gboot = "/dev/sda1"
            # get disk
            match = re.subn("[0-9]","",gboot)
            gdisk = match[0]
            if gdisk == '':

                mytxt = "%s: %s %s %s. %s! %s (hd0,0)." % (
                    bold(_("QA")),
                    brown(_("cannot match device")),
                    brown(str(gboot)),
                    brown(_("with a grub one")), # 'cannot match device /dev/foo with a grub one'
                    brown(_("Cannot properly configure the kernel")),
                    brown(_("Defaulting to")),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
                return "(hd0,0)"
            match = re.subn("[a-z/]","",gboot)
            try:
                gpartnum = str(int(match[0])-1)
            except ValueError:
                mytxt = "%s: %s: %s. %s. %s (hd0,0)." % (
                    bold(_("QA")),
                    brown(_("grub translation not supported for")),
                    brown(str(gboot)),
                    brown(_("Cannot properly configure grub.conf")),
                    brown(_("Defaulting to")),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
                return "(hd0,0)"
            # now match with grub
            device_map = etpConst['packagestmpdir']+"/grub.map"
            if os.path.isfile(device_map):
                os.remove(device_map)
            # generate device.map
            subprocess.call('echo "quit" | grub --device-map="%s" --no-floppy --batch &> /dev/null' % (device_map,), shell = True)
            if os.path.isfile(device_map):
                f = open(device_map,"r")
                device_map_file = f.readlines()
                f.close()
                grub_dev = [x for x in device_map_file if (x.find(gdisk) != -1)]
                if grub_dev:
                    grub_disk = grub_dev[0].strip().split()[0]
                    grub_dev = grub_disk[:-1]+","+gpartnum+")"
                    return grub_dev
                else:
                    mytxt = "%s: %s. %s! %s (hd0,0)." % (
                        bold(_("QA")),
                        brown(_("cannot match grub device with a Linux one")),
                        brown(_("Cannot properly configure the kernel")),
                        brown(_("Defaulting to")),
                    )
                    self.Entropy.updateProgress(
                        mytxt,
                        importance = 0,
                        header = red("   ## ")
                    )
                    return "(hd0,0)"
            else:
                mytxt = "%s: %s. %s! %s (hd0,0)." % (
                    bold(_("QA")),
                    brown(_("cannot find generated device.map")),
                    brown(_("Cannot properly configure the kernel")),
                    brown(_("Defaulting to")),
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 0,
                    header = red("   ## ")
                )
                return "(hd0,0)"
        else:
            mytxt = "%s: %s. %s! %s (hd0,0)." % (
                bold(_("QA")),
                brown(_("cannot run df /boot")),
                brown(_("Cannot properly configure the kernel")),
                brown(_("Defaulting to")),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 0,
                header = red("   ## ")
            )
            return "(hd0,0)"

class SystemSettings:

    import entropyTools
    def __init__(self, EquoInstance):

        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid Equo instance or subclass is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
        self.Entropy = EquoInstance

        self.etpSettingFiles = {
            'keywords': etpConst['confpackagesdir']+"/package.keywords", # keywording configuration files
            'unmask': etpConst['confpackagesdir']+"/package.unmask", # unmasking configuration files
            'mask': etpConst['confpackagesdir']+"/package.mask", # masking configuration files
            'license_mask': etpConst['confpackagesdir']+"/license.mask", # masking configuration files
            'repos_system_mask': {},
            'system_mask': etpConst['confpackagesdir']+"/system.mask",
            'repos_mask': {},
            'repos_license_whitelist': {},
            'system_package_sets': {},
            'conflicting_tagged_packages': {},
            'system_dirs': etpConst['confdir']+"/fsdirs.conf",
            'system_dirs_mask': etpConst['confdir']+"/fsdirsmask.conf",
        }

        ## XXX trunk support, for a while - exp. date 10/10/2009
        trunk_fsdirs_conf = "../conf/fsdirs.conf"
        trunk_fsdirsmask_conf = "../conf/fsdirsmask.conf"
        if os.path.isfile(trunk_fsdirs_conf):
            self.etpSettingFiles['system_dirs'] = trunk_fsdirs_conf
        if os.path.isfile(trunk_fsdirsmask_conf):
            self.etpSettingFiles['system_dirs_mask'] = trunk_fsdirsmask_conf

        self.etpMtimeFiles = {
            'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime",
            'unmask_mtime': etpConst['dumpstoragedir']+"/unmask.mtime",
            'mask_mtime': etpConst['dumpstoragedir']+"/mask.mtime",
            'license_mask_mtime': etpConst['dumpstoragedir']+"/license_mask.mtime",
            'system_mask_mtime': etpConst['dumpstoragedir']+"/system_mask.mtime",
            'repos_system_mask': {},
            'repos_mask': {},
            'repos_license_whitelist': {},
        }

        self.__settings = None
        self.__persistent_settings = {
            'live_packagemasking': {
                'unmask_matches': set(),
                'mask_matches': set(),
            },
            'pkg_masking_reasons': {
                0: _('reason not available'),
                1: _('user package.mask'),
                2: _('system keywords'),
                3: _('user package.unmask'),
                4: _('user repo package.keywords (all packages)'),
                5: _('user repo package.keywords'),
                6: _('user package.keywords'),
                7: _('completely masked'),
                8: _('repository general packages.db.mask'),
                9: _('repository in branch packages.db.mask'), # FIXME: this has been removed
                10: _('user license.mask'),
                11: _('user live unmask'),
                12: _('user live mask'),
            },
            'pkg_masking_reference': {
                'reason_not_avail': 0,
                'user_package_mask': 1,
                'system_keyword': 2,
                'user_package_unmask': 3,
                'user_repo_package_keywords_all': 4,
                'user_repo_package_keywords': 5,
                'user_package_keywords': 6,
                'completely_masked': 7,
                'repository_packages_db_mask': 8,
                'repository_in_branch_pacakges_db_mask': 9,
                'user_license_mask': 10,
                'user_live_unmask': 11,
                'user_live_mask': 12,
            },
        }


    def clear(self):
        self.__settings = None

    def keys(self):
        return self.__settings.keys()

    def scan(self):

        self.__settings = self.parse()
        # merge universal keywords
        for x in self.__settings['keywords']['universal']:
            etpConst['keywords'].add(x)

        # match installed packages of system_mask
        self.__settings['repos_system_mask_installed'] = []
        self.__settings['repos_system_mask_installed_keys'] = {}
        if isinstance(self.Entropy.clientDbconn,EntropyDatabaseInterface):
            while 1:
                try:
                    self.Entropy.clientDbconn.validateDatabase()
                except exceptionTools.SystemDatabaseError:
                    break
                mc_cache = set()
                m_list = self.__settings['repos_system_mask']+self.__settings['system_mask']
                for atom in m_list:
                    m_ids,m_r = self.Entropy.clientDbconn.atomMatch(atom, multiMatch = True)
                    if m_r != 0: continue
                    mykey = self.entropyTools.dep_getkey(atom)
                    if mykey not in self.__settings['repos_system_mask_installed_keys']:
                        self.__settings['repos_system_mask_installed_keys'][mykey] = set()
                    for xm in m_ids:
                        if xm in mc_cache: continue
                        mc_cache.add(xm)
                        self.__settings['repos_system_mask_installed'].append(xm)
                        self.__settings['repos_system_mask_installed_keys'][mykey].add(xm)
                break

        # Live package masking
        self.__settings.update(self.__persistent_settings)

    def __setitem__(self, mykey, myvalue):
        if self.__settings == None: self.scan()
        if mykey in self.__persistent_settings: # backup here too
            self.__persistent_settings[mykey] = myvalue
            self.__settings.update(self.__persistent_settings)
        self.__settings[mykey] = myvalue

    def __getitem__(self, mykey):
        if self.__settings == None: self.scan()
        if mykey in self.__persistent_settings: # backup here too
            return self.__persistent_settings[mykey]
        return self.__settings[mykey]

    def __contains__(self, mykey):
        if self.__settings == None: self.scan()
        here = mykey in self.__persistent_settings
        if here: return here
        return mykey in self.__settings

    def __cmp__(self, other):
        if self.__settings == None: self.scan()
        return cmp(self.__settings,other)

    def get(self, mykey):
        if self.__settings == None: self.scan()
        if mykey in self.__persistent_settings:
            return self.__persistent_settings.get(mykey)
        return self.__settings.get(mykey)

    def has_key(self, mykey):
        if self.__settings == None: self.scan()
        here = mykey in self.__persistent_settings
        if here: return True
        return mykey in self.__settings

    def parse(self):

        # append repositories mask files
        # append repositories mtime files
        for repoid in etpRepositoriesOrder:
            maskpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasemaskfile'])
            wlpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabaselicwhitelistfile'])
            sm_path = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasesytemmaskfile'])
            ct_path = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabaseconflictingtaggedfile'])
            if os.path.isfile(maskpath) and os.access(maskpath,os.R_OK):
                self.etpSettingFiles['repos_mask'][repoid] = maskpath
                self.etpMtimeFiles['repos_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasemaskfile']+".mtime"
            if os.path.isfile(wlpath) and os.access(wlpath,os.R_OK):
                self.etpSettingFiles['repos_license_whitelist'][repoid] = wlpath
                self.etpMtimeFiles['repos_license_whitelist'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabaselicwhitelistfile']+".mtime"
            if os.path.isfile(sm_path) and os.access(sm_path,os.R_OK):
                self.etpSettingFiles['repos_system_mask'][repoid] = sm_path
                self.etpMtimeFiles['repos_system_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasesytemmaskfile']+".mtime"
            if os.path.isfile(ct_path) and os.access(ct_path,os.R_OK):
                self.etpSettingFiles['conflicting_tagged_packages'][repoid] = ct_path

        # user defined package sets
        sets_dir = etpConst['confsetsdir']
        if (os.path.isdir(sets_dir) and os.access(sets_dir,os.R_OK)):
            set_files = [x for x in os.listdir(sets_dir) if (os.path.isfile(os.path.join(sets_dir,x)) and os.access(os.path.join(sets_dir,x),os.R_OK))]
            for set_file in set_files:
                try:
                    set_file = str(set_file)
                except (UnicodeDecodeError,UnicodeEncodeError,):
                    continue
                self.etpSettingFiles['system_package_sets'][set_file] = os.path.join(sets_dir,set_file)

        data = {}
        for item in self.etpSettingFiles:
            myattr = '%s_parser' % (item,)
            if not hasattr(self,myattr): continue
            f = getattr(self,myattr)
            data[item] = f()
        return data


    '''
    parser of package.keywords file
    '''
    def keywords_parser(self):

        data = {
                'universal': set(),
                'packages': {},
                'repositories': {},
        }

        self.__validateEntropyCache(self.etpSettingFiles['keywords'],self.etpMtimeFiles['keywords_mtime'])
        content = [x.split() for x in self.__generic_parser(self.etpSettingFiles['keywords']) if len(x.split()) < 4]
        for keywordinfo in content:
            # skip wrong lines
            if len(keywordinfo) > 3: continue
            if len(keywordinfo) == 1: # inversal keywording, check if it's not repo=
                # repo=?
                if keywordinfo[0].startswith("repo="): continue
                #kinfo = keywordinfo[0]
                if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format
                data['universal'].add(keywordinfo[0])
                continue # needed?
            if len(keywordinfo) in (2,3): # inversal keywording, check if it's not repo=
                # repo=?
                if keywordinfo[0].startswith("repo="): continue
                # add to repo?
                items = keywordinfo[1:]
                if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format
                reponame = [x for x in items if x.startswith("repo=") and (len(x.split("=")) == 2)]
                if reponame:
                    reponame = reponame[0].split("=")[1]
                    if reponame not in data['repositories']:
                        data['repositories'][reponame] = {}
                    # repository unmask or package in repository unmask?
                    if keywordinfo[0] not in data['repositories'][reponame]:
                        data['repositories'][reponame][keywordinfo[0]] = set()
                    if len(items) == 1:
                        # repository unmask
                        data['repositories'][reponame][keywordinfo[0]].add('*')
                    else:
                        if "*" not in data['repositories'][reponame][keywordinfo[0]]:
                            item = [x for x in items if not x.startswith("repo=")]
                            data['repositories'][reponame][keywordinfo[0]].add(item[0])
                else:
                    # it's going to be a faulty line!!??
                    if len(items) == 2: # can't have two items and no repo=
                        continue
                    # add keyword to packages
                    if keywordinfo[0] not in data['packages']:
                        data['packages'][keywordinfo[0]] = set()
                    data['packages'][keywordinfo[0]].add(items[0])
        return data


    def unmask_parser(self):
        self.__validateEntropyCache(self.etpSettingFiles['unmask'],self.etpMtimeFiles['unmask_mtime'])
        return self.__generic_parser(self.etpSettingFiles['unmask'])

    def mask_parser(self):
        self.__validateEntropyCache(self.etpSettingFiles['mask'],self.etpMtimeFiles['mask_mtime'])
        return self.__generic_parser(self.etpSettingFiles['mask'])

    def system_mask_parser(self):
        self.__validateEntropyCache(self.etpSettingFiles['system_mask'],self.etpMtimeFiles['system_mask_mtime'])
        return self.__generic_parser(self.etpSettingFiles['system_mask'])

    def license_mask_parser(self):
        self.__validateEntropyCache(self.etpSettingFiles['license_mask'],self.etpMtimeFiles['license_mask_mtime'])
        return self.__generic_parser(self.etpSettingFiles['license_mask'])

    def repos_license_whitelist_parser(self):
        data = {}
        for repoid in self.etpSettingFiles['repos_license_whitelist']:
            self.__validateEntropyCache(self.etpSettingFiles['repos_license_whitelist'][repoid],self.etpMtimeFiles['repos_license_whitelist'][repoid], repoid = repoid)
            data[repoid] = self.__generic_parser(self.etpSettingFiles['repos_license_whitelist'][repoid])
        return data

    def repos_mask_parser(self):
        data = {}
        for repoid in self.etpSettingFiles['repos_mask']:
            self.__validateEntropyCache(self.etpSettingFiles['repos_mask'][repoid],self.etpMtimeFiles['repos_mask'][repoid], repoid = repoid)
            data[repoid] = self.__generic_parser(self.etpSettingFiles['repos_mask'][repoid])
            # why ? line = line.split()[0] in the previous one?
        return data

    def repos_system_mask_parser(self):
        data = []
        for repoid in self.etpSettingFiles['repos_system_mask']:
            self.__validateEntropyCache(self.etpSettingFiles['repos_system_mask'][repoid],self.etpMtimeFiles['repos_system_mask'][repoid], repoid = repoid)
            data += [x for x in self.__generic_parser(self.etpSettingFiles['repos_system_mask'][repoid]) if x not in data]
            # why ? line = line.split()[0] in the previous one?
        return data

    def system_package_sets_parser(self):
        data = {}
        for set_name in self.etpSettingFiles['system_package_sets']:
            set_filepath = self.etpSettingFiles['system_package_sets'][set_name]
            set_elements = self.entropyTools.extract_packages_from_set_file(set_filepath)
            if set_elements: data[set_name] = set_elements.copy()
        return data

    def system_dirs_parser(self):
        return self.__generic_parser(self.etpSettingFiles['system_dirs'])

    def system_dirs_mask_parser(self):
        return self.__generic_parser(self.etpSettingFiles['system_dirs_mask'])

    def conflicting_tagged_packages_parser(self):
        data = {}
        # keep priority order
        repoids = [x for x in etpRepositoriesOrder if x in self.etpSettingFiles['conflicting_tagged_packages']]
        for repoid in repoids:
            filepath = self.etpSettingFiles['conflicting_tagged_packages'].get(repoid)
            if os.path.isfile(filepath) and os.access(filepath,os.R_OK):
                f = open(filepath,"r")
                content = f.readlines()
                f.close()
                content = [x.strip().rsplit("#",1)[0].strip().split() for x in content if not x.startswith("#") and x.strip()]
                for mydata in content:
                    if len(mydata) < 2: continue
                    data[mydata[0]] = mydata[1:]
        return data

    '''
    internal functions
    '''

    def __generic_parser(self, filepath):
        data = []
        if os.path.isfile(filepath) and os.access(filepath,os.R_OK):
            f = open(filepath,"r")
            content = f.readlines()
            f.close()
            # filter comments and white lines
            content = [x.strip().rsplit("#",1)[0].strip() for x in content if not x.startswith("#") and x.strip()]
            for line in content:
                if line in data: continue
                data.append(line)
        return data

    def __removeRepoCache(self, repoid = None):
        if os.path.isdir(etpConst['dumpstoragedir']):
            if repoid:
                self.Entropy.repository_move_clear_cache(repoid)
                return
            for repoid in etpRepositoriesOrder:
                self.Entropy.repository_move_clear_cache(repoid)
        else:
            os.makedirs(etpConst['dumpstoragedir'])

    def __saveFileMtime(self,toread,tosaveinto):

        if not os.path.isfile(toread):
            currmtime = 0.0
        else:
            currmtime = os.path.getmtime(toread)

        if not os.path.isdir(etpConst['dumpstoragedir']):
            os.makedirs(etpConst['dumpstoragedir'],0775)
            const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid'])

        f = open(tosaveinto,"w")
        f.write(str(currmtime))
        f.flush()
        f.close()
        os.chmod(tosaveinto,0664)
        if etpConst['entropygid'] != None:
            os.chown(tosaveinto,0,etpConst['entropygid'])


    def __validateEntropyCache(self, maskfile, mtimefile, repoid = None):

        if os.getuid() != 0: # can't validate if running as user, moreover users can't make changes, so...
            return

        # handle on-disk cache validation
        # in this case, repositories cache
        # if package.keywords is changed, we must destroy cache
        if not os.path.isfile(mtimefile):
            # we can't know if package.keywords has been updated
            # remove repositories caches
            self.__removeRepoCache(repoid = repoid)
            self.__saveFileMtime(maskfile,mtimefile)
        else:
            # check mtime
            try:
                f = open(mtimefile,"r")
                mtime = float(f.readline().strip())
                f.close()
                # compare with current mtime
                try:
                    currmtime = os.path.getmtime(maskfile)
                except OSError:
                    currmtime = 0.0
                if mtime != currmtime:
                    self.__removeRepoCache(repoid = repoid)
                    self.__saveFileMtime(maskfile,mtimefile)
            except:
                self.__removeRepoCache(repoid = repoid)
                self.__saveFileMtime(maskfile,mtimefile)

class Callable:
    def __init__(self, anycallable):
        self.__call__ = anycallable

class MultipartPostHandler(urllib2.BaseHandler):
    handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first

    def http_request(self, request):

        import urllib
        doseq = 1

        data = request.get_data()
        if data is not None and type(data) != str:
            v_files = []
            v_vars = []
            try:
                 for(key, value) in data.items():
                     if type(value) == file:
                         v_files.append((key, value))
                     else:
                         v_vars.append((key, value))
            except TypeError:
                systype, value, traceback = sys.exc_info()
                raise TypeError, "not a valid non-string sequence or mapping object", traceback

            if len(v_files) == 0:
                data = urllib.urlencode(v_vars, doseq)
            else:
                boundary, data = self.multipart_encode(v_vars, v_files)

                contenttype = 'multipart/form-data; boundary=%s' % boundary
                '''
                if (request.has_header('Content-Type')
                   and request.get_header('Content-Type').find('multipart/form-data') != 0):
                    print "Replacing %s with %s" % (request.get_header('content-type'), 'multipart/form-data')
                '''
                request.add_unredirected_header('Content-Type', contenttype)
            request.add_data(data)
        return request

    def multipart_encode(vars, files, boundary = None, buf = None):

        from cStringIO import StringIO
        import mimetools, mimetypes

        if boundary is None:
            boundary = mimetools.choose_boundary()
        if buf is None:
            buf = StringIO()
        for(key, value) in vars:
            buf.write('--%s\r\n' % boundary)
            buf.write('Content-Disposition: form-data; name="%s"' % key)
            buf.write('\r\n\r\n' + value + '\r\n')
        for(key, fd) in files:
            file_size = os.fstat(fd.fileno())[stat.ST_SIZE]
            filename = fd.name.split('/')[-1]
            contenttype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
            buf.write('--%s\r\n' % boundary)
            buf.write('Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (key, filename))
            buf.write('Content-Type: %s\r\n' % contenttype)
            # buffer += 'Content-Length: %s\r\n' % file_size
            fd.seek(0)
            buf.write('\r\n' + fd.read() + '\r\n')
        buf.write('--' + boundary + '--\r\n\r\n')
        buf = buf.getvalue()
        return boundary, buf
    multipart_encode = Callable(multipart_encode)

    https_request = http_request

class ErrorReportInterface:

    import entropyTools
    def __init__(self, post_url = etpConst['handlers']['errorsend']):
        self.url = post_url
        self.opener = urllib2.build_opener(MultipartPostHandler)
        self.generated = False
        self.params = {}

        mydict = {}
        if etpConst['proxy']['ftp']:
            mydict['ftp'] = etpConst['proxy']['ftp']
        if etpConst['proxy']['http']:
            mydict['http'] = etpConst['proxy']['http']
        if mydict:
            mydict['username'] = etpConst['proxy']['username']
            mydict['password'] = etpConst['proxy']['password']
            self.entropyTools.add_proxy_opener(urllib2,mydict)
        else:
            # unset
            urllib2._opener = None

    def prepare(self, tb_text, name, email, report_data = "", description = ""):

        self.params['arch'] = etpConst['currentarch']
        self.params['stacktrace'] = tb_text
        self.params['name'] = name
        self.params['email'] = email
        self.params['version'] = etpConst['entropyversion']
        self.params['errordata'] = report_data
        self.params['description'] = description
        self.params['arguments'] = ' '.join(sys.argv)
        self.params['uid'] = etpConst['uid']
        self.params['system_version'] = "N/A"
        if os.access(etpConst['systemreleasefile'],os.R_OK):
            f = open(etpConst['systemreleasefile'],"r")
            self.params['system_version'] = f.readlines()
            f.close()

        self.params['processes'] = commands.getoutput('ps auxf')
        self.params['lspci'] = commands.getoutput('/usr/sbin/lspci')
        self.params['dmesg'] = commands.getoutput('dmesg')
        self.params['locale'] = commands.getoutput('locale -v')
        self.params['stacktrace'] += "\n\n"+self.params['locale'] # just for a while, won't hurt

        self.generated = True

    # params is a dict, key(HTTP post item name): value
    def submit(self):
        if self.generated:
            result = self.opener.open(self.url, self.params).read()
            if result.strip() == "1":
                return True
            return False
        else:
            mytxt = _("Not prepared yet")
            raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,))


'''
   ~~ GIVES YOU WINGS ~~
'''
class SecurityInterface:

    # thanks to Gentoo "gentoolkit" package, License below:

    # This program is licensed under the GPL, version 2

    # WARNING: this code is only tested by a few people and should NOT be used
    # on production systems at this stage. There are possible security holes and probably
    # bugs in this code. If you test it please report ANY success or failure to
    # me (genone@gentoo.org).

    # The following planned features are currently on hold:
    # - getting GLSAs from http/ftp servers (not really useful without the fixed ebuilds)
    # - GPG signing/verification (until key policy is clear)

    def __init__(self, EquoInstance):

        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid EquoInterface instance or subclass is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
        self.Entropy = EquoInstance
        self.lastfetch = None
        self.previous_checksum = "0"
        self.advisories_changed = None
        self.adv_metadata = None
        self.affected_atoms = None

        from xml.dom import minidom
        self.minidom = minidom

        self.op_mappings = {
                            "le": "<=",
                            "lt": "<",
                            "eq": "=",
                            "gt": ">",
                            "ge": ">=",
                            "rge": ">=", # >=~
                            "rle": "<=", # <=~
                            "rgt": ">", # >~
                            "rlt": "<" # <~
        }

        self.unpackdir = os.path.join(etpConst['entropyunpackdir'],"security-"+str(self.Entropy.entropyTools.getRandomNumber()))
        self.security_url = etpConst['securityurl']
        self.unpacked_package = os.path.join(self.unpackdir,"glsa_package")
        self.security_url_checksum = etpConst['securityurl']+etpConst['packageshashfileext']
        self.download_package = os.path.join(self.unpackdir,os.path.basename(etpConst['securityurl']))
        self.download_package_checksum = self.download_package+etpConst['packageshashfileext']
        self.old_download_package_checksum = os.path.join(etpConst['dumpstoragedir'],os.path.basename(etpConst['securityurl']))+etpConst['packageshashfileext']

        self.security_package = os.path.join(etpConst['securitydir'],os.path.basename(etpConst['securityurl']))
        self.security_package_checksum = self.security_package+etpConst['packageshashfileext']

        try:
            if os.path.isfile(etpConst['securitydir']) or os.path.islink(etpConst['securitydir']):
                os.remove(etpConst['securitydir'])
            if not os.path.isdir(etpConst['securitydir']):
                os.makedirs(etpConst['securitydir'],0775)
        except OSError:
            pass
        const_setup_perms(etpConst['securitydir'],etpConst['entropygid'])

        if os.path.isfile(self.old_download_package_checksum):
            f = open(self.old_download_package_checksum)
            try:
                self.previous_checksum = f.readline().strip().split()[0]
            except:
                pass
            f.close()

    def __prepare_unpack(self):

        if os.path.isfile(self.unpackdir) or os.path.islink(self.unpackdir):
            os.remove(self.unpackdir)
        if os.path.isdir(self.unpackdir):
            shutil.rmtree(self.unpackdir,True)
            try:
                os.rmdir(self.unpackdir)
            except OSError:
                pass
        os.makedirs(self.unpackdir,0775)
        const_setup_perms(self.unpackdir,etpConst['entropygid'])

    def __download_glsa_package(self):
        return self.__generic_download(self.security_url, self.download_package)

    def __download_glsa_package_checksum(self):
        return self.__generic_download(self.security_url_checksum, self.download_package_checksum, showSpeed = False)

    def __generic_download(self, url, save_to, showSpeed = True):
        fetchConn = self.Entropy.urlFetcher(url, save_to, resume = False, showSpeed = showSpeed)
        fetchConn.progress = self.Entropy.progress
        rc = fetchConn.download()
        del fetchConn
        if rc in ("-1","-2","-3"):
            return False
        # setup permissions
        self.Entropy.setup_default_file_perms(save_to)
        return True

    def __verify_checksum(self):

        # read checksum
        if not os.path.isfile(self.download_package_checksum) or not os.access(self.download_package_checksum,os.R_OK):
            return 1

        f = open(self.download_package_checksum)
        try:
            checksum = f.readline().strip().split()[0]
            f.close()
        except:
            return 2

        if checksum == self.previous_checksum:
            self.advisories_changed = False
        else:
            self.advisories_changed = True
        md5res = self.Entropy.entropyTools.compareMd5(self.download_package,checksum)
        if not md5res:
            return 3
        return 0

    def __unpack_advisories(self):
        rc = self.Entropy.entropyTools.uncompressTarBz2(
                                                            self.download_package,
                                                            self.unpacked_package,
                                                            catchEmpty = True
                                                        )
        const_setup_perms(self.unpacked_package,etpConst['entropygid'])
        return rc

    def __clear_previous_advisories(self):
        if os.listdir(etpConst['securitydir']):
            shutil.rmtree(etpConst['securitydir'],True)
            if not os.path.isdir(etpConst['securitydir']):
                os.makedirs(etpConst['securitydir'],0775)
            const_setup_perms(self.unpackdir,etpConst['entropygid'])

    def __put_advisories_in_place(self):
        for advfile in os.listdir(self.unpacked_package):
            from_file = os.path.join(self.unpacked_package,advfile)
            to_file = os.path.join(etpConst['securitydir'],advfile)
            shutil.move(from_file,to_file)

    def __cleanup_garbage(self):
        shutil.rmtree(self.unpackdir,True)

    def clear(self, xcache = False):
        self.adv_metadata = None
        if xcache:
            self.Entropy.clear_dump_cache(etpCache['advisories'])

    def get_advisories_cache(self):

        if self.adv_metadata != None:
            return self.adv_metadata

        if self.Entropy.xcache:
            dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir'])
            c_hash = "%s%s" % (etpCache['advisories'],hash("%s|%s|%s" % (hash(etpConst['branch']),hash(dir_checksum),hash(etpConst['systemroot']),)),)
            adv_metadata = self.Entropy.Cacher.pop(c_hash)
            if adv_metadata != None:
                self.adv_metadata = adv_metadata.copy()
                return self.adv_metadata

    def set_advisories_cache(self, adv_metadata):
        if self.Entropy.xcache:
            dir_checksum = self.Entropy.entropyTools.md5sum_directory(etpConst['securitydir'])
            c_hash = "%s%s" % (etpCache['advisories'],hash("%s|%s|%s" % (hash(etpConst['branch']),hash(dir_checksum),hash(etpConst['systemroot']),)),)
            self.Entropy.Cacher.push(c_hash,adv_metadata.copy())

    def get_advisories_list(self):
        if not self.check_advisories_availability():
            return []
        xmls = os.listdir(etpConst['securitydir'])
        xmls = sorted([x for x in xmls if x.endswith(".xml") and x.startswith("glsa-")])
        return xmls

    def get_advisories_metadata(self):

        cached = self.get_advisories_cache()
        if cached != None:
            return cached

        adv_metadata = {}
        xmls = self.get_advisories_list()
        maxlen = len(xmls)
        count = 0
        for xml in xmls:

            count += 1
            if not etpUi['quiet']: self.Entropy.updateProgress(":: "+str(round((float(count)/maxlen)*100,1))+"% ::", importance = 0, type = "info", back = True)

            xml_metadata = None
            exc_string = ""
            exc_err = ""
            try:
                xml_metadata = self.get_xml_metadata(xml)
            except KeyboardInterrupt:
                return {}
            except Exception, e:
                exc_string = str(Exception)
                exc_err = str(e)
            if xml_metadata == None:
                more_info = ""
                if exc_string:
                    mytxt = _("Error")
                    more_info = " %s: %s: %s" % (mytxt,exc_string,exc_err,)
                mytxt = "%s: %s: %s! %s" % (
                    blue(_("Warning")),
                    bold(xml),
                    blue(_("advisory broken")),
                    more_info,
                )
                self.Entropy.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = red(" !!! ")
                )
                continue
            elif not xml_metadata:
                continue
            adv_metadata.update(xml_metadata)

        adv_metadata = self.filter_advisories(adv_metadata)
        self.set_advisories_cache(adv_metadata)
        self.adv_metadata = adv_metadata.copy()
        return adv_metadata

    # this function filters advisories for packages that aren't
    # in the repositories. Note: only keys will be matched
    def filter_advisories(self, adv_metadata):
        keys = adv_metadata.keys()
        for key in keys:
            valid = True
            if adv_metadata[key]['affected']:
                affected = adv_metadata[key]['affected']
                affected_keys = affected.keys()
                valid = False
                skipping_keys = set()
                for a_key in affected_keys:
                    match = self.Entropy.atomMatch(a_key)
                    if match[0] != -1:
                        # it's in the repos, it's valid
                        valid = True
                    else:
                        skipping_keys.add(a_key)
                if not valid:
                    del adv_metadata[key]
                for a_key in skipping_keys:
                    try:
                        del adv_metadata[key]['affected'][a_key]
                    except KeyError:
                        pass
                try:
                    if not adv_metadata[key]['affected']:
                        del adv_metadata[key]
                except KeyError:
                    pass

        return adv_metadata

    def is_affected(self, adv_key, adv_data = {}):
        if not adv_data:
            adv_data = self.get_advisories_metadata()
        if adv_key not in adv_data:
            return False
        mydata = adv_data[adv_key].copy()
        del adv_data

        if not mydata['affected']:
            return False

        for key in mydata['affected']:

            vul_atoms = mydata['affected'][key][0]['vul_atoms']
            unaff_atoms = mydata['affected'][key][0]['unaff_atoms']
            unaffected_atoms = set()
            if not vul_atoms:
                return False
            # XXX: does multimatch work correctly?
            for atom in unaff_atoms:
                matches = self.Entropy.clientDbconn.atomMatch(atom, multiMatch = True)
                if matches[1] == 0:
                    for idpackage in matches[0]:
                        unaffected_atoms.add((idpackage,0))

            for atom in vul_atoms:
                match = self.Entropy.clientDbconn.atomMatch(atom)
                if (match[0] != -1) and (match not in unaffected_atoms):
                    if self.affected_atoms == None:
                        self.affected_atoms = set()
                    self.affected_atoms.add(atom)
                    return True
        return False

    def get_vulnerabilities(self):
        return self.get_affection()

    def get_fixed_vulnerabilities(self):
        return self.get_affection(affected = False)

    # if not affected: not affected packages will be returned
    # if affected: affected packages will be returned
    def get_affection(self, affected = True):
        adv_data = self.get_advisories_metadata()
        adv_data_keys = adv_data.keys()
        valid_keys = set()
        for adv in adv_data_keys:
            is_affected = self.is_affected(adv,adv_data)
            if affected == is_affected:
                valid_keys.add(adv)
        # we need to filter our adv_data and return
        for key in adv_data_keys:
            if key not in valid_keys:
                try:
                    del adv_data[key]
                except KeyError:
                    pass
        # now we need to filter packages in adv_dat
        for adv in adv_data:
            for key in adv_data[adv]['affected'].keys():
                atoms = adv_data[adv]['affected'][key][0]['vul_atoms']
                applicable = True
                for atom in atoms:
                    if atom in self.affected_atoms:
                        applicable = False
                        break
                if applicable == affected:
                    del adv_data[adv]['affected'][key]
        return adv_data

    def get_affected_atoms(self):
        adv_data = self.get_advisories_metadata()
        adv_data_keys = adv_data.keys()
        del adv_data
        self.affected_atoms = set()
        for key in adv_data_keys:
            self.is_affected(key)
        return self.affected_atoms

    def get_xml_metadata(self, xmlfilename):
        xml_data = {}
        xmlfile = os.path.join(etpConst['securitydir'],xmlfilename)
        try:
            xmldoc = self.minidom.parse(xmlfile)
        except:
            return None

        # get base data
        glsa_tree = xmldoc.getElementsByTagName("glsa")[0]
        glsa_product = glsa_tree.getElementsByTagName("product")[0]
        if glsa_product.getAttribute("type") != "ebuild":
            return {}

        glsa_id = glsa_tree.getAttribute("id")
        glsa_title = glsa_tree.getElementsByTagName("title")[0].firstChild.data
        glsa_synopsis = glsa_tree.getElementsByTagName("synopsis")[0].firstChild.data
        glsa_announced = glsa_tree.getElementsByTagName("announced")[0].firstChild.data
        glsa_revised = glsa_tree.getElementsByTagName("revised")[0].firstChild.data

        xml_data['filename'] = xmlfilename
        xml_data['url'] = "http://www.gentoo.org/security/en/glsa/%s" % (xmlfilename,)
        xml_data['title'] = glsa_title.strip()
        xml_data['synopsis'] = glsa_synopsis.strip()
        xml_data['announced'] = glsa_announced.strip()
        xml_data['revised'] = glsa_revised.strip()
        xml_data['bugs'] = ["https://bugs.gentoo.org/show_bug.cgi?id="+x.firstChild.data.strip() for x in glsa_tree.getElementsByTagName("bug")]
        xml_data['access'] = ""
        try:
            xml_data['access'] = glsa_tree.getElementsByTagName("access")[0].firstChild.data.strip()
        except IndexError:
            pass

        # references
        references = glsa_tree.getElementsByTagName("references")[0]
        xml_data['references'] = [x.getAttribute("link").strip() for x in references.getElementsByTagName("uri")]

        try:
            xml_data['description'] = ""
            xml_data['description_items'] = []
            desc = glsa_tree.getElementsByTagName("description")[0].getElementsByTagName("p")[0].firstChild.data.strip()
            xml_data['description'] = desc
            items = glsa_tree.getElementsByTagName("description")[0].getElementsByTagName("ul")
            for item in items:
                li_items = item.getElementsByTagName("li")
                for li_item in li_items:
                    xml_data['description_items'].append(' '.join([x.strip() for x in li_item.firstChild.data.strip().split("\n")]))
        except IndexError:
            xml_data['description'] = ""
            xml_data['description_items'] = []
        try:
            workaround = glsa_tree.getElementsByTagName("workaround")[0]
            xml_data['workaround'] = workaround.getElementsByTagName("p")[0].firstChild.data.strip()
        except IndexError:
            xml_data['workaround'] = ""

        try:
            xml_data['resolution'] = []
            resolution = glsa_tree.getElementsByTagName("resolution")[0]
            p_elements = resolution.getElementsByTagName("p")
            for p_elem in p_elements:
                xml_data['resolution'].append(p_elem.firstChild.data.strip())
        except IndexError:
            xml_data['resolution'] = []

        try:
            impact = glsa_tree.getElementsByTagName("impact")[0]
            xml_data['impact'] = impact.getElementsByTagName("p")[0].firstChild.data.strip()
        except IndexError:
            xml_data['impact'] = ""
        xml_data['impacttype'] = glsa_tree.getElementsByTagName("impact")[0].getAttribute("type").strip()

        try:
            background = glsa_tree.getElementsByTagName("background")[0]
            xml_data['background'] = background.getElementsByTagName("p")[0].firstChild.data.strip()
        except IndexError:
            xml_data['background'] = ""

        # affection information
        affected = glsa_tree.getElementsByTagName("affected")[0]
        affected_packages = {}
        # we will then filter affected_packages using repositories information
        # if not affected_packages: advisory will be dropped
        for p in affected.getElementsByTagName("package"):
            name = p.getAttribute("name")
            if not affected_packages.has_key(name):
                affected_packages[name] = []

            pdata = {}
            pdata["arch"] = p.getAttribute("arch").strip()
            pdata["auto"] = (p.getAttribute("auto") == "yes")
            pdata["vul_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("vulnerable")]
            pdata["unaff_vers"] = [self.__make_version(v) for v in p.getElementsByTagName("unaffected")]
            pdata["vul_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("vulnerable")]
            pdata["unaff_atoms"] = [self.__make_atom(name, v) for v in p.getElementsByTagName("unaffected")]
            affected_packages[name].append(pdata)
        xml_data['affected'] = affected_packages.copy()

        return {glsa_id: xml_data}

    def __make_version(self, vnode):
        """
        creates from the information in the I{versionNode} a 
        version string (format <op><version>).

        @type	vnode: xml.dom.Node
        @param	vnode: a <vulnerable> or <unaffected> Node that
                                                    contains the version information for this atom
        @rtype:		String
        @return:	the version string
        """
        return self.op_mappings[vnode.getAttribute("range")] + vnode.firstChild.data.strip()

    def __make_atom(self, pkgname, vnode):
        """
        creates from the given package name and information in the 
        I{versionNode} a (syntactical) valid portage atom.

        @type	pkgname: String
        @param	pkgname: the name of the package for this atom
        @type	vnode: xml.dom.Node
        @param	vnode: a <vulnerable> or <unaffected> Node that
                                                    contains the version information for this atom
        @rtype:		String
        @return:	the portage atom
        """
        return str(self.op_mappings[vnode.getAttribute("range")] + pkgname + "-" + vnode.firstChild.data.strip())

    def check_advisories_availability(self):
        if not os.path.lexists(etpConst['securitydir']):
            return False
        if not os.path.isdir(etpConst['securitydir']):
            return False
        else:
            return True
        return False

    def fetch_advisories(self):

        mytxt = "%s: %s" % (bold(_("Security Advisories")),blue(_("testing service connection")),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = red(" @@ "),
            footer = red(" ...")
        )

        # Test network connectivity
        conntest = self.Entropy.entropyTools.get_remote_data(etpConst['conntestlink'])
        if not conntest:
            mytxt = _("Cannot connect to %s") % (etpConst['conntestlink'],)
            raise exceptionTools.OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,))

        mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("getting latest GLSAs")),red("..."),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )

        gave_up = self.Entropy.lock_check(self.Entropy._resources_run_check_lock)
        if gave_up:
            return 7

        locked = self.Entropy.application_lock_check()
        if locked:
            self.Entropy._resources_run_remove_lock()
            return 4

        # lock
        self.Entropy._resources_run_create_lock()
        try:
            rc = self.run_fetch()
        except:
            self.Entropy._resources_run_remove_lock()
            raise
        if rc != 0: return rc

        self.Entropy._resources_run_remove_lock()

        if self.advisories_changed:
            advtext = "%s: %s" % (bold(_("Security Advisories")),darkgreen(_("updated successfully")),)
        else:
            advtext = "%s: %s" % (bold(_("Security Advisories")),darkgreen(_("already up to date")),)

        self.Entropy.updateProgress(
            advtext,
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )

        return 0

    def run_fetch(self):
        # prepare directories
        self.__prepare_unpack()

        # download package
        status = self.__download_glsa_package()
        self.lastfetch = status
        if not status:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("unable to download the package, sorry")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 1

        mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("Verifying checksum")),red("..."),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   # "),
            back = True
        )

        # download digest
        status = self.__download_glsa_package_checksum()
        if not status:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot download the checksum, sorry")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 2

        # verify digest
        status = self.__verify_checksum()

        if status == 1:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot open packages, sorry")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 3
        elif status == 2:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("cannot read the checksum, sorry")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 4
        elif status == 3:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("digest verification failed, sorry")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 5
        elif status == 0:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkgreen(_("verification Successful")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red("   # ")
            )
        else:
            mytxt = _("Return status not valid")
            raise exceptionTools.InvalidData("InvalidData: %s." % (mytxt,))

        # save downloaded md5
        if os.path.isfile(self.download_package_checksum) and os.path.isdir(etpConst['dumpstoragedir']):
            if os.path.isfile(self.old_download_package_checksum):
                os.remove(self.old_download_package_checksum)
            shutil.copy2(self.download_package_checksum,self.old_download_package_checksum)
            self.Entropy.setup_default_file_perms(self.old_download_package_checksum)

        # now unpack in place
        status = self.__unpack_advisories()
        if status != 0:
            mytxt = "%s: %s." % (bold(_("Security Advisories")),darkred(_("digest verification failed, try again later")),)
            self.Entropy.updateProgress(
                mytxt,
                importance = 2,
                type = "error",
                header = red("   ## ")
            )
            self.Entropy._resources_run_remove_lock()
            return 6

        mytxt = "%s: %s %s" % (bold(_("Security Advisories")),blue(_("installing")),red("..."),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red("   # ")
        )

        # clear previous
        self.__clear_previous_advisories()
        # copy over
        self.__put_advisories_in_place()
        # remove temp stuff
        self.__cleanup_garbage()
        return 0

class SpmInterface:

    def __init__(self, OutputInterface):
        if not isinstance(OutputInterface, (EquoInterface, TextInterface, ServerInterface)):
                if OutputInterface == None:
                    OutputInterface = TextInterface()
                else:
                    mytxt = _("A valid TextInterface based instance is needed")
                    raise exceptionTools.IncorrectParameter(
                            "IncorrectParameter: %s" % (mytxt,)
                    )

        self.spm_backend = etpConst['spm']['backend']
        self.valid_backends = etpConst['spm']['available_backends']
        if self.spm_backend not in self.valid_backends:
            mytxt = "%s: %s" % (_("Invalid backend"),self.spm_backend,)
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        if self.spm_backend == "portage":
            self.intf = PortageInterface(OutputInterface)

    @staticmethod
    def get_spm_interface():
        backend = etpConst['spm']['backend']
        if backend == "portage":
            return PortageInterface

class PortageInterface:

    import entropyTools

    class paren_normalize(list):
        """Take a dependency structure as returned by paren_reduce or use_reduce
        and generate an equivalent structure that has no redundant lists."""
        def __init__(self, src):
            list.__init__(self)
            self._zap_parens(src, self)

        def _zap_parens(self, src, dest, disjunction=False):
            if not src:
                return dest
            i = iter(src)
            for x in i:
                if isinstance(x, basestring):
                    if x == '||':
                        x = self._zap_parens(i.next(), [], disjunction=True)
                        if len(x) == 1:
                            dest.append(x[0])
                        else:
                            dest.append("||")
                            dest.append(x)
                    elif x.endswith("?"):
                        dest.append(x)
                        dest.append(self._zap_parens(i.next(), []))
                    else:
                        dest.append(x)
                else:
                    if disjunction:
                        x = self._zap_parens(x, [])
                        if len(x) == 1:
                            dest.append(x[0])
                        else:
                            dest.append(x)
                    else:
                        self._zap_parens(x, dest)
            return dest

    def __init__(self, OutputInterface):
        if not isinstance(OutputInterface, (EquoInterface, TextInterface, ServerInterface)):
            mytxt = _("A valid TextInterface based instance is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        # interface only needed OutputInterface functions
        self.updateProgress = OutputInterface.updateProgress
        self.askQuestion = OutputInterface.askQuestion
        sys.path.append("/usr/lib/gentoolkit/pym")

        # importing portage stuff
        import portage
        self.portage = portage
        self.EAPI = 1
        try:
            import portage.const as portage_const
        except ImportError:
            import portage_const
        if hasattr(portage_const,"EAPI"):
            self.EAPI = portage_const.EAPI
        self.portage_const = portage_const

        try:
            from portage.versions import best
            self.portage_best = best
        except ImportError:
            from portage_versions import best
            self.portage_best = best

        try:
            import portage.util as portage_util
        except ImportError:
            import portage_util
        self.portage_util = portage_util

        try:
            import portage.sets as portage_sets
            self.portage_sets = portage_sets
        except ImportError:
            self.portage_sets = None

        try:
            import glsa
            self.glsa = glsa
        except ImportError:
            self.glsa = None

        if hasattr(self.portage,'exception'):
            self.portage_exception = self.portage.exception
        else: # portage <2.2 workaround
            self.portage_exception = Exception

        self.builtin_pkg_sets = [
            "system","world","installed","module-rebuild",
            "security","preserved-rebuild","live-rebuild",
            "downgrade","unavailable"
        ]

    def write_to_log(self, message):
        spmLog = LogFile(
            level = etpConst['spmloglevel'],
            filename = etpConst['spmlogfile'],
            header = "[spm]"
        )
        spmLog.write(message)
        spmLog.flush()
        spmLog.close()

    def write_traceback_to_log(self):
        spmLog = LogFile(
            level = etpConst['spmloglevel'],
            filename = etpConst['spmlogfile'],
            header = "[spm]"
        )
        self.entropyTools.printTraceback(f = spmLog)
        spmLog.flush()
        spmLog.close()

    def list_glsa_packages(self, command = "affected"):

        if not self.glsa: return
        if command not in ['new','all','affected']: return

        glsaconfig = self.glsa.checkconfig(self.portage.config(clone=self.portage.settings))
        completelist = self.glsa.get_glsa_list(glsaconfig["GLSA_DIR"], glsaconfig)

        glsalist = []
        if command == "new":
            checklist = []
            if os.access(glsaconfig["CHECKFILE"], os.R_OK):
                checklist = [line.strip() for line in open(glsaconfig["CHECKFILE"], "r").readlines()]
            glsalist = [e for e in completelist if e not in checklist]
        elif command == "all":
            glsalist = completelist
        elif command == "affected":
            # maybe this should be todolist instead
            for x in completelist:
                try:
                    myglsa = self.glsa.Glsa(x, glsaconfig)
                except (self.glsa.GlsaTypeException, self.glsa.GlsaFormatException), e:
                    continue
                if not myglsa.isVulnerable():
                    continue
                glsalist.append(x)

        return glsalist

    def get_glsa_id_information(self, glsa_id):
        if not self.glsa: return {}

        glsaconfig = self.glsa.checkconfig(self.portage.config(clone=self.portage.settings))
        try:
            myglsa = self.glsa.Glsa(glsa_id, glsaconfig)
        except (self.glsa.GlsaTypeException, self.glsa.GlsaFormatException):
            return {}

        mydict = {
            'glsa_id': glsa_id,
            'number': myglsa.nr,
            'access': myglsa.access,
            'title': myglsa.title,
            'synopsis': myglsa.synopsis,
            'announced': myglsa.announced,
            'revised': myglsa.revised,
            'bugs': myglsa.bugs,
            'description': myglsa.description,
            'resolution': myglsa.resolution,
            'impact': myglsa.impact_text,
            'impacttype': myglsa.impact_type,
            'affected': myglsa.affected,
            'background': myglsa.background,
            'glsatype': myglsa.glsatype,
            'packages': myglsa.packages,
            'services': myglsa.services,
            'product': myglsa.product,
            'references': myglsa.references,
            'workaround': myglsa.workaround,
        }
        if myglsa.isApplied():
            status = "[A]"
        elif myglsa.isVulnerable():
            status = "[N]"
        else:
            status = "[U]"
        mydict['status'] = status

        return mydict.copy()

    def run_fixpackages(self, myroot = None):
        if myroot == None:
            myroot = etpConst['systemroot']+"/"
        mydb = {}
        mydb[myroot] = {}
        mydb[myroot]['vartree'] = self._get_portage_vartree(myroot)
        mydb[myroot]['porttree'] = self._get_portage_portagetree(myroot)
        mydb[myroot]['bintree'] = self._get_portage_binarytree(myroot)
        mydb[myroot]['virtuals'] = self.portage.settings.getvirtuals(myroot)
        if etpUi['mute']:
            pid = os.fork()
            if pid > 0:
                os.waitpid(pid, 0)
            else:
                f = open("/dev/null","w")
                old_stdout = sys.stdout
                old_stderr = sys.stderr
                sys.stdout = f
                sys.stderr = f
                self.portage._global_updates(mydb, {})
                sys.stdout = old_stdout
                sys.stderr = old_stderr
                f.close()
                os._exit(0)
        else:
            self.portage._global_updates(mydb, {}) # always force

    def get_world_file(self):
        return os.path.join(etpConst['systemroot'],"/",self.portage_const.WORLD_FILE)

    def get_third_party_mirrors(self, mirrorname):
        x = []
        if self.portage.thirdpartymirrors.has_key(mirrorname):
            x = self.portage.thirdpartymirrors[mirrorname]
        return x

    def get_spm_setting(self, var):
        return self.portage.settings[var]

    def get_config_protect_and_mask(self):
        config_protect = self.portage.settings['CONFIG_PROTECT']
        config_protect = config_protect.split()
        config_protect_mask = self.portage.settings['CONFIG_PROTECT_MASK']
        config_protect_mask = config_protect_mask.split()
        # explode
        protect = []
        for x in config_protect:
            x = os.path.expandvars(x)
            protect.append(x)
        mask = []
        for x in config_protect_mask:
            x = os.path.expandvars(x)
            mask.append(x)
        return ' '.join(protect),' '.join(mask)

    def _get_portage_vartree(self, root):

        if not etpConst['spm']['cache'].has_key('portage'):
            etpConst['spm']['cache']['portage'] = {}
        if not etpConst['spm']['cache']['portage'].has_key('vartree'):
            etpConst['spm']['cache']['portage']['vartree'] = {}

        cached = etpConst['spm']['cache']['portage']['vartree'].get(root)
        if cached != None:
            return cached

        try:
            mytree = self.portage.vartree(root=root)
        except Exception, e:
            raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,))
        etpConst['spm']['cache']['portage']['vartree'][root] = mytree
        return mytree

    def _get_portage_portagetree(self, root):

        if not etpConst['spm']['cache'].has_key('portage'):
            etpConst['spm']['cache']['portage'] = {}
        if not etpConst['spm']['cache']['portage'].has_key('portagetree'):
            etpConst['spm']['cache']['portage']['portagetree'] = {}

        cached = etpConst['spm']['cache']['portage']['portagetree'].get(root)
        if cached != None:
            return cached

        try:
            mytree = self.portage.portagetree(root=root)
        except Exception, e:
            raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,))
        etpConst['spm']['cache']['portage']['portagetree'][root] = mytree
        return mytree

    def _get_portage_binarytree(self, root):

        if not etpConst['spm']['cache'].has_key('portage'):
            etpConst['spm']['cache']['portage'] = {}
        if not etpConst['spm']['cache']['portage'].has_key('binarytree'):
            etpConst['spm']['cache']['portage']['binarytree'] = {}

        cached = etpConst['spm']['cache']['portage']['binarytree'].get(root)
        if cached != None:
            return cached

        pkgdir = root+self.portage.settings['PKGDIR']
        try:
            mytree = self.portage.binarytree(root,pkgdir)
        except Exception, e:
            raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,))
        etpConst['spm']['cache']['portage']['binarytree'][root] = mytree
        return mytree

    def _get_portage_config(self, config_root, root, use_cache = True):

        if use_cache:
            if not etpConst['spm']['cache'].has_key('portage'):
                etpConst['spm']['cache']['portage'] = {}
            if not etpConst['spm']['cache']['portage'].has_key('config'):
                etpConst['spm']['cache']['portage']['config'] = {}

            cached = etpConst['spm']['cache']['portage']['config'].get((config_root,root))
            if cached != None:
                return cached

        try:
            mysettings = self.portage.config(config_root = config_root, target_root = root, config_incrementals = self.portage_const.INCREMENTALS)
        except Exception, e:
            raise exceptionTools.SPMError("SPMError: %s: %s" % (Exception,e,))
        if use_cache:
            etpConst['spm']['cache']['portage']['config'][(config_root,root)] = mysettings
        return mysettings

    # resolve atoms automagically (best, not current!)
    # sys-libs/application --> sys-libs/application-1.2.3-r1
    def get_best_atom(self, atom, match = "bestmatch-visible"):
        try:
            return self.portage.portdb.xmatch(match,str(atom))
        except ValueError:
            return None

    # same as above but includes masked ebuilds
    def get_best_masked_atom(self, atom):
        atoms = self.portage.portdb.xmatch("match-all",str(atom))
        return self.portage_best(atoms)

    def get_category_description_data(self, category):
        from xml.dom import minidom
        data = {}
        portdir = self.portage.settings['PORTDIR']
        myfile = os.path.join(portdir,category,"metadata.xml")
        if os.access(myfile,os.R_OK) and os.path.isfile(myfile):
            doc = minidom.parse(myfile)
            longdescs = doc.getElementsByTagName("longdescription")
            for longdesc in longdescs:
                data[longdesc.getAttribute("lang").strip()] = ' '.join([x.strip() for x in longdesc.firstChild.data.strip().split("\n")])
        return data

    def get_atom_category(self, atom):
        try:
            return self.portage.portdb.xmatch("match-all",str(atom))[0].split("/")[0]
        except:
            return None

    # Packages in system (in the Portage language -> emerge system, remember?)
    def get_atoms_in_system(self):
        system = self.portage.settings.packages
        sysoutput = []
        for x in system:
            y = self.get_installed_atoms(x)
            if (y != None):
                for z in y:
                    sysoutput.append(z)
        sysoutput.extend(etpConst['spm']['system_packages']) # add our packages
        return sysoutput

    def get_installed_atom(self, atom):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        rc = mytree.dep_match(str(atom))
        if rc: return rc[-1]

    def get_package_description(self, atom):
        if atom.startswith("="): atom = atom[1:]
        return self.portage.portdb.aux_get(atom,['DESCRIPTION'])[0]

    def get_installed_package_description(self, atom):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        if atom.startswith("="): atom = atom[1:]
        rc = mytree.dbapi.aux_get(atom, ["DESCRIPTION"])[0]
        if rc: return rc

    def get_package_slot(self, atom):
        if atom.startswith("="): atom = atom[1:]
        return self.portage.portdb.aux_get(atom,['SLOT'])[0]

    def get_installed_package_slot(self, atom):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        if atom.startswith("="): atom = atom[1:]
        rc = mytree.getslot(atom)
        if rc: return rc

    def get_installed_atoms(self, atom):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        rc = mytree.dep_match(str(atom))
        if rc: return rc

    def search_keys(self, key):
        key_split = key.split("/")
        cat = key_split[0]
        name = key_split[1]
        cat_dir = os.path.join(self.get_vdb_path(),cat)
        if not os.path.isdir(cat_dir):
            return None
        dir_content = [os.path.join(cat,x) for x in os.listdir(cat_dir) if x.startswith(name)]
        if not dir_content:
            return None
        return dir_content

    # create a .tbz2 file in the specified path
    def quickpkg(self, atom, dirpath):

        # getting package info
        pkgname = atom.split("/")[1]
        pkgcat = atom.split("/")[0]
        #pkgfile = pkgname+".tbz2"
        if not os.path.isdir(dirpath):
            os.makedirs(dirpath)
        dirpath += "/"+pkgname+etpConst['packagesext']
        dbdir = self.get_vdb_path()+"/"+pkgcat+"/"+pkgname+"/"

        import tarfile
        import stat
        trees = self.portage.db["/"]
        vartree = trees["vartree"]
        dblnk = self.portage.dblink(pkgcat, pkgname, "/", vartree.settings, treetype="vartree", vartree=vartree)
        dblnk.lockdb()
        tar = tarfile.open(dirpath,"w:bz2")

        contents = dblnk.getcontents()
        id_strings = {}
        paths = sorted(contents.keys())

        for path in paths:
            try:
                exist = os.lstat(path)
            except OSError:
                continue # skip file
            ftype = contents[path][0]
            lpath = path
            arcname = path[1:]
            if 'dir' == ftype and \
                not stat.S_ISDIR(exist.st_mode) and \
                os.path.isdir(lpath):
                lpath = os.path.realpath(lpath)
            tarinfo = tar.gettarinfo(lpath, arcname)
            tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
            tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))

            if stat.S_ISREG(exist.st_mode):
                tarinfo.type = tarfile.REGTYPE
                f = open(path)
                try:
                    tar.addfile(tarinfo, f)
                finally:
                    f.close()
            else:
                tar.addfile(tarinfo)

        tar.close()

        # appending xpak informations
        import etpXpak
        tbz2 = etpXpak.tbz2(dirpath)
        tbz2.recompose(dbdir)

        dblnk.unlockdb()

        if os.path.isfile(dirpath):
            return dirpath
        else:
            raise exceptionTools.FileNotFound("FileNotFound: Spm:quickpkg %s: %s %s" % (
                    _("error"),
                    dirpath,
                    _("not found"),
                )
            )

    def get_package_use_file(self):
        return os.path.join(self.portage_const.USER_CONFIG_PATH,'package.use')

    def enable_package_useflags(self, atom, useflags):
        result = self.unset_package_useflags(atom, useflags)
        if not result: return False
        return self._handle_new_useflags(atom, useflags, "")

    def disable_package_useflags(self, atom, useflags):
        result = self.unset_package_useflags(atom, useflags)
        if not result: return False
        return self._handle_new_useflags(atom, useflags, "-")

    def _handle_new_useflags(self, atom, useflags, mark):
        matched_atom = self.get_best_atom(atom)
        if not matched_atom:
            return False
        use_file = self.get_package_use_file()

        if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)):
            return False
        f = open(use_file,"r")
        content = [x.strip() for x in f.readlines()]
        f.close()

        def handle_line(line, useflags):

            data = line.split()
            if len(data) < 2:
                return False, line

            myatom = data[0]
            if matched_atom != self.get_best_atom(myatom):
                return False, line

            flags = data[1:]
            base_flags = []
            added_flags = []
            for flag in flags:
                myflag = flag
                if myflag.startswith("+"):
                    myflag = myflag[1:]
                elif myflag.startswith("-"):
                    myflag = myflag[1:]
                if not myflag:
                    continue
                base_flags.append(myflag)

            for useflag in useflags:
                if mark+useflag in base_flags:
                    continue
                added_flags.append(mark+useflag)

            new_line = "%s %s" % (myatom, ' '.join(flags+added_flags))
            return True, new_line


        atom_found = False
        new_content = []
        for line in content:

            changed, elaborated_line = handle_line(line, useflags)
            if changed: atom_found = True
            new_content.append(elaborated_line)

        if not atom_found:
            myline = "%s %s" % (atom, ' '.join([mark+x for x in useflags]))
            new_content.append(myline)


        f = open(use_file+".tmp","w")
        for line in new_content:
            f.write(line+"\n")
        f.flush()
        f.close()
        shutil.move(use_file+".tmp",use_file)
        return True

    def unset_package_useflags(self, atom, useflags):
        matched_atom = self.get_best_atom(atom)
        if not matched_atom:
            return False

        use_file = self.get_package_use_file()
        if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)):
            return False

        f = open(use_file,"r")
        content = [x.strip() for x in f.readlines()]
        f.close()

        new_content = []
        for line in content:

            data = line.split()
            if len(data) < 2:
                new_content.append(line)
                continue

            myatom = data[0]
            if matched_atom != self.get_best_atom(myatom):
                new_content.append(line)
                continue

            flags = data[1:]
            new_flags = []
            for flag in flags:
                myflag = flag

                if myflag.startswith("+"):
                    myflag = myflag[1:]
                elif myflag.startswith("-"):
                    myflag = myflag[1:]

                if myflag in useflags:
                    continue
                elif not flag:
                    continue

                new_flags.append(flag)

            if new_flags:
                new_line = "%s %s" % (myatom, ' '.join(new_flags))
                new_content.append(new_line)

        f = open(use_file+".tmp","w")
        for line in new_content:
            f.write(line+"\n")
        f.flush()
        f.close()
        shutil.move(use_file+".tmp",use_file)
        return True

    def get_package_useflags(self, atom):
        matched_atom = self.get_best_atom(atom)
        if not matched_atom:
            return {}
        global_useflags = self.get_useflags()
        use_force = self.get_useflags_force()
        use_mask = self.get_useflags_mask()
        package_use_useflags = self.get_package_use_useflags(atom)

        data = {}
        data['use_force'] = use_force.copy()
        data['use_mask'] = use_mask.copy()
        data['global_use'] = global_useflags.split()

        iuse = self.get_package_setting(atom, "IUSE")
        if not isinstance(iuse,basestring):
            iuse = ''
        data['iuse'] = iuse.split()[:]
        iuse = set()
        for myiuse in data['iuse']:
            if myiuse.startswith("+"):
                myiuse = myiuse[1:]
            iuse.add(myiuse)

        use = [f for f in data['global_use']+list(package_use_useflags['enabled']) if (f in iuse) and (f not in use_mask) and (f not in package_use_useflags['disabled'])]
        use_disabled = [f for f in iuse if (f not in data['global_use']) and (f not in use_mask) and (f not in package_use_useflags['enabled'])]
        data['use'] = use[:]
        data['use_disabled'] = use_disabled[:]

        matched_slot = self.get_package_slot(matched_atom)
        try:
            installed_atom = self.get_installed_atom("%s:%s" % (self.entropyTools.dep_getkey(atom),matched_slot,))
        except self.portage_exception:
            installed_atom = None

        if installed_atom:

            # get its useflags
            previous_iuse = self.get_installed_package_setting(installed_atom, "IUSE").split()
            previous_use = self.get_installed_package_setting(installed_atom, "USE").split()

            new_previous_iuse = set()
            for myuse in previous_iuse:
                if myuse.startswith("+"):
                    myuse = myuse[1:]
                new_previous_iuse.add(myuse)
            previous_iuse = list(new_previous_iuse)

            inst_use = [f for f in previous_iuse if (f in previous_use) and (f not in use_mask)]
            #inst_use_disabled = [f for f in previous_use if (f not in previous_iuse) and (f not in use_mask)]

            # check removed use
            use_removed = []
            for myuse in inst_use:
                if myuse not in use:
                    use_removed.append(myuse)

            # use not available
            use_not_avail = []
            for myuse in previous_iuse:
                if (myuse not in iuse) and (myuse not in use_removed):
                    use_not_avail.append(myuse)

            # check new use
            t_use = []
            for myuse in use:
                if myuse not in inst_use:
                    myuse = "+%s*" % (myuse,)
                t_use.append(myuse)
            use = t_use

            # check disabled use
            t_use_disabled = []
            for myuse in use_disabled:
                if myuse in inst_use:
                    if myuse in use_removed+use_not_avail:
                        continue
                    myuse = "-%s*" % (myuse,)
                else:
                    myuse = "-%s" % (myuse,)
                t_use_disabled.append(myuse)
            use_disabled = t_use_disabled

            for myuse in use_removed:
                use_disabled.append("(-%s*)" % (myuse,))
            for myuse in use_not_avail:
                use_disabled.append("(-%s)" % (myuse,))
        else:
            use_disabled = ["-"+x for x in use_disabled]

        data['use_string'] = ' '.join(sorted(use)+sorted([x for x in use_disabled]))
        data['use_string_colored'] = ' '.join(
                sorted([darkred(x) for x in use if not x.startswith("+")] + \
                        [darkgreen(x) for x in use if x.startswith("+")]) + \
                sorted([darkblue(x) for x in use_disabled if x.startswith("-")] + \
                        [brown(x) for x in use_disabled if x.startswith("(") and (x.find("*") == -1)] + \
                        [purple(x) for x in use_disabled if x.startswith("(") and (x.find("*") != -1)]
                )
        )
        return data

    def get_installed_package_useflags(self, atom):

        matched_atom = self.get_installed_atom(atom)
        if not matched_atom:
            return {}

        global_use = self.get_installed_package_setting(matched_atom, "USE")
        use_mask = self.get_useflags_mask()

        data = {}
        data['use_mask'] = use_mask.copy()
        data['global_use'] = global_use.split()

        iuse = self.get_installed_package_setting(matched_atom, "IUSE")
        if not isinstance(iuse,basestring): iuse = ''
        data['iuse'] = iuse.split()[:]
        iuse = set()
        for myiuse in data['iuse']:
            if myiuse.startswith("+"):
                myiuse = myiuse[1:]
            iuse.add(myiuse)

        use = [f for f in data['global_use'] if (f in iuse) and (f not in use_mask)]
        use_disabled = [f for f in iuse if (f not in data['global_use']) and (f not in use_mask)]
        data['use'] = use[:]
        data['use_disabled'] = use_disabled[:]

        data['use_string'] = ' '.join(sorted(use)+sorted([x for x in use_disabled]))
        data['use_string_colored'] = ' '.join(
                sorted([darkred(x) for x in use if not x.startswith("+")] + \
                        [darkgreen(x) for x in use if x.startswith("+")]) + \
                sorted([darkblue(x) for x in use_disabled if x.startswith("-")] + \
                        [brown(x) for x in use_disabled if x.startswith("(") and (x.find("*") == -1)] + \
                        [purple(x) for x in use_disabled if x.startswith("(") and (x.find("*") != -1)]
                )
        )
        return data

    # package.use
    def get_package_use_useflags(self, atom):

        data = {
            'enabled': set(),
            'disabled': set(),
        }

        matched_atom = self.get_best_atom(atom)
        if not matched_atom:
            return data

        use_file = self.get_package_use_file()
        if not (os.path.isfile(use_file) and os.access(use_file,os.W_OK)):
            return data

        use_data = self.portage_util.grabdict(use_file)
        for myatom in use_data:
            mymatch = self.get_best_atom(myatom)
            if mymatch != matched_atom:
                continue
            for flag in use_data[myatom]:
                if flag.startswith("-"):
                    myflag = flag[1:]
                    data['enabled'].discard(myflag)
                    data['disabled'].add(myflag)
                else:
                    myflag = flag
                    if myflag.startswith("+"):
                        myflag = myflag[1:]
                    data['disabled'].discard(myflag)
                    data['enabled'].add(myflag)

        return data

    def get_useflags(self):
        return self.portage.settings['USE']

    def get_useflags_force(self):
        return self.portage.settings.useforce

    def get_useflags_mask(self):
        return self.portage.settings.usemask

    def get_installed_package_setting(self, atom, setting):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        if atom.startswith("="): atom = atom[1:]
        return mytree.dbapi.aux_get(atom, [setting])[0]

    def get_package_setting(self, atom, setting):
        if atom.startswith("="): atom = atom[1:]
        return self.portage.portdb.aux_get(atom,[setting])[0]

    def query_files(self, atom):
        mypath = etpConst['systemroot']+"/"
        mysplit = atom.split("/")
        content = self.portage.dblink(mysplit[0], mysplit[1], mypath, self.portage.settings).getcontents()
        return content.keys()

    def query_belongs(self, filename, like = False):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        packages = mytree.dbapi.cpv_all()
        matches = set()
        for package in packages:
            mysplit = package.split("/")
            content = self.portage.dblink(mysplit[0], mysplit[1], mypath, self.portage.settings).getcontents()
            if not like:
                if filename in content:
                    matches.add(package)
            else:
                for myfile in content:
                    if myfile.find(filename) != -1:
                        matches.add(package)
        return matches

    def query_belongs_multiple(self, filenames, like = False):
        mypath = etpConst['systemroot']+"/"
        mytree = self._get_portage_vartree(mypath)
        packages = mytree.dbapi.cpv_all()
        matches = {}
        filenames = filenames.copy()
        for package in packages:
            cat, pkgv = package.split("/")
            content = self.portage.dblink(cat, pkgv, mypath, self.portage.settings).getcontents()
            if not like:
                for filename in filenames:
                    if filename in content:
                        myslot = self.get_installed_package_slot(package)
                        if not matches.has_key((package,myslot)):
                            matches[(package,myslot)] = set()
                        matches[(package,myslot)].add(filename)
            else:
                for filename in filenames:
                    for myfile in content:
                        if myfile.find(filename) != -1:
                            myslot = self.get_installed_package_slot(package)
                            if not matches.has_key((package,myslot)):
                                matches[(package,myslot)] = set()
                            matches[(package,myslot)].add(filename)
        return matches

    def calculate_dependencies(self, my_iuse, my_use, my_license, my_depend, my_rdepend, my_pdepend, my_provide, my_src_uri):
        metadata = {}
        metadata['USE'] = my_use
        metadata['IUSE'] = my_iuse
        metadata['LICENSE'] = my_license
        metadata['DEPEND'] = my_depend
        metadata['PDEPEND'] = my_pdepend
        metadata['RDEPEND'] = my_rdepend
        metadata['PROVIDE'] = my_provide
        metadata['SRC_URI'] = my_src_uri
        use = metadata['USE'].split()
        raw_use = use
        metadata['USE_MASK'] = self.get_useflags_mask()
        iuse = set()
        for myiuse in metadata['IUSE'].split():
            if myiuse.startswith("+"):
                myiuse = myiuse[1:]
                if (myiuse not in use) and ("-"+myiuse not in use):
                    use.append(myiuse)
            elif myiuse.startswith("-"):
                myiuse = myiuse[1:]
            iuse.add(myiuse)
        use = sorted([f for f in use if f in iuse])
        metadata['USE'] = " ".join(use)
        for k in "LICENSE", "RDEPEND", "DEPEND", "PDEPEND", "PROVIDE", "SRC_URI":
            try:
                deps = self.paren_reduce(metadata[k])
                deps = self.use_reduce(deps, uselist=raw_use)
                deps = self.paren_normalize(deps)
                if k == "LICENSE":
                    deps = self.paren_license_choose(deps)
                else:
                    deps = self.paren_choose(deps)
                if k.endswith("DEPEND"):
                    deps = self.usedeps_reduce(deps)
                deps = ' '.join(deps)
            except Exception, e:
                self.entropyTools.printTraceback()
                self.updateProgress(
                    darkred("%s: %s: %s :: %s") % (
                        _("Error calculating dependencies"),
                        str(Exception),
                        k,
                        e,
                    ),
                    importance = 1,
                    type = "error",
                    header = red(" !!! ")
                )
                deps = ''
                continue
            metadata[k] = deps
        return metadata

    def usedeps_reduce(self, dependencies):
        newlist = []
        for dependency in dependencies:
            use_deps = self.entropyTools.dep_getusedeps(dependency)
            if use_deps:
                use_deps = [x for x in use_deps if (x[0] not in ("!",)) and (x[-1] not in ("=","?",))]
                if use_deps:
                    dependency = "%s[%s]" % (self.entropyTools.remove_usedeps(dependency),','.join(use_deps),)
                else:
                    dependency = self.entropyTools.remove_usedeps(dependency)
            newlist.append(dependency)
        return newlist

    def paren_reduce(self, mystr,tokenize=1):
        """

            # deps.py -- Portage dependency resolution functions
            # Copyright 2003-2004 Gentoo Foundation
            # Distributed under the terms of the GNU General Public License v2
            # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $

        Take a string and convert all paren enclosed entities into sublists, optionally
        futher splitting the list elements by spaces.

        Example usage:
                >>> paren_reduce('foobar foo ( bar baz )',1)
                ['foobar', 'foo', ['bar', 'baz']]
                >>> paren_reduce('foobar foo ( bar baz )',0)
                ['foobar foo ', [' bar baz ']]

        @param mystr: The string to reduce
        @type mystr: String
        @param tokenize: Split on spaces to produces further list breakdown
        @type tokenize: Integer
        @rtype: Array
        @return: The reduced string in an array
        """
        mylist = []
        while mystr:
            left_paren = mystr.find("(")
            has_left_paren = left_paren != -1
            right_paren = mystr.find(")")
            has_right_paren = right_paren != -1
            if not has_left_paren and not has_right_paren:
                freesec = mystr
                subsec = None
                tail = ""
            elif mystr[0] == ")":
                return [mylist,mystr[1:]]
            elif has_left_paren and not has_right_paren:
                raise exceptionTools.InvalidDependString(
                        "InvalidDependString: %s: '%s'" % (_("missing right parenthesis"),mystr,))
            elif has_left_paren and left_paren < right_paren:
                freesec,subsec = mystr.split("(",1)
                subsec,tail = self.paren_reduce(subsec,tokenize)
            else:
                subsec,tail = mystr.split(")",1)
                if tokenize:
                    subsec = self.strip_empty(subsec.split(" "))
                    return [mylist+subsec,tail]
                return mylist+[subsec],tail
            mystr = tail
            if freesec:
                if tokenize:
                    mylist = mylist + self.strip_empty(freesec.split(" "))
                else:
                    mylist = mylist + [freesec]
            if subsec is not None:
                mylist = mylist + [subsec]
        return mylist

    def strip_empty(self, myarr):
        """

            # deps.py -- Portage dependency resolution functions
            # Copyright 2003-2004 Gentoo Foundation
            # Distributed under the terms of the GNU General Public License v2
            # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $

        Strip all empty elements from an array

        @param myarr: The list of elements
        @type myarr: List
        @rtype: Array
        @return: The array with empty elements removed
        """
        for x in range(len(myarr)-1, -1, -1):
                if not myarr[x]:
                        del myarr[x]
        return myarr

    def use_reduce(self, deparray, uselist=[], masklist=[], matchall=0, excludeall=[]):
        """

            # deps.py -- Portage dependency resolution functions
            # Copyright 2003-2004 Gentoo Foundation
            # Distributed under the terms of the GNU General Public License v2
            # $Id: portage_dep.py 9174 2008-01-11 05:49:02Z zmedico $

        Takes a paren_reduce'd array and reduces the use? conditionals out
        leaving an array with subarrays

        @param deparray: paren_reduce'd list of deps
        @type deparray: List
        @param uselist: List of use flags
        @type uselist: List
        @param masklist: List of masked flags
        @type masklist: List
        @param matchall: Resolve all conditional deps unconditionally.  Used by repoman
        @type matchall: Integer
        @rtype: List
        @return: The use reduced depend array
        """
        # Quick validity checks
        for x in range(len(deparray)):
            if deparray[x] in ["||","&&"]:
                if len(deparray) - 1 == x or not isinstance(deparray[x+1], list):
                    mytxt = _("missing atom list in")
                    raise exceptionTools.InvalidDependString(deparray[x]+" "+mytxt+" \""+str(deparray)+"\"")
        if deparray and deparray[-1] and deparray[-1][-1] == "?":
            mytxt = _("Conditional without target in")
            raise exceptionTools.InvalidDependString("InvalidDependString: "+mytxt+" \""+str(deparray)+"\"")

        # This is just for use by emerge so that it can enable a backward compatibility
        # mode in order to gracefully deal with installed packages that have invalid
        # atoms or dep syntax.  For backward compatibility with api consumers, strict
        # behavior will be explicitly enabled as necessary.
        _dep_check_strict = False

        mydeparray = deparray[:]
        rlist = []
        while mydeparray:
            head = mydeparray.pop(0)

            if isinstance(head,list):
                additions = self.use_reduce(head, uselist, masklist, matchall, excludeall)
                if additions:
                    rlist.append(additions)
                elif rlist and rlist[-1] == "||":
                    #XXX: Currently some DEPEND strings have || lists without default atoms.
                    #	raise portage_exception.InvalidDependString("No default atom(s) in \""+paren_enclose(deparray)+"\"")
                    rlist.append([])
            else:
                if head[-1] == "?": # Use reduce next group on fail.
                    # Pull any other use conditions and the following atom or list into a separate array
                    newdeparray = [head]
                    while isinstance(newdeparray[-1], str) and newdeparray[-1][-1] == "?":
                        if mydeparray:
                            newdeparray.append(mydeparray.pop(0))
                        else:
                            raise ValueError, _("Conditional with no target")

                    # Deprecation checks
                    warned = 0
                    if len(newdeparray[-1]) == 0:
                        mytxt = "%s. (%s)" % (_("Empty target in string"),_("Deprecated"),)
                        self.updateProgress(
                            darkred("PortageInterface.use_reduce(): %s" % (mytxt,)),
                            importance = 0,
                            type = "error",
                            header = bold(" !!! ")
                        )
                        warned = 1
                    if len(newdeparray) != 2:
                        mytxt = "%s. (%s)" % (_("Nested use flags without parenthesis"),_("Deprecated"),)
                        self.updateProgress(
                            darkred("PortageInterface.use_reduce(): %s" % (mytxt,)),
                            importance = 0,
                            type = "error",
                            header = bold(" !!! ")
                        )
                        warned = 1
                    if warned:
                        self.updateProgress(
                            darkred("PortageInterface.use_reduce(): "+" ".join(map(str,[head]+newdeparray))),
                            importance = 0,
                            type = "error",
                            header = bold(" !!! ")
                        )

                    # Check that each flag matches
                    ismatch = True
                    missing_flag = False
                    for head in newdeparray[:-1]:
                        head = head[:-1]
                        if not head:
                            missing_flag = True
                            break
                        if head.startswith("!"):
                            head_key = head[1:]
                            if not head_key:
                                missing_flag = True
                                break
                            if not matchall and head_key in uselist or \
                                head_key in excludeall:
                                ismatch = False
                                break
                        elif head not in masklist:
                            if not matchall and head not in uselist:
                                    ismatch = False
                                    break
                        else:
                            ismatch = False
                    if missing_flag:
                        mytxt = _("Conditional without flag")
                        raise exceptionTools.InvalidDependString(
                                "InvalidDependString: "+mytxt+": \"" + \
                                str([head+"?", newdeparray[-1]])+"\"")

                    # If they all match, process the target
                    if ismatch:
                        target = newdeparray[-1]
                        if isinstance(target, list):
                            additions = self.use_reduce(target, uselist, masklist, matchall, excludeall)
                            if additions:
                                    rlist.append(additions)
                        elif not _dep_check_strict:
                            # The old deprecated behavior.
                            rlist.append(target)
                        else:
                            mytxt = _("Conditional without parenthesis")
                            raise exceptionTools.InvalidDependString(
                                    "InvalidDependString: "+mytxt+": '%s?'" % head)

                else:
                    rlist += [head]
        return rlist

    def paren_choose(self, dep_list):
        newlist = []
        do_skip = False
        for idx in range(len(dep_list)):

            if do_skip:
                do_skip = False
                continue

            item = dep_list[idx]
            if item == "||": # or
                next_item = dep_list[idx+1]
                if not next_item: # || ( asd? ( atom ) dsa? ( atom ) ) => [] if use asd and dsa are disabled
                    do_skip = True
                    continue
                item = self.dep_or_select(next_item) # must be a list
                if not item:
                    # no matches, transform to string and append, so reagent will fail
                    newlist.append(str(next_item))
                else:
                    newlist += item
                do_skip = True
            elif isinstance(item, list): # and
                item = self.dep_and_select(item)
                newlist += item
            else:
                newlist.append(item)

        return newlist

    def dep_and_select(self, and_list):
        do_skip = False
        newlist = []
        for idx in range(len(and_list)):

            if do_skip:
                do_skip = False
                continue

            x = and_list[idx]
            if x == "||":
                x = self.dep_or_select(and_list[idx+1])
                do_skip = True
                if not x:
                    x = str(and_list[idx+1])
                else:
                    newlist += x
            elif isinstance(x, list):
                x = self.dep_and_select(x)
                newlist += x
            else:
                newlist.append(x)

        # now verify if all are satisfied
        for x in newlist:
            match = self.get_installed_atom(x)
            if match == None:
                return []

        return newlist

    def dep_or_select(self, or_list):
        do_skip = False
        for idx in range(len(or_list)):
            if do_skip:
                do_skip = False
                continue
            x = or_list[idx]
            if x == "||": # or
                x = self.dep_or_select(or_list[idx+1])
                do_skip = True
            elif isinstance(x, list): # and
                x = self.dep_and_select(x)
                if not x:
                    continue
                # found
                return x
            else:
                x = [x]

            for y in x:
                match = self.get_installed_atom(y)
                if match != None:
                    return [y]

        return []

    def paren_license_choose(self, dep_list):

        newlist = set()
        for item in dep_list:

            if isinstance(item, list):
                # match the first
                data = set(self.paren_license_choose(item))
                newlist.update(data)
            else:
                if item not in ["||"]:
                    newlist.add(item)

        return list(newlist)

    def get_vdb_path(self):
        rc = etpConst['systemroot']+"/"+self.portage_const.VDB_PATH
        if (not rc.endswith("/")):
            return rc+"/"
        return rc

    def get_available_packages(self, categories = [], filter_reinstalls = True):
        mypath = etpConst['systemroot']+"/"
        mysettings = self._get_portage_config("/",mypath)
        portdb = self.portage.portdbapi(mysettings["PORTDIR"], mysettings = mysettings)
        cps = portdb.cp_all()
        visibles = set()
        for cp in cps:
            if categories and cp.split("/")[0] not in categories:
                continue
            # get slots
            slots = set()
            atoms = self.get_best_atom(cp, "match-visible")
            if atoms:
                for atom in atoms:
                    slots.add(portdb.aux_get(atom, ["SLOT"])[0])
                for slot in slots:
                    visibles.add(cp+":"+slot)
        del cps

        # now match visibles
        available = set()
        for visible in visibles:
            match = self.get_best_atom(visible)
            if match == None:
                continue
            if filter_reinstalls:
                installed = self.get_installed_atom(visible)
                # if not installed, installed == None
                if installed != match:
                    available.add(match)
            else:
                available.add(match)
        del visibles

        return available

    # Collect installed packages
    def get_installed_packages(self, dbdir = None, categories = []):
        if not dbdir:
            appDbDir = self.get_vdb_path()
        else:
            appDbDir = dbdir
        dbDirs = os.listdir(appDbDir)
        installedAtoms = set()
        for pkgsdir in dbDirs:
            if os.path.isdir(appDbDir+pkgsdir):
                pkgdir = os.listdir(appDbDir+pkgsdir)
                for pdir in pkgdir:
                    pkgcat = pkgsdir.split("/")[-1]
                    if categories and (pkgcat not in categories):
                        continue
                    pkgatom = pkgcat+"/"+pdir
                    if pkgatom.find("-MERGING-") == -1:
                        installedAtoms.add(pkgatom)
        return sorted(list(installedAtoms)), len(installedAtoms)

    def get_installed_packages_counter(self, dbdir = None):
        if not dbdir:
            appDbDir = self.get_vdb_path()
        else:
            appDbDir = dbdir
        installedAtoms = set()

        for current_dirpath, subdirs, files in os.walk(appDbDir):
            pvs = os.listdir(current_dirpath)
            for mypv in pvs:
                if mypv.startswith("-MERGING-"):
                    continue
                mypvpath = current_dirpath+"/"+mypv
                if not os.path.isdir(mypvpath):
                    continue
                mycounter_file = mypvpath+"/"+etpConst['spm']['xpak_entries']['counter']
                if not os.access(mycounter_file,os.R_OK):
                    continue
                f = open(mycounter_file)
                try:
                    counter = int(f.readline().strip())
                except (IOError, ValueError):
                    f.close()
                    continue
                installedAtoms.add((os.path.basename(current_dirpath)+"/"+mypv,counter))
        return installedAtoms

    def _load_sets_config(self, settings, trees):

        # from portage.const import USER_CONFIG_PATH, GLOBAL_CONFIG_PATH
        setconfigpaths = [os.path.join(self.portage_const.GLOBAL_CONFIG_PATH, etpConst['setsconffilename'])]
        setconfigpaths.append(os.path.join(settings["PORTDIR"], etpConst['setsconffilename']))
        setconfigpaths += [os.path.join(x, etpConst['setsconffilename']) for x in settings["PORTDIR_OVERLAY"].split()]
        setconfigpaths.append(os.path.join(settings["PORTAGE_CONFIGROOT"],
            self.portage_const.USER_CONFIG_PATH.lstrip(os.path.sep), etpConst['setsconffilename']))
        return self.portage_sets.SetConfig(setconfigpaths, settings, trees)

    def get_set_config(self):
        # old portage
        if self.portage_sets == None: return
        myroot = etpConst['systemroot']+"/"
        return self._load_sets_config(
            self.portage.settings,
            self.portage.db[myroot]
        )

    def get_sets(self, builtin_sets):
        config = self.get_set_config()
        if config == None: return {}
        mysets = config.getSets()
        if not builtin_sets:
            builtin_pkg_sets = [x for x in self.builtin_pkg_sets if x in mysets]
            for pkg_set in builtin_pkg_sets: mysets.pop(pkg_set)
        return mysets


    def get_sets_expanded(self, builtin_sets = True):
        config = self.get_set_config()
        if config == None: return {}
        mysets = {}
        sets = config.getSets()
        if not builtin_sets:
            builtin_pkg_sets = [x for x in self.builtin_pkg_sets if x in sets]
            for pkg_set in builtin_pkg_sets: sets.pop(pkg_set)
        for myset in sorted(sets):
            try: atoms = config.getSetAtoms(myset).copy()
            except: continue
            mysets[myset] = atoms
        return mysets

    def refill_counter(self, dbdir = None):
        if not dbdir:
            appDbDir = self.get_vdb_path()
        else:
            appDbDir = dbdir
        counters = set()
        for catdir in os.listdir(appDbDir):
            catdir = appDbDir+catdir
            if not os.path.isdir(catdir):
                continue
            for pkgdir in os.listdir(catdir):
                pkgdir = catdir+"/"+pkgdir
                if not os.path.isdir(pkgdir):
                    continue
                counterfile = pkgdir+"/"+etpConst['spm']['xpak_entries']['counter']
                if not os.path.isfile(pkgdir+"/"+etpConst['spm']['xpak_entries']['counter']):
                    continue
                try:
                    f = open(counterfile,"r")
                    counter = int(f.readline().strip())
                    counters.add(counter)
                    f.close()
                except:
                    continue
        if counters:
            newcounter = max(counters)
        else:
            newcounter = 0
        if not os.path.isdir(os.path.dirname(etpConst['edbcounter'])):
            os.makedirs(os.path.dirname(etpConst['edbcounter']))
        try:
            f = open(etpConst['edbcounter'],"w")
        except IOError, e:
            if e[0] == 21:
                shutil.rmtree(etpConst['edbcounter'],True)
                try:
                    os.rmdir(etpConst['edbcounter'])
                except:
                    pass
            f = open(etpConst['edbcounter'],"w")
        f.write(str(newcounter))
        f.flush()
        f.close()
        del counters
        return newcounter


    def spm_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = [], fork = False):
        if fork:
            # memory leak: some versions of portage were memleaking here
            return self.entropyTools.spawnFunction(
                self._portage_doebuild, myebuild,
                mydo, tree, cpv,
                portage_tmpdir, licenses
            )
        return self._portage_doebuild(myebuild, mydo, tree, cpv, portage_tmpdir, licenses)

    def _portage_doebuild(self, myebuild, mydo, tree, cpv, portage_tmpdir = None, licenses = []):
        # myebuild = path/to/ebuild.ebuild with a valid unpacked xpak metadata
        # tree = "bintree"
        # cpv = atom
        # mydbapi = portage.fakedbapi(settings=portage.settings)
        # vartree = portage.vartree(root=myroot)

        oldsystderr = sys.stderr
        f = open("/dev/null","w")
        if not etpUi['debug']:
            sys.stderr = f

        ### SETUP ENVIRONMENT
        # if mute, supress portage output
        domute = False
        if etpUi['mute']:
            domute = True
            oldsysstdout = sys.stdout
            sys.stdout = f

        mypath = etpConst['systemroot']+"/"
        os.environ["SKIP_EQUO_SYNC"] = "1"
        os.environ["CD_ROOT"] = "/tmp" # workaround for scripts asking for user intervention
        os.environ["ROOT"] = mypath

        if licenses:
            os.environ["ACCEPT_LICENSE"] = str(' '.join(licenses)) # we already do this early

        # load metadata
        myebuilddir = os.path.dirname(myebuild)
        keys = self.portage.auxdbkeys
        metadata = {}

        for key in keys:
            mykeypath = os.path.join(myebuilddir,key)
            if os.path.isfile(mykeypath) and os.access(mykeypath,os.R_OK):
                f = open(mykeypath,"r")
                metadata[key] = f.readline().strip()
                f.close()

        ### END SETUP ENVIRONMENT

        # find config
        mysettings = self._get_portage_config("/",mypath)
        mysettings['EBUILD_PHASE'] = mydo
        mysettings['EAPI'] = "0"
        if metadata.has_key('EAPI'):
            mysettings['EAPI'] = metadata['EAPI']
        mysettings.backup_changes("EAPI")

        try: # this is a >portage-2.1.4_rc11 feature
            mysettings._environ_whitelist = set(mysettings._environ_whitelist)
            # put our vars into whitelist
            mysettings._environ_whitelist.add("SKIP_EQUO_SYNC")
            mysettings._environ_whitelist.add("ACCEPT_LICENSE")
            mysettings._environ_whitelist.add("CD_ROOT")
            mysettings._environ_whitelist.add("ROOT")
            mysettings._environ_whitelist = frozenset(mysettings._environ_whitelist)
        except:
            self.write_traceback_to_log()

        cpv = str(cpv)
        mysettings.setcpv(cpv)
        portage_tmpdir_created = False # for pkg_postrm, pkg_prerm
        if portage_tmpdir:
            if not os.path.isdir(portage_tmpdir):
                os.makedirs(portage_tmpdir)
                portage_tmpdir_created = True
            mysettings['PORTAGE_TMPDIR'] = str(portage_tmpdir)
            mysettings.backup_changes("PORTAGE_TMPDIR")

        mydbapi = self.portage.fakedbapi(settings=mysettings)
        mydbapi.cpv_inject(cpv, metadata = metadata)

        # cached vartree class
        vartree = self._get_portage_vartree(mypath)

        try:
            rc = self.portage.doebuild(
                myebuild = str(myebuild),
                mydo = str(mydo),
                myroot = mypath,
                tree = tree,
                mysettings = mysettings,
                mydbapi = mydbapi,
                vartree = vartree,
                use_cache = 0
            )
        except:
            self.write_traceback_to_log()
            raise

        # if mute, restore old stdout/stderr
        if domute:
            sys.stdout = oldsysstdout

        sys.stderr = oldsystderr
        f.close()

        if portage_tmpdir_created:
            shutil.rmtree(portage_tmpdir,True)

        del mydbapi
        del metadata
        del keys
        return rc

class LogFile:

    def __init__ (self, level = 0, filename = None, header = "[LOG]"):
        self.handler = self.default_handler
        self.level = level
        self.header = header
        self.logFile = None
        self.open(filename)

    def __del__(self):
        self.close()

    def close(self):
        try:
            self.logFile.close()
        except (IOError,OSError,):
            pass

    def flush(self):
        self.logFile.flush()

    def fileno(self):
        return self.getFile()

    def isatty(self):
        return False

    def read(self, a):
        return ''

    def readline(self):
        return ''

    def readlines(self):
        return []

    def seek(self, a):
        return self.logFile.seek(a)

    def tell(self):
        return self.logFile.tell()

    def truncate(self):
        return self.logFile.truncate()

    def open (self, file = None):
        if isinstance(file,basestring):
            if os.access(file,os.W_OK) and os.path.isfile(file):
                self.logFile = open(file, "aw")
            else:
                self.logFile = open("/dev/null", "aw")
        elif hasattr(file,'write'):
            self.logFile = file
        else:
            self.logFile = sys.stderr

    def getFile (self):
        return self.logFile.fileno()

    def __call__(self, format, *args):
        self.handler (format % args)

    def default_handler (self, mystr):
        try:
            self.logFile.write ("* %s\n" % (mystr))
        except UnicodeEncodeError:
            self.logFile.write ("* %s\n" % (mystr.encode('utf-8'),))
        self.logFile.flush()

    def set_loglevel(self, level):
        self.level = level

    def log(self, messagetype, level, message):
        if self.level >= level and not etpUi['nolog']:
            self.handler("%s%s %s %s" % (self.getTimeDateHeader(),messagetype,self.header,message,))

    def write(self, s):
        self.handler(s)

    def writelines(self, lst):
        for s in lst:
            self.write(s)

    def getTimeDateHeader(self):
        return time.strftime('[%X %x %Z] ')

    def ladd(self, level, file, message):
        if self.level >= level:
            self.handler("++ %s \t%s" % (file, message))

    def ldel(self, level, file, message):
        if self.level >= level:
            self.handler("-- %s \t%s" % (file, message))

    def lch(self, level, file, message):
        if self.level >= level:
            self.handler("-+ %s \t%s" % (file, message))

class SocketCommandsSkel:

    def __str__(self):
        return self.inst_name

    inst_name = 'command-skel'
    def __init__(self, HostInterface, inst_name = 'command-skel'):
        self.HostInterface = HostInterface
        self.no_acked_commands = []
        self.termination_commands = []
        self.initialization_commands = []
        self.login_pass_commands = []
        self.no_session_commands = []
        self.raw_commands = []
        self.config_commands = []
        self.valid_commands = set()
        self.inst_name = inst_name

    def register(
            self,
            valid_commands,
            no_acked_commands,
            termination_commands,
            initialization_commands,
            login_pass_commads,
            no_session_commands,
            raw_commands,
            config_commands
        ):
        valid_commands.update(self.valid_commands)
        no_acked_commands.extend(self.no_acked_commands)
        termination_commands.extend(self.termination_commands)
        initialization_commands.extend(self.initialization_commands)
        login_pass_commads.extend(self.login_pass_commands)
        no_session_commands.extend(self.no_session_commands)
        raw_commands.extend(self.raw_commands)
        config_commands.extend(self.config_commands)

class SocketAuthenticatorSkel:

    def __init__(self, HostInterface):
        self.HostInterface = HostInterface
        self.session = None

    def set_session(self, session):
        self.session = session

class SocketHostInterface:

    import socket
    import SocketServer
    import entropyTools
    from threading import Thread

    class BasicPamAuthenticator(SocketAuthenticatorSkel):

        import entropyTools

        def __init__(self, HostInterface, *args, **kwargs):
            self.valid_auth_types = [ "plain", "shadow", "md5" ]
            SocketAuthenticatorSkel.__init__(self, HostInterface)

        def docmd_login(self, arguments):

            with self.HostInterface.AuthenticatorLock:

                # filter n00bs
                if not arguments or (len(arguments) != 3):
                    return False,None,None,'wrong arguments'

                user = arguments[0]
                auth_type = arguments[1]
                auth_string = arguments[2]

                # check auth type validity
                if auth_type not in self.valid_auth_types:
                    return False,user,None,'invalid auth type'

                udata = self.__get_user_data(user)
                if udata == None:
                    return False,user,None,'invalid user'

                uid = udata[2]
                # check if user is in the Entropy group
                if not self.entropyTools.is_user_in_entropy_group(uid):
                    return False,user,uid,'user not in %s group' % (etpConst['sysgroup'],)

                # now validate password
                valid = self.__validate_auth(user,auth_type,auth_string)
                if not valid:
                    return False,user,uid,'auth failed'

                if not uid:
                    self.HostInterface.sessions[self.session]['admin'] = True
                else:
                    self.HostInterface.sessions[self.session]['user'] = True
                return True,user,uid,"ok"

        # it we get here is because user is logged in
        def docmd_userdata(self):

            with self.HostInterface.AuthenticatorLock:

                auth_uid = self.HostInterface.sessions[self.session]['auth_uid']
                mydata = {}
                udata = self.__get_uid_data(auth_uid)
                if udata:
                    mydata['username'] = udata[0]
                    mydata['uid'] = udata[2]
                    mydata['gid'] = udata[3]
                    mydata['references'] = udata[4]
                    mydata['home'] = udata[5]
                    mydata['shell'] = udata[6]
                return True,mydata,'ok'

        def __get_uid_data(self, user_id):
            import pwd
            # check user validty
            try:
                udata = pwd.getpwuid(user_id)
            except KeyError:
                return None
            return udata

        def __get_user_data(self, user):
            import pwd
            # check user validty
            try:
                udata = pwd.getpwnam(user)
            except KeyError:
                return None
            return udata

        def __validate_auth(self, user, auth_type, auth_string):
            valid = False
            if auth_type == "plain":
                valid = self.__do_auth(user, auth_string)
            elif auth_type == "shadow":
                valid = self.__do_auth(user, auth_string, auth_type = "shadow")
            elif auth_type == "md5":
                valid = self.__do_auth(user, auth_string, auth_type = "md5")
            return valid

        def __do_auth(self, user, password, auth_type = None):
            import spwd

            try:
                enc_pass = spwd.getspnam(user)[1]
            except KeyError:
                return False

            if auth_type == None: # plain
                import crypt
                generated_pass = crypt.crypt(str(password), enc_pass)
            elif auth_type == "shadow":
                generated_pass = password
            elif auth_type == "md5": # md5
                import hashlib
                m = hashlib.md5()
                m.update(enc_pass)
                enc_pass = m.hexdigest()
                generated_pass = str(password)
            else: # haha, fuck!
                generated_pass = None

            if generated_pass == enc_pass:
                return True
            return False

        def docmd_logout(self, myargs):

            with self.HostInterface.AuthenticatorLock:

                # filter n00bs
                if (len(myargs) < 1) or (len(myargs) > 1):
                    return False,None,'wrong arguments'

                user = myargs[0]
                # filter n00bs
                if not user or not isinstance(user,basestring):
                    return False,None,"wrong user"

                return True,user,"ok"

        def set_exc_permissions(self, uid, gid):
            with self.HostInterface.AuthenticatorLock:
                if gid != None:
                    os.setgid(gid)
                if uid != None:
                    os.setuid(uid)

        def hide_login_data(self, args):
            myargs = args[:]
            myargs[-1] = 'hidden'
            return myargs

        def terminate_instance(self):
            pass

    class HostServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):

        class ConnWrapper:
            '''
            Base class for implementing the rest of the wrappers in this module.
            Operates by taking a connection argument which is used when 'self' doesn't
            provide the functionality being requested.
            '''
            def __init__(self, connection) :
                self.connection = connection

            def __getattr__(self, function) :
                return getattr(self.connection, function)

        import socket as socket_mod
        import select
        import SocketServer
        # This means the main server will not do the equivalent of a
        # pthread_join() on the new threads.  With this set, Ctrl-C will
        # kill the server reliably.
        daemon_threads = True

        # By setting this we allow the server to re-bind to the address by
        # setting SO_REUSEADDR, meaning you don't have to wait for
        # timeouts when you kill the server and the sockets don't get
        # closed down correctly.
        allow_reuse_address = True

        def __init__(self, server_address, RequestHandlerClass, processor, HostInterface, authorized_clients_only = False):

            self.alive = True
            self.socket = self.socket_mod
            self.processor = processor
            self.server_address = server_address
            self.HostInterface = HostInterface
            self.SSL = self.HostInterface.SSL
            self.real_sock = None
            self.ssl_authorized_clients_only = authorized_clients_only

            if self.SSL:
                self.SocketServer.BaseServer.__init__(self, server_address, RequestHandlerClass)
                self.load_ssl_context()
                self.make_ssl_connection_alive()
            else:
                try:
                    self.SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
                except self.socket_mod.error, e:
                    if e[0] == 13:
                        raise exceptionTools.ConnectionError('ConnectionError: %s' % (_("Cannot bind the service"),))
                    raise

        def load_ssl_context(self):
            # setup an SSL context.
            self.context = self.SSL['m'].Context(self.SSL['m'].SSLv23_METHOD)
            self.context.set_verify(self.SSL['m'].VERIFY_PEER, self.verify_ssl_cb) # ask for a certificate
            self.context.set_options(self.SSL['m'].OP_NO_SSLv2)
            # load up certificate stuff.
            self.context.use_privatekey_file(self.SSL['key'])
            self.context.use_certificate_file(self.SSL['cert'])
            self.context.load_verify_locations(self.SSL['ca_cert'])
            self.context.load_client_ca(self.SSL['ca_cert'])
            self.HostInterface.updateProgress('SSL context loaded, key: %s - cert: %s, CA cert: %s, CA pkey: %s' % (
                    self.SSL['key'],
                    self.SSL['cert'],
                    self.SSL['ca_cert'],
                    self.SSL['ca_pkey']
                )
            )

        def make_ssl_connection_alive(self):
            self.real_sock = self.socket_mod.socket(self.address_family, self.socket_type)
            self.socket = self.ConnWrapper(self.SSL['m'].Connection(self.context, self.real_sock))

            self.server_bind()
            self.server_activate()

        # this function should do the authentication checking to see that
        # the client is who they say they are.
        def verify_ssl_cb(self, conn, cert, errnum, depth, ok) :
            return ok

        def verify_request(self, request, client_address):

            self.do_ssl = self.HostInterface.SSL
            if self.do_ssl: self.do_ssl = True
            else: self.do_ssl = False

            allowed = self.ip_blacklist_check(client_address[0])
            if allowed: allowed = self.ip_max_connections_check(client_address[0])
            if not allowed:
                self.HostInterface.updateProgress(
                    '[from: %s | SSL: %s] connection refused, ip blacklisted or maximum connections per IP reached' % (
                        client_address,
                        self.do_ssl,
                    )
                )
                return False

            allowed = self.max_connections_check(request)
            if not allowed:
                self.HostInterface.updateProgress(
                    '[from: %s | SSL: %s] connection refused (max connections reached: %s)' % (
                        client_address,
                        self.do_ssl,
                        self.HostInterface.max_connections,
                    )
                )
                return False

            ### let's go!
            self.HostInterface.connections += 1
            self.HostInterface.updateProgress(
                '[from: %s | SSL: %s] connection established (%s of %s max connections)' % (
                    client_address,
                    self.do_ssl,
                    self.HostInterface.connections,
                    self.HostInterface.max_connections,
                )
            )
            return True

        def ip_blacklist_check(self, client_addr):
            if client_addr in self.HostInterface.ip_blacklist:
                return False
            return True

        def ip_max_connections_check(self, ip_address):
            max_conn_per_ip = self.HostInterface.max_connections_per_host
            max_conn_per_ip_barrier = self.HostInterface.max_connections_per_host_barrier
            per_host_connections = self.HostInterface.per_host_connections
            conn_data = per_host_connections.get(ip_address)
            if conn_data == None:
                per_host_connections[ip_address] = 1
            else:
                conn_data += 1
                per_host_connections[ip_address] += 1
                if conn_data > max_conn_per_ip:
                    self.HostInterface.updateProgress(
                        '[from: %s] ------- :EEK: !! connection closed too many simultaneous connections from host (current: %s | limit: %s) -------' % (
                            ip_address,
                            conn_data,
                            max_conn_per_ip,
                        )
                    )
                    return False
                elif conn_data > max_conn_per_ip_barrier:
                    times = [5,6,7,8]
                    self.HostInterface.updateProgress(
                        '[from: %s] ------- :EEEK: !! connection warning simultaneous connection barrier reached from host (current: %s | soft limit: %s) -------' % (
                            ip_address,
                            conn_data,
                            max_conn_per_ip_barrier,
                        )
                    )
                    time.sleep(times[abs(hash(os.urandom(1)))%len(times)])

            return True

        def max_connections_check(self, request):
            current = self.HostInterface.connections
            maximum = self.HostInterface.max_connections
            if current >= maximum:
                self.HostInterface.transmit(
                    request,
                    self.HostInterface.answers['mcr']
                )
                return False
            else:
                return True

        def serve_forever(self):
            while self.alive:
                #r,w,e = self.select.select([self.socket], [], [], 1)
                #if r:
                self.handle_request()

        # taken from SocketServer.py
        def finish_request(self, request, client_address):
            """Finish one request by instantiating RequestHandlerClass."""
            self.RequestHandlerClass(request, client_address, self)

            self.HostInterface.updateProgress(
                '[from: %s] connection closed (%s of %s max connections)' % (
                    client_address,
                    self.HostInterface.connections - 1,
                    self.HostInterface.max_connections,
                )
            )
            per_host_connections = self.HostInterface.per_host_connections
            conn_data = per_host_connections.get(client_address[0])
            if conn_data != None:
                if conn_data < 1:
                    del per_host_connections[client_address[0]]
                else:
                    per_host_connections[client_address[0]] -= 1

        def close_request(self, request):
            if self.HostInterface.connections > 0:
                self.HostInterface.connections -= 1

    class RequestHandler(SocketServer.BaseRequestHandler):

        import SocketServer
        import select
        import socket
        import entropyTools
        import gc
        timed_out = False

        def __init__(self, request, client_address, server):
            self.SocketServer.BaseRequestHandler.__init__(self, request, client_address, server)

        def data_receiver(self):

            if self.timed_out:
                return True
            self.timed_out = True
            try:
                ready_to_read, ready_to_write, in_error = self.select.select([self.request], [], [], self.default_timeout)
            except KeyboardInterrupt:
                self.timed_out = True
                return True

            if len(ready_to_read) == 1 and ready_to_read[0] == self.request:

                self.timed_out = False

                try:

                    data = self.request.recv(1024)
                    if self.ssl:
                        while self.request.pending():
                            data += self.request.recv(1024)

                    if self.data_counter == None:
                        if data == '': # client wants to close
                            return True
                        elif data == self.server.processor.HostInterface.answers['noop']:
                            return False
                        elif len(data) < len(self.myeos):
                            self.server.processor.HostInterface.updateProgress(
                                'interrupted: %s, reason: %s - from client: %s - data: "%s" - counter: %s' % (
                                    self.server.server_address,
                                    "malformed EOS",
                                    self.client_address,
                                    repr(data),
                                    self.data_counter,
                                )
                            )
                            self.buffered_data = ''
                            return True
                        mystrlen = data.split(self.myeos)[0]
                        self.data_counter = int(mystrlen)
                        data = data[len(mystrlen)+1:]
                        self.data_counter -= len(data)
                        self.buffered_data += data

                    # command length exceeds our command length limit
                    if self.data_counter > self.max_command_length:
                        raise exceptionTools.InterruptError('InterruptError: command too long: %s, limit: %s' % (self.data_counter,self.max_command_length,))

                    while self.data_counter > 0:
                        if self.ssl:
                            x = ''
                            while self.request.pending():
                                x += self.request.recv(1024)
                        else:
                            x = self.request.recv(1024)
                        xlen = len(x)
                        self.data_counter -= xlen
                        self.buffered_data += x
                        if not xlen:
                            break

                    self.data_counter = None
                except ValueError:
                    #self.entropyTools.printTraceback()
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: %s, reason: %s - from client: %s' % (
                            self.server.server_address,
                            "malformed transmission",
                            self.client_address,
                        )
                    )
                    return True
                except self.socket.timeout, e:
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: %s, reason: %s - from client: %s' % (
                            self.server.server_address,
                            e,
                            self.client_address,
                        )
                    )
                    return True
                except self.socket.sslerror, e:
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: %s, SSL socket error reason: %s - from client: %s' % (
                            self.server.server_address,
                            e,
                            self.client_address,
                        )
                    )
                    return True
                except (self.ssl_exceptions['WantReadError'], self.ssl_exceptions['WantX509LookupError'],):
                    return False
                except self.ssl_exceptions['ZeroReturnError']:
                    return True
                except self.ssl_exceptions['Error'], e:
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: SSL Error, reason: %s - from client: %s' % (
                            e,
                            self.client_address,
                        )
                    )
                    return True
                except exceptionTools.InterruptError, e:
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: Command Error, reason: %s - from client: %s' % (
                            e,
                            self.client_address,
                        )
                    )
                    return True

                if not self.buffered_data:
                    return True

                cmd = self.server.processor.process(self.buffered_data, self.request, self.client_address)
                if cmd == 'close':
                    # send KAPUTT signal JA!
                    self.server.processor.transmit(self.server.processor.HostInterface.answers['cl'])
                    return True
                self.buffered_data = ''
                return False

        def handle(self):
            # not using spawnFunction because it causes some mess
            # forking this way avoids having memory leaks
            if self.server.processor.HostInterface.fork_requests:
                my_timeout = self.server.processor.HostInterface.fork_request_timeout_seconds
                pid = os.fork()
                seconds = 0
                if pid > 0:
                    # pid killer after timeout
                    passed_away = False
                    while 1:
                        time.sleep(1)
                        seconds += 1
                        try:
                            dead = os.waitpid(pid, os.WNOHANG)[0]
                        except OSError, e:
                            if e.errno != 10: raise
                            dead = True
                        if passed_away:
                            break
                        if dead: break
                        if seconds > my_timeout:
                            self.server.processor.HostInterface.updateProgress(
                                'interrupted: forked request timeout: %s,%s from client: %s' % (
                                    seconds,
                                    dead,
                                    self.client_address,
                                )
                            )
                            if not dead:
                                import signal
                                os.kill(pid,signal.SIGKILL)
                                passed_away = True # in this way, the process table should be clean
                                continue
                            break
                else:
                    self.do_handle()
                    os._exit(0)
            else:
                self.do_handle()
            #self.entropyTools.spawnFunction(self.do_handle)

        def do_handle(self):

            self.default_timeout = self.server.processor.HostInterface.timeout
            self.ssl = self.server.processor.HostInterface.SSL
            self.ssl_exceptions = self.server.processor.HostInterface.SSL_exceptions
            self.myeos = self.server.processor.HostInterface.answers['eos']
            self.max_command_length = self.server.processor.HostInterface.max_command_length

            while 1:

                try:
                    dobreak = self.data_receiver()
                    if dobreak: break
                except Exception, e:
                    self.server.processor.HostInterface.updateProgress(
                        'interrupted: Unhandled exception: %s, error: %s - from client: %s' % (
                            Exception,
                            e,
                            self.client_address,
                        )
                    )
                    # print exception
                    self.entropyTools.printTraceback()
                    self.entropyTools.printTraceback(f = self.server.processor.HostInterface.socketLog)
                    break

            self.request.close()

        def setup(self):
            self.data_counter = None
            self.buffered_data = ''


    class CommandProcessor:

        import entropyTools
        import socket
        import gc

        def __init__(self, HostInterface):
            self.HostInterface = HostInterface
            self.channel = None
            self.lastoutput = None

        def handle_termination_commands(self, data):
            if data.strip() in self.HostInterface.termination_commands:
                self.HostInterface.updateProgress('close: %s' % (self.client_address,))
                self.transmit(self.HostInterface.answers['cl'])
                return "close"

            if not data.strip():
                return "ignore"

        def handle_command_string(self, string):
            # validate command
            args = string.strip().split()
            session = args[0]
            if (session in self.HostInterface.initialization_commands) or len(args) < 2:
                cmd = args[0]
                session = None
            else:
                cmd = args[1]
                args = args[1:] # remove session

            stream_enabled = False
            if (session != None) and self.HostInterface.sessions.has_key(session):
                stream_enabled = self.HostInterface.sessions[session].get('stream_mode')

            if stream_enabled and (cmd not in self.HostInterface.config_commands):
                session_len = 0
                if session: session_len = len(session)+1
                return cmd,[string[session_len+len(cmd)+1:]],session
            else:
                myargs = []
                if len(args) > 1:
                    myargs = args[1:]

                return cmd,myargs,session

        def handle_end_answer(self, cmd, whoops, valid_cmd):
            if not valid_cmd:
                self.transmit(self.HostInterface.answers['no'])
            elif whoops:
                self.transmit(self.HostInterface.answers['er'])
            elif cmd not in self.HostInterface.no_acked_commands:
                self.transmit(self.HostInterface.answers['ok'])

        def validate_command(self, cmd, args, session):

            # answer to invalid commands
            if (cmd not in self.HostInterface.valid_commands):
                return False,"not a valid command"

            if session == None:
                if cmd not in self.HostInterface.no_session_commands:
                    return False,"need a valid session"
            elif session not in self.HostInterface.sessions:
                return False,"session is not alive"

            # check if command needs authentication
            if session != None:
                auth = self.HostInterface.valid_commands[cmd]['auth']
                if auth:
                    # are we?
                    authed = self.HostInterface.sessions[session]['auth_uid']
                    if authed == None:
                        # nope
                        return False,"not authenticated"

            # keep session alive
            if session != None:
                self.HostInterface.set_session_running(session)
                self.HostInterface.update_session_time(session)

            return True,"all good"

        def load_authenticator(self):
            f, args, kwargs = self.HostInterface.AuthenticatorInst
            myinst = f(*args,**kwargs)
            return myinst

        def load_service_interface(self, session):

            uid = None
            if session != None:
                uid = self.HostInterface.sessions[session]['auth_uid']

            intf = self.HostInterface.EntropyInstantiation[0]
            args = self.HostInterface.EntropyInstantiation[1]
            kwds = self.HostInterface.EntropyInstantiation[2]
            Entropy = intf(*args, **kwds)
            Entropy.urlFetcher = SocketUrlFetcher
            Entropy.updateProgress = self.remoteUpdateProgress
            try:
                Entropy.clientDbconn.updateProgress = self.remoteUpdateProgress
            except AttributeError:
                pass
            Entropy.progress = self.remoteUpdateProgress
            return Entropy

        def process(self, data, channel, client_address):

            self.channel = channel
            self.client_address = client_address

            term = self.handle_termination_commands(data)
            if term:
                del authenticator
                return term

            cmd, args, session = self.handle_command_string(data)
            valid_cmd, reason = self.validate_command(cmd, args, session)

            # decide if we need to load authenticator or Entropy
            authenticator = None
            cmd_data = self.HostInterface.valid_commands.get(cmd)
            if not isinstance(cmd_data,dict):
                self.HostInterface.updateProgress(
                    '[from: %s] command error: invalid command: %s' % (
                        self.client_address,
                        cmd,
                    )
                )
                return "close"
            elif (("authenticator" in cmd_data['args']) or (cmd in self.HostInterface.login_pass_commands)):
                try:
                    authenticator = self.load_authenticator()
                except exceptionTools.ConnectionError, e:
                    self.HostInterface.updateProgress(
                        '[from: %s] authenticator error: cannot load: %s' % (
                            self.client_address,
                            e,
                        )
                    )
                    self.entropyTools.printTraceback()
                    self.entropyTools.printTraceback(f = self.HostInterface.socketLog)
                    return "close"
                except Exception, e:
                    self.HostInterface.updateProgress(
                        '[from: %s] authenticator error: cannot load: %s - unknown error' % (
                            self.client_address,
                            e,
                        )
                    )
                    self.entropyTools.printTraceback()
                    self.entropyTools.printTraceback(f = self.HostInterface.socketLog)
                    return "close"

            p_args = args
            if (cmd in self.HostInterface.login_pass_commands) and authenticator != None:
                p_args = authenticator.hide_login_data(p_args)
            elif cmd in self.HostInterface.raw_commands:
                p_args = ['raw data']
            self.HostInterface.updateProgress(
                '[from: %s] command validation :: called %s: length: %s, args: %s, session: %s, valid: %s, reason: %s' % (
                    self.client_address,
                    cmd,
                    len(data),
                    p_args,
                    session,
                    valid_cmd,
                    reason,
                )
            )

            whoops = False
            if valid_cmd:

                if authenticator != None:
                    # now set session
                    authenticator.set_session(session)

                Entropy = None
                if "Entropy" in cmd_data['args']:
                    Entropy = self.load_service_interface(session)
                try:
                    self.run_task(cmd, args, session, Entropy, authenticator)
                except self.socket.timeout:
                    self.HostInterface.updateProgress(
                        '[from: %s] command error: timeout, closing connection' % (
                            self.client_address,
                        )
                    )
                    # close connection
                    del authenticator
                    del Entropy
                    return "close"
                except self.socket.error, e:
                    self.HostInterface.updateProgress(
                        '[from: %s] command error: socket error: %s' % (
                            self.client_address,
                            e,
                        )
                    )
                    # close connection
                    del authenticator
                    del Entropy
                    return "close"
                except self.HostInterface.SSL_exceptions['SysCallError'], e:
                    self.HostInterface.updateProgress(
                        '[from: %s] command error: SSL SysCallError: %s' % (
                            self.client_address,
                            e,
                        )
                    )
                    # close connection
                    del authenticator
                    del Entropy
                    return "close"
                except Exception, e:
                    # write to self.HostInterface.socketLog
                    self.entropyTools.printTraceback()
                    self.entropyTools.printTraceback(f = self.HostInterface.socketLog)
                    # store error
                    self.HostInterface.updateProgress(
                        '[from: %s] command error: %s, type: %s' % (
                            self.client_address,
                            e,
                            type(e),
                        )
                    )
                    if session != None:
                        self.HostInterface.store_rc(str(e),session)
                    whoops = True

                del Entropy

            if session != None:
                self.HostInterface.update_session_time(session)
                self.HostInterface.unset_session_running(session)
            rcmd = None
            try:
                self.handle_end_answer(cmd, whoops, valid_cmd)
            except (self.socket.error, self.socket.timeout,self.HostInterface.SSL_exceptions['SysCallError'],):
                rcmd = "close"

            if authenticator != None:
                authenticator.terminate_instance()
            del authenticator
            if not self.HostInterface.fork_requests:
                self.gc.collect()
            return rcmd

        def transmit(self, data):
            self.HostInterface.transmit(self.channel, data)

        def remoteUpdateProgress(
                self,
                text,
                header = "",
                footer = "",
                back = False,
                importance = 0,
                type = "info",
                count = [],
                percent = False
            ):
                if text != self.lastoutput:
                    text = chr(27)+"[2K\r"+text
                    if not back:
                        text += "\n"
                    self.transmit(text)
                self.lastoutput = text

        def run_task(self, cmd, args, session, Entropy, authenticator):

            p_args = args
            if cmd in self.HostInterface.login_pass_commands:
                p_args = authenticator.hide_login_data(p_args)
            elif cmd in self.HostInterface.raw_commands:
                p_args = ['raw data']
            self.HostInterface.updateProgress(
                '[from: %s] run_task :: called %s: args: %s, session: %s' % (
                    self.client_address,
                    cmd,
                    p_args,
                    session,
                )
            )

            myargs = args
            mykwargs = {}
            if cmd not in self.HostInterface.raw_commands:
                myargs, mykwargs = self._get_args_kwargs(args)

            rc = self.spawn_function(cmd, myargs, mykwargs, session, Entropy, authenticator)
            if session != None and self.HostInterface.sessions.has_key(session):
                self.HostInterface.store_rc(rc, session)
            return rc

        def _get_args_kwargs(self, args):
            myargs = []
            mykwargs = {}

            def is_int(x):
                try:
                    int(x)
                except ValueError:
                    return False
                return True

            for arg in args:
                if (arg.find("=") != -1) and not arg.startswith("="):
                    x = arg.split("=")
                    a = x[0]
                    b = ''.join(x[1:])
                    if (b in ("True","False",)) or is_int(b):
                        mykwargs[a] = eval(b)
                    else:
                        myargs.append(arg)
                else:
                    if (arg in ("True","False",)) or is_int(arg):
                        myargs.append(eval(arg))
                    else:
                        myargs.append(arg)
            return myargs, mykwargs

        def spawn_function(self, cmd, myargs, mykwargs, session, Entropy, authenticator):

            p_args = myargs
            if cmd in self.HostInterface.login_pass_commands:
                p_args = authenticator.hide_login_data(p_args)
            elif cmd in self.HostInterface.raw_commands:
                p_args = ['raw data']
            self.HostInterface.updateProgress(
                '[from: %s] called %s: args: %s, kwargs: %s' % (
                    self.client_address,
                    cmd,
                    p_args,
                    mykwargs,
                )
            )
            return self.do_spawn(cmd, myargs, mykwargs, session, Entropy, authenticator)

        def do_spawn(self, cmd, myargs, mykwargs, session, Entropy, authenticator):

            cmd_data = self.HostInterface.valid_commands.get(cmd)
            do_fork = cmd_data['as_user']
            f = cmd_data['cb']
            func_args = []
            for arg in cmd_data['args']:
                try:
                    func_args.append(eval(arg))
                except (NameError, SyntaxError):
                    func_args.append(str(arg))

            if do_fork:
                myfargs = func_args[:]
                myfargs.extend(myargs)
                return self.fork_task(f, session, authenticator, *myfargs, **mykwargs)
            else:
                return f(*func_args)

        def fork_task(self, f, session, authenticator, *args, **kwargs):
            gid = None
            uid = None
            if session != None:
                logged_in = self.HostInterface.sessions[session]['auth_uid']
                if logged_in != None:
                    uid = logged_in
                    gid = etpConst['entropygid']
            return self.entropyTools.spawnFunction(self._do_fork, f, authenticator, uid, gid, *args, **kwargs)

        def _do_fork(self, f, authenticator, uid, gid, *args, **kwargs):
            authenticator.set_exc_permissions(uid,gid)
            rc = f(*args,**kwargs)
            return rc

    class BuiltInCommands(SocketCommandsSkel):

        import dumpTools
        import zlib

        def __init__(self, HostInterface):

            SocketCommandsSkel.__init__(self, HostInterface, inst_name = "builtin")

            self.valid_commands = {
                'begin':    {
                                'auth': False, # does it need authentication ?
                                'built_in': True, # is it built-in ?
                                'cb': self.docmd_begin, # function to call
                                'args': ["self.transmit", "self.client_address"], # arguments to be passed before *args and **kwards, in SocketHostInterface.do_spawn()
                                'as_user': False, # do I have to fork the process and run it as logged user?
                                                  # needs auth = True
                                'desc': "instantiate a session", # description
                                'syntax': "begin", # syntax
                                'from': unicode(self), # from what class
                            },
                'end':      {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_end,
                                'args': ["self.transmit", "session"],
                                'as_user': False,
                                'desc': "end a session",
                                'syntax': "<SESSION_ID> end",
                                'from': unicode(self),
                            },
                'session_config':      {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_session_config,
                                'args': ["session","myargs"],
                                'as_user': False,
                                'desc': "set session configuration options",
                                'syntax': "<SESSION_ID> session_config <option> [parameters]",
                                'from': unicode(self),
                            },
                'rc':       {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_rc,
                                'args': ["self.transmit","session"],
                                'as_user': False,
                                'desc': "get data returned by the last valid command (streamed python object)",
                                'syntax': "<SESSION_ID> rc",
                                'from': unicode(self),
                            },
                'hello':    {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_hello,
                                'args': ["self.transmit"],
                                'as_user': False,
                                'desc': "get server status",
                                'syntax': "hello",
                                'from': unicode(self),
                            },
                'alive':    {
                                'auth': True,
                                'built_in': True,
                                'cb': self.docmd_alive,
                                'args': ["self.transmit","session"],
                                'as_user': False,
                                'desc': "check if a session is still alive",
                                'syntax': "<SESSION_ID> alive",
                                'from': unicode(self),
                            },
                'login':    {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_login,
                                'args': ["self.transmit", "authenticator", "session", "self.client_address", "myargs"],
                                'as_user': False,
                                'desc': "login on the running server (allows running extra commands)",
                                'syntax': "<SESSION_ID> login <authenticator parameters, default: <user> <auth_type> <password> >",
                                'from': unicode(self),
                            },
                'user_data':    {
                                'auth': True,
                                'built_in': True,
                                'cb': self.docmd_userdata,
                                'args': ["self.transmit", "authenticator", "session"],
                                'as_user': False,
                                'desc': "get general user information, user must be logged in",
                                'syntax': "<SESSION_ID> user_data",
                                'from': unicode(self),
                            },
                'logout':   {
                                'auth': True,
                                'built_in': True,
                                'cb': self.docmd_logout,
                                'args': ["self.transmit", "authenticator", "session", "self.client_address", "myargs"],
                                'as_user': False,
                                'desc': "logout on the running server",
                                'syntax': "<SESSION_ID> logout <USER>",
                                'from': unicode(self),
                            },
                'help':   {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_help,
                                'args': ["self.transmit"],
                                'as_user': False,
                                'desc': "this output",
                                'syntax': "help",
                                'from': unicode(self),
                            },
                'available_commands':   {
                                'auth': False,
                                'built_in': True,
                                'cb': self.docmd_available_commands,
                                'args': ["self.HostInterface"],
                                'as_user': False,
                                'desc': "get info about available commands (you must retrieve this using the 'rc' command)",
                                'syntax': "available_commands",
                                'from': unicode(self),
                            },
                'stream':   {
                                'auth': True,
                                'built_in': True,
                                'cb': self.docmd_stream,
                                'args': ["session", "myargs"],
                                'as_user': False,
                                'desc': "send a chunk of data to be saved on the session temp file path (will be removed on session expiration)",
                                'syntax': "<SESSION_ID> stream <chunk of byte-string to write to file>",
                                'from': unicode(self),
                            },
            }

            self.no_acked_commands = ["rc", "begin", "end", "hello", "alive", "login", "logout","help"]
            self.termination_commands = ["quit","close"]
            self.initialization_commands = ["begin"]
            self.login_pass_commands = ["login"]
            self.no_session_commands = ["begin","hello","alive","help"]
            self.raw_commands = ["stream"]
            self.config_commands = ["session_config"]

        def docmd_session_config(self, session, myargs):

            if not myargs:
                return False,"not enough parameters"

            option = myargs[0]
            myopts = myargs[1:]

            if option == "compression":
                docomp = True
                do_zlib = False
                if "zlib" in myopts:
                    do_zlib = True
                if myopts:
                    if isinstance(myopts[0],bool):
                        docomp = myopts[0]
                    else:
                        try:
                            docomp = eval(myopts[0])
                        except (NameError, TypeError,):
                            pass
                if docomp and do_zlib:
                    docomp = "zlib"
                elif docomp and not do_zlib:
                    docomp = "gzip"
                else:
                    docomp = None
                self.HostInterface.sessions[session]['compression'] = docomp
                return True,"compression now: %s" % (docomp,)
            elif option == "stream":
                dostream = True
                if "off" in myopts:
                    dostream = False
                self.HostInterface.sessions[session]['stream_mode'] = dostream
                return True,'stream mode: %s' % (dostream,)
            else:
                return False,"invalid config option"

        def docmd_available_commands(self, host_interface):

            def copy_obj(obj):
                if isinstance(obj,set) or isinstance(obj,dict):
                    return obj.copy()
                elif isinstance(obj,list) or isinstance(obj,tuple):
                    return obj[:]
                return obj

            def can_be_streamed(obj):
                if isinstance(obj,(bool,basestring,int,float,list,tuple,set,dict,)):
                    return True
                return False

            mydata = {}
            mydata['disabled_commands'] = copy_obj(host_interface.disabled_commands)
            valid_cmds = copy_obj(host_interface.valid_commands)
            mydata['valid_commands'] = {}
            for cmd in valid_cmds:
                mydict = {}
                for item in valid_cmds[cmd]:
                    param = valid_cmds[cmd][item]
                    if not can_be_streamed(param):
                        continue
                    mydict[item] = param
                mydata['valid_commands'][cmd] = mydict.copy()

            return mydata

        def docmd_stream(self, session, myargs):

            if not self.HostInterface.sessions[session]['stream_mode']:
                return False,'not in stream mode'
            if not myargs:
                return False,'no stream sent'

            compression = self.HostInterface.sessions[session]['compression']

            stream = myargs[0]
            stream_path = self.HostInterface.sessions[session]['stream_path']
            stream_dir = os.path.dirname(stream_path)
            if not os.path.isdir(os.path.dirname(stream_path)):
                try:
                    os.makedirs(stream_dir)
                    if etpConst['entropygid'] != None:
                        const_setup_perms(stream_dir,etpConst['entropygid'])
                except OSError:
                    return False,'cannot initialize stream directory'

            f = open(stream_path,'abw')
            if compression:
                stream = self.zlib.decompress(stream)
            f.write(stream)
            f.flush()
            f.close()

            return True,'ok'

        def docmd_login(self, transmitter, authenticator, session, client_address, myargs):

            # is already auth'd?
            auth_uid = self.HostInterface.sessions[session]['auth_uid']
            if auth_uid != None:
                return False,"already authenticated"

            status, user, uid, reason = authenticator.docmd_login(myargs)
            if status:
                self.HostInterface.updateProgress(
                    '[from: %s] user %s logged in successfully, session: %s' % (
                        client_address,
                        user,
                        session,
                    )
                )
                self.HostInterface.sessions[session]['auth_uid'] = uid
                transmitter(self.HostInterface.answers['ok'])
                return True,reason
            elif user == None:
                self.HostInterface.updateProgress(
                    '[from: %s] user -not specified- login failed, session: %s, reason: %s' % (
                        client_address,
                        session,
                        reason,
                    )
                )
                transmitter(self.HostInterface.answers['no'])
                return False,reason
            else:
                self.HostInterface.updateProgress(
                    '[from: %s] user %s login failed, session: %s, reason: %s' % (
                        client_address,
                        user,
                        session,
                        reason,
                    )
                )
                transmitter(self.HostInterface.answers['no'])
                return False,reason

        def docmd_userdata(self, transmitter, authenticator, session):

            auth_uid = self.HostInterface.sessions[session]['auth_uid']
            if auth_uid == None:
                return False,None,"not authenticated"

            return authenticator.docmd_userdata()

        def docmd_logout(self, transmitter, authenticator, session, client_address, myargs):
            status, user, reason = authenticator.docmd_logout(myargs)
            if status:
                self.HostInterface.updateProgress(
                    '[from: %s] user %s logged out successfully, session: %s, args: %s ' % (
                        client_address,
                        user,
                        session,
                        myargs,
                    )
                )
                self.HostInterface.sessions[session]['auth_uid'] = None
                transmitter(self.HostInterface.answers['ok'])
                return True,reason
            elif user == None:
                self.HostInterface.updateProgress(
                    '[from: %s] user -not specified- logout failed, session: %s, args: %s, reason: %s' % (
                        client_address,
                        session,
                        myargs,
                        reason,
                    )
                )
                transmitter(self.HostInterface.answers['no'])
                return False,reason
            else:
                self.HostInterface.updateProgress(
                    '[from: %s] user %s logout failed, session: %s, args: %s, reason: %s' % (
                        client_address,
                        user,
                        session,
                        myargs,
                        reason,
                    )
                )
                transmitter(self.HostInterface.answers['no'])
                return False,reason

        def docmd_alive(self, transmitter, session):
            cmd = self.HostInterface.answers['no']
            if session in self.HostInterface.sessions:
                cmd = self.HostInterface.answers['ok']
            transmitter(cmd)

        def docmd_hello(self, transmitter):
            uname = os.uname()
            kern_string = uname[2]
            running_host = uname[1]
            running_arch = uname[4]
            load_stats = commands.getoutput('uptime').split("\n")[0]
            text = "Entropy Server %s, connections: %s ~ running on: %s ~ host: %s ~ arch: %s, kernel: %s, stats: %s\n" % (
                    etpConst['entropyversion'],
                    self.HostInterface.connections,
                    etpConst['systemname'],
                    running_host,
                    running_arch,
                    kern_string,
                    load_stats
                    )
            transmitter(text)

        def docmd_help(self, transmitter):
            text = '\nEntropy Socket Interface Help Menu\n' + \
                   'Available Commands:\n\n'
            valid_cmds = sorted(self.HostInterface.valid_commands.keys())
            for cmd in valid_cmds:
                if self.HostInterface.valid_commands[cmd].has_key('desc'):
                    desc = self.HostInterface.valid_commands[cmd]['desc']
                else:
                    desc = 'no description available'

                if self.HostInterface.valid_commands[cmd].has_key('syntax'):
                    syntax = self.HostInterface.valid_commands[cmd]['syntax']
                else:
                    syntax = 'no syntax available'
                if self.HostInterface.valid_commands[cmd].has_key('from'):
                    myfrom = self.HostInterface.valid_commands[cmd]['from']
                else:
                    myfrom = 'N/A'
                text += "[%s] %s\n   %s: %s\n   %s: %s\n" % (
                    myfrom,
                    blue(cmd),
                    red("description"),
                    desc.strip(),
                    darkgreen("syntax"),
                    syntax,
                )
            transmitter(text)

        def docmd_end(self, transmitter, session):
            rc = self.HostInterface.destroy_session(session)
            cmd = self.HostInterface.answers['no']
            if rc: cmd = self.HostInterface.answers['ok']
            transmitter(cmd)
            return rc

        def docmd_begin(self, transmitter, client_address):
            session = self.HostInterface.get_new_session(client_address[0])
            transmitter(session)
            return session

        def docmd_rc(self, transmitter, session):
            rc = self.HostInterface.get_rc(session)
            comp = self.HostInterface.sessions[session]['compression']
            myserialized = self.dumpTools.serialize_string(rc)
            if comp == "zlib": # new shiny zlib
                myserialized = self.zlib.compress(myserialized, 7) # compression level 1-9
            elif comp == "gzip": # old and burried
                import gzip
                try:
                    import cStringIO as stringio
                except ImportError:
                    import StringIO as stringio
                f = stringio.StringIO()
                self.dumpTools.serialize(rc, f)
                myf = stringio.StringIO()
                mygz = gzip.GzipFile(
                    mode = 'wb',
                    fileobj = myf
                )
                f.seek(0)
                chunk = f.read(8192)
                while chunk:
                    mygz.write(chunk)
                    chunk = f.read(8192)
                mygz.flush()
                mygz.close()
                myserialized = myf.getvalue()
                f.close()
                myf.close()


            transmitter(myserialized)

            return rc

    import gc, threading
    def __init__(self, service_interface, *args, **kwds):

        self.Server = None
        self.Gc = None
        self.PythonGarbageCollector = None
        self.AuthenticatorInst = None

        self.args = args
        self.kwds = kwds
        self.socketLog = LogFile(
            level = etpConst['socketloglevel'],
            filename = etpConst['socketlogfile'],
            header = "[Socket]"
        )

        # settings
        self.SessionsLock = self.threading.Lock()
        self.fork_requests = True # used by the command processor
        self.fork_request_timeout_seconds = etpConst['socket_service']['forked_requests_timeout']
        self.stdout_logging = True
        self.timeout = etpConst['socket_service']['timeout']
        self.hostname = etpConst['socket_service']['hostname']
        self.session_ttl = etpConst['socket_service']['session_ttl']
        if self.hostname == "*": self.hostname = ''
        self.port = etpConst['socket_service']['port']
        self.threads = etpConst['socket_service']['threads'] # maximum number of allowed sessions
        self.max_connections = etpConst['socket_service']['max_connections']
        self.max_connections_per_host = etpConst['socket_service']['max_connections_per_host']
        self.max_connections_per_host_barrier = etpConst['socket_service']['max_connections_per_host_barrier']
        self.max_command_length = etpConst['socket_service']['max_command_length']
        self.disabled_commands = etpConst['socket_service']['disabled_cmds']
        self.ip_blacklist = etpConst['socket_service']['ip_blacklist']
        self.connections = 0
        self.per_host_connections = {}
        self.sessions = {}
        self.answers = etpConst['socket_service']['answers']
        self.__output = None
        self.SSL = {}
        self.SSL_exceptions = {}
        self.SSL_exceptions['WantReadError'] = None
        self.SSL_exceptions['WantWriteError'] = None
        self.SSL_exceptions['WantX509LookupError'] = None
        self.SSL_exceptions['ZeroReturnError'] = None
        self.SSL_exceptions['SysCallError'] = None
        self.SSL_exceptions['Error'] = []
        self.last_print = ''
        self.valid_commands = {}
        self.no_acked_commands = []
        self.raw_commands = []
        self.config_commands = []
        self.termination_commands = []
        self.initialization_commands = []
        self.login_pass_commands = []
        self.no_session_commands = []
        self.command_classes = [self.BuiltInCommands]
        self.command_instances = []
        self.EntropyInstantiation = (service_interface, self.args, self.kwds)

        self.setup_external_command_classes()
        self.start_local_output_interface()
        self.setup_authenticator()
        self.setup_hostname()
        self.setup_commands()
        self.disable_commands()
        self.start_session_garbage_collector()
        self.setup_ssl()
        self.start_python_garbage_collector()

    def killall(self):
        if hasattr(self,'socketLog'):
            self.socketLog.close()
        if self.Server != None:
            self.Server.alive = False
        if self.Gc != None:
            self.Gc.kill()
            try:
                self.Gc.nuke()
            except exceptionTools.InterruptError:
                pass
        if self.PythonGarbageCollector != None:
            self.PythonGarbageCollector.kill()
            try:
                self.PythonGarbageCollector.nuke()
            except exceptionTools.InterruptError:
                pass

    def append_eos(self, data):
        return str(len(data)) + \
            self.answers['eos'] + \
                data

    def setup_ssl(self):

        do_ssl = False
        if self.kwds.has_key('ssl'):
            do_ssl = self.kwds.pop('ssl')

        if not do_ssl:
            return

        try:
            from OpenSSL import SSL, crypto
        except ImportError, e:
            self.updateProgress('Unable to load OpenSSL, error: %s' % (repr(e),))
            return
        self.SSL_exceptions['WantReadError'] = SSL.WantReadError
        self.SSL_exceptions['Error'] = SSL.Error
        self.SSL_exceptions['WantWriteError'] = SSL.WantWriteError
        self.SSL_exceptions['WantX509LookupError'] = SSL.WantX509LookupError
        self.SSL_exceptions['ZeroReturnError'] = SSL.ZeroReturnError
        self.SSL_exceptions['SysCallError'] = SSL.SysCallError
        self.SSL['m'] = SSL
        self.SSL['crypto'] = crypto
        self.SSL['key'] = etpConst['socket_service']['ssl_key']
        self.SSL['cert'] = etpConst['socket_service']['ssl_cert']
        self.SSL['ca_cert'] = etpConst['socket_service']['ssl_ca_cert']
        self.SSL['ca_pkey'] = etpConst['socket_service']['ssl_ca_pkey']
        # change port
        self.port = etpConst['socket_service']['ssl_port']
        self.SSL['not_before'] = 0
        self.SSL['not_after'] = 60*60*24*365*5 # 5 years
        self.SSL['serial'] = 0
        self.SSL['digest'] = 'md5'

        if not (os.path.isfile(self.SSL['ca_cert']) and \
            os.path.isfile(self.SSL['ca_pkey']) and \
            os.path.isfile(self.SSL['key']) and \
            os.path.isfile(self.SSL['cert'])):
                self.create_ca_server_certs(
                    self.SSL['serial'],
                    self.SSL['digest'],
                    self.SSL['not_before'],
                    self.SSL['not_after'],
                    self.SSL['ca_pkey'],
                    self.SSL['ca_cert'],
                    self.SSL['key'],
                    self.SSL['cert']
                )
                os.chmod(self.SSL['ca_cert'],0644)
                try:
                    os.chown(self.SSL['ca_cert'],-1,0)
                except OSError:
                    pass
                os.chmod(self.SSL['ca_pkey'],0600)
                try:
                    os.chown(self.SSL['ca_pkey'],-1,0)
                except OSError:
                    pass

        os.chmod(self.SSL['key'],0600)
        try:
            os.chown(self.SSL['key'],-1,0)
        except OSError:
            pass
        os.chmod(self.SSL['cert'],0644)
        try:
            os.chown(self.SSL['cert'],-1,0)
        except OSError:
            pass

    def create_ca_server_certs(self, serial, digest, not_before, not_after, ca_pkey_dest, ca_cert_dest, server_key, server_cert):

        mycn = 'Entropy Repository Service'
        cakey = self.create_ssl_key_pair(self.SSL['crypto'].TYPE_RSA, 1024)
        careq = self.create_ssl_certificate_request(cakey, digest, CN = mycn)
        cert = self.SSL['crypto'].X509()
        cert.set_serial_number(serial)
        cert.gmtime_adj_notBefore(not_before)
        cert.gmtime_adj_notAfter(not_after)
        cert.set_issuer(careq.get_subject())
        cert.set_subject(careq.get_subject())
        cert.sign(cakey, digest)

        # now create server key + cert
        s_pkey = self.create_ssl_key_pair(self.SSL['crypto'].TYPE_RSA, 1024)
        s_req = self.create_ssl_certificate_request(s_pkey, digest, CN = mycn)
        s_cert = self.SSL['crypto'].X509()
        s_cert.set_serial_number(serial+1)
        s_cert.gmtime_adj_notBefore(not_before)
        s_cert.gmtime_adj_notAfter(not_after)
        s_cert.set_issuer(cert.get_subject())
        s_cert.set_subject(s_req.get_subject())
        s_cert.set_pubkey(s_req.get_pubkey())
        s_cert.sign(cakey, digest)

        # write CA
        if os.path.isfile(ca_pkey_dest):
            shutil.move(ca_pkey_dest,ca_pkey_dest+".moved")
        f = open(ca_pkey_dest,"w")
        f.write(self.SSL['crypto'].dump_privatekey(self.SSL['crypto'].FILETYPE_PEM, cakey))
        f.flush()
        f.close()
        if os.path.isfile(ca_cert_dest):
            shutil.move(ca_cert_dest,ca_cert_dest+".moved")
        f = open(ca_cert_dest,"w")
        f.write(self.SSL['crypto'].dump_certificate(self.SSL['crypto'].FILETYPE_PEM, cert))
        f.flush()
        f.close()

        if os.path.isfile(server_key):
            shutil.move(server_key,server_key+".moved")
        # write server
        f = open(server_key,"w")
        f.write(self.SSL['crypto'].dump_privatekey(self.SSL['crypto'].FILETYPE_PEM, s_pkey))
        f.flush()
        f.close()
        if os.path.isfile(server_cert):
            shutil.move(server_cert,server_cert+".moved")
        f = open(server_cert,"w")
        f.write(self.SSL['crypto'].dump_certificate(self.SSL['crypto'].FILETYPE_PEM, s_cert))
        f.flush()
        f.close()

    def create_ssl_key_pair(self, keytype, bits):
        pkey = self.SSL['crypto'].PKey()
        pkey.generate_key(keytype, bits)
        return pkey

    def create_ssl_certificate_request(self, pkey, digest, **name):
        req = self.SSL['crypto'].X509Req()
        subj = req.get_subject()
        for (key,value) in name.items():
            setattr(subj, key, value)
        req.set_pubkey(pkey)
        req.sign(pkey, digest)
        return req

    def setup_external_command_classes(self):

        if self.kwds.has_key('external_cmd_classes'):
            ext_commands = self.kwds.pop('external_cmd_classes')
            if not isinstance(ext_commands,list):
                raise exceptionTools.InvalidDataType("InvalidDataType: external_cmd_classes must be a list")
            self.command_classes += ext_commands

    def setup_commands(self):

        identifiers = set()
        for myclass in self.command_classes:

            myargs = []
            mykwargs = {}
            if isinstance(myclass,tuple) or isinstance(myclass,list):
                if len(myclass) > 2:
                    mykwargs = myclass[2]
                if len(myclass) > 1:
                    myargs = myclass[1]
                myclass = myclass[0]

            myinst = myclass(self, *myargs, **mykwargs)
            if str(myinst) in identifiers:
                raise exceptionTools.PermissionDenied("PermissionDenied: another command instance is owning this name")
            identifiers.add(str(myinst))
            self.command_instances.append(myinst)
            # now register
            myinst.register(    self.valid_commands,
                                self.no_acked_commands,
                                self.termination_commands,
                                self.initialization_commands,
                                self.login_pass_commands,
                                self.no_session_commands,
                                self.raw_commands,
                                self.config_commands
                            )

    def disable_commands(self):
        for cmd in self.disabled_commands:

            if cmd in self.valid_commands:
                self.valid_commands.pop(cmd)

            if cmd in self.no_acked_commands:
                self.no_acked_commands.remove(cmd)

            if cmd in self.termination_commands:
                self.termination_commands.remove(cmd)

            if cmd in self.initialization_commands:
                self.initialization_commands.remove(cmd)

            if cmd in self.login_pass_commands:
                self.login_pass_commands.remove(cmd)

            if cmd in self.no_session_commands:
                self.no_session_commands.remove(cmd)

            if cmd in self.raw_commands:
                self.raw_commands.remove(cmd)

            if cmd in self.config_commands:
                self.config_commands.remove(cmd)

    def start_local_output_interface(self):
        if self.kwds.has_key('sock_output'):
            outputIntf = self.kwds.pop('sock_output')
            self.__output = outputIntf

    def setup_authenticator(self):

        # lock, if perhaps some implementations need it
        self.AuthenticatorLock = self.threading.Lock()
        auth_inst = (self.BasicPamAuthenticator, [self], {},) # authentication class, args, keywords
        # external authenticator
        if self.kwds.has_key('sock_auth'):
            authIntf = self.kwds.pop('sock_auth')
            if type(authIntf) is tuple:
                if len(authIntf) == 3:
                    auth_inst = authIntf[:]
                else:
                    raise exceptionTools.IncorrectParameter("IncorrectParameter: wront authentication interface specified")
            else:
                raise exceptionTools.IncorrectParameter("IncorrectParameter: wront authentication interface specified")
            # initialize authenticator
        self.AuthenticatorInst = (auth_inst[0],[self]+auth_inst[1],auth_inst[2],)

    def start_python_garbage_collector(self):
        self.PythonGarbageCollector = self.entropyTools.TimeScheduled( self.python_garbage_collect, 3600 )
        self.PythonGarbageCollector.setName("Garbage_Collector::"+str(abs(hash(os.urandom(20)))))
        self.PythonGarbageCollector.exc = exceptionTools.InterruptError('InterruptError: interrupted')
        self.PythonGarbageCollector.accurate = False
        self.PythonGarbageCollector.start()

    def start_session_garbage_collector(self):
        self.Gc = self.entropyTools.TimeScheduled( self.gc_clean, 5 )
        self.Gc.setName("Socket_GC::"+str(abs(hash(os.urandom(20)))))
        self.Gc.exc = exceptionTools.InterruptError('InterruptError: interrupted')
        self.Gc.start()

    def python_garbage_collect(self):
        self.gc.collect()
        self.gc.collect()
        self.gc.collect()

    def gc_clean(self):
        if not self.sessions:
            return

        self.SessionsLock.acquire()
        try:
            for session_id in self.sessions.keys():
                sess_time = self.sessions[session_id]['t']
                is_running = self.sessions[session_id]['running']
                auth_uid = self.sessions[session_id]['auth_uid'] # is kept alive?
                if (is_running) or (auth_uid == -1):
                    if auth_uid == -1:
                        self.updateProgress('not killing session %s, since it is kept alive by auth_uid=-1' % (session_id,) )
                    continue
                cur_time = time.time()
                ttl = self.session_ttl
                check_time = sess_time + ttl
                if cur_time > check_time:
                    self.updateProgress('killing session %s, ttl: %ss: no activity' % (session_id,ttl,) )
                    self.destroy_session(session_id, lock = False)
        finally:
            self.SessionsLock.release()

    def setup_hostname(self):
        if self.hostname:
            try:
                self.hostname = self.get_ip_address(self.hostname)
            except IOError: # it isn't a device name
                pass

    def get_ip_address(self, ifname):
        import fcntl
        import struct
        mysock = self.socket.socket ( self.socket.AF_INET, self.socket.SOCK_STREAM )
        return self.socket.inet_ntoa(fcntl.ioctl(mysock.fileno(), 0x8915, struct.pack('256s', ifname[:15]))[20:24])

    def get_new_session(self, ip_address = None):
        self.SessionsLock.acquire()
        try:
            if len(self.sessions) > self.threads:
                # fuck!
                return "0"
            rng = str(abs(hash(os.urandom(20))))
            while rng in self.sessions:
                rng = str(abs(hash(os.urandom(20))))
            self.sessions[rng] = {}
            self.sessions[rng]['running'] = False
            self.sessions[rng]['auth_uid'] = None
            self.sessions[rng]['admin'] = False
            self.sessions[rng]['moderator'] = False
            self.sessions[rng]['user'] = False
            self.sessions[rng]['developer'] = False
            self.sessions[rng]['compression'] = None
            self.sessions[rng]['stream_mode'] = False
            try:
                self.sessions[rng]['stream_path'] = self.entropyTools.getRandomTempFile()
            except (IOError,OSError,):
                self.sessions[rng]['stream_path'] = ''
            self.sessions[rng]['t'] = time.time()
            self.sessions[rng]['ip_address'] = ip_address
            return rng
        finally:
            self.SessionsLock.release()

    def update_session_time(self, session):
        self.SessionsLock.acquire()
        try:
            if self.sessions.has_key(session):
                self.sessions[session]['t'] = time.time()
                self.updateProgress('session time updated for %s' % (session,) )
        finally:
            self.SessionsLock.release()

    def set_session_running(self, session):
        self.SessionsLock.acquire()
        try:
            if self.sessions.has_key(session):
                self.sessions[session]['running'] = True
        finally:
            self.SessionsLock.release()

    def unset_session_running(self, session):
        self.SessionsLock.acquire()
        try:
            if self.sessions.has_key(session):
                self.sessions[session]['running'] = False
        finally:
            self.SessionsLock.release()

    def destroy_session(self, session, lock = True):
        if lock:
            self.SessionsLock.acquire()
        try:
            if self.sessions.has_key(session):
                stream_path = self.sessions[session]['stream_path']
                del self.sessions[session]
                if os.path.isfile(stream_path) and os.access(stream_path,os.W_OK) and not os.path.islink(stream_path):
                    try:
                        os.remove(stream_path)
                    except OSError:
                        pass
                return True
            return False
        finally:
            if lock: self.SessionsLock.release()

    def go(self):
        self.socket.setdefaulttimeout(self.timeout)
        while 1:
            try:
                self.Server = self.HostServer(
                                                (self.hostname, self.port),
                                                self.RequestHandler,
                                                self.CommandProcessor(self),
                                                self
                                            )
                break
            except self.socket.error, e:
                if e[0] == 98:
                    # Address already in use
                    self.updateProgress('address already in use (%s, port: %s), waiting 5 seconds...' % (self.hostname,self.port,))
                    time.sleep(5)
                    continue
                else:
                    raise
        self.updateProgress('server connected, listening on: %s, port: %s, timeout: %s' % (self.hostname,self.port,self.timeout,))
        self.Server.serve_forever()
        self.Gc.kill()

    def store_rc(self, rc, session):
        self.SessionsLock.acquire()
        try:
            if session in self.sessions:
                if type(rc) in (list,tuple,):
                    rc_item = rc[:]
                elif type(rc) in (set,frozenset,dict,):
                    rc_item = rc.copy()
                else:
                    rc_item = rc
                self.sessions[session]['rc'] = rc_item
        finally:
            self.SessionsLock.release()

    def get_rc(self, session):
        self.SessionsLock.acquire()
        try:
            if session in self.sessions:
                return self.sessions[session].get('rc')
        finally:
            self.SessionsLock.release()

    def transmit(self, channel, data):
        if self.SSL:
            mydata = self.append_eos(data)
            encode_done = False
            while 1:
                try:
                    sent = channel.send(mydata)
                    if sent == len(mydata):
                        break
                    mydata = mydata[sent:]
                except (self.SSL_exceptions['WantWriteError'],self.SSL_exceptions['WantReadError']):
                    time.sleep(0.2)
                    continue
                except UnicodeEncodeError:
                    if encode_done:
                        raise
                    mydata = mydata.encode('utf-8')
                    encode_done = True
                    continue
        else:
            channel.sendall(self.append_eos(data))

    def updateProgress(self, *args, **kwargs):
        message = args[0]
        if message != self.last_print:
            self.socketLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,str(args[0]))
            if self.__output != None and self.stdout_logging:
                self.__output.updateProgress(*args,**kwargs)
            self.last_print = message

class SocketUrlFetcher(urlFetcher):

    import entropyTools
    # reimplementing updateProgress
    def updateProgress(self):
        kbprogress = " (%s/%s kB @ %s)" % (
                                        str(round(float(self.downloadedsize)/1024,1)),
                                        str(round(self.remotesize,1)),
                                        str(self.entropyTools.bytesIntoHuman(self.datatransfer))+"/sec",
                                    )
        mytxt = _("Fetch")
        self.progress( mytxt+": "+str((round(float(self.average),1)))+"%"+kbprogress, back = True )


class ServerInterface(TextInterface):

    def __init__(self, default_repository = None, save_repository = False, community_repo = False):

        if etpConst['uid'] != 0:
            mytxt = _("Entropy ServerInterface must be run as root")
            raise exceptionTools.PermissionDenied("PermissionDenied: %s" % (mytxt,))

        self.serverLog = LogFile(
            level = etpConst['entropyloglevel'],
            filename = etpConst['entropylogfile'],
            header = "[server]"
        )

        self.default_repository = default_repository
        if self.default_repository == None:
            self.default_repository = etpConst['officialserverrepositoryid']

        if self.default_repository in etpConst['server_repositories']:
            self.ensure_paths(self.default_repository)
        self.migrate_repository_databases_to_new_branched_path()
        self.community_repo = community_repo
        self.dbapi2 = dbapi2 # export for third parties
        # settings
        etpSys['serverside'] = True
        self.indexing = False
        self.xcache = False
        self.MirrorsService = None
        self.FtpInterface = FtpInterface
        self.rssFeed = rssFeed
        self.serverDbCache = {}
        self.repository_treeupdate_digests = {}
        self.package_match_validator_cache = {}
        self.settings_to_backup = []
        self.do_save_repository = save_repository
        self.rssMessages = {
            'added': {},
            'removed': {},
            'commitmessage': "",
            'light': {},
        }


        if self.default_repository not in etpConst['server_repositories']:
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (
                        self.default_repository,
                        _("repository not configured"),
                    )
            )
        if etpConst['clientserverrepoid'] in etpConst['server_repositories']:
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (
                        etpConst['clientserverrepoid'],
                        _("protected repository id, can't use this, sorry dude..."),
                    )
            )

        if self.community_repo:
            self.add_client_database_to_repositories()
        self.switch_default_repository(self.default_repository)

    def destroy(self):
        if hasattr(self,'serverLog'):
            self.serverLog.close()
        if hasattr(self,'ClientService'):
            self.ClientService.destroy()
        self.close_server_databases()

    def __del__(self):
        self.destroy()

    def ensure_paths(self, repo):
        upload_dir = os.path.join(self.get_local_upload_directory(repo),etpConst['branch'])
        db_dir = self.get_local_database_dir(repo)
        for mydir in [upload_dir,db_dir]:
            if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
                os.makedirs(mydir)
                const_setup_perms(mydir,etpConst['entropygid'])


    # FIXME: this will be removed in future, creation date: 2008-10-08
    def migrate_repository_databases_to_new_branched_path(self):
        migrated_filename = '.branch_migrated'
        for repoid in etpConst['server_repositories'].keys():

            if repoid == etpConst['clientserverrepoid']: continue
            mydir = etpConst['server_repositories'][repoid]['database_dir']
            if not os.path.isdir(mydir): # empty ?
                continue

            migrated_filepath = os.path.join(mydir,migrated_filename)
            if os.path.isfile(migrated_filepath):
                continue

            my_branched_dir = self.get_local_database_dir(repoid)
            if os.path.isdir(my_branched_dir): # wtf? do not touch
                continue

            self.updateProgress(
                "[%s:%s] %s: %s, %s: %s" % (
                        brown("repo"),
                        purple(repoid),
                        _("migrating database path from"),
                        brown(mydir),
                        _('to'),
                        brown(my_branched_dir),
                ),
                importance = 1,
                type = "info",
                header = bold(" @@ ")
            )

            repo_files = [os.path.join(mydir,x) for x in os.listdir(mydir) if \
                (os.path.isfile(os.path.join(mydir,x)) and \
                os.access(os.path.join(mydir,x),os.W_OK))
            ]
            os.makedirs(my_branched_dir)
            const_setup_perms(my_branched_dir,etpConst['entropygid'])

            for repo_file in repo_files:
                repo_filename = os.path.basename(repo_file)
                shutil.move(repo_file,os.path.join(my_branched_dir,repo_filename))

            f = open(migrated_filepath,"w")
            f.write("done\n")
            f.flush()
            f.close()

    def add_client_database_to_repositories(self):
        etpConst['server_repositories'][etpConst['clientserverrepoid']] = {}
        mydata = {}
        mydata['description'] = "Community Repositories System Database"
        mydata['mirrors'] = []
        mydata['community'] = False
        etpConst['server_repositories'][etpConst['clientserverrepoid']].update(mydata)

    def setup_services(self):
        self.setup_entropy_settings()
        if hasattr(self,'ClientService'):
            self.ClientService.destroy()
        self.ClientService = EquoInterface(
            indexing = self.indexing,
            xcache = self.xcache,
            repo_validation = False,
            noclientdb = 1
        )
        self.Cacher = self.ClientService.Cacher
        self.ClientService.updateProgress = self.updateProgress
        self.ClientService.FtpInterface = self.FtpInterface
        self.SystemSettings = self.ClientService.SystemSettings
        self.validRepositories = self.ClientService.validRepositories
        self.entropyTools = self.ClientService.entropyTools
        self.dumpTools = self.ClientService.dumpTools
        self.QA = self.ClientService.QA
        self.backup_entropy_settings()
        self.SpmService = self.ClientService.Spm()
        self.MirrorsService = ServerMirrorsInterface(self)

    def setup_entropy_settings(self, repo = None):
        backup_list = [
            'etpdatabaseclientfilepath',
            'clientdbid',
            'officialserverrepositoryid'
        ]
        for setting in backup_list:
            if setting not in self.settings_to_backup:
                self.settings_to_backup.append(setting)
        # setup client database
        if not self.community_repo:
            etpConst['etpdatabaseclientfilepath'] = self.get_local_database_file(repo)
            etpConst['clientdbid'] = etpConst['serverdbid']
        const_createWorkingDirectories()

    def close_server_databases(self):
        if hasattr(self,'serverDbCache'):
            for item in self.serverDbCache:
                try:
                    self.serverDbCache[item].closeDB()
                except self.dbapi2.ProgrammingError: # already closed?
                    pass
        self.serverDbCache.clear()

    def close_server_database(self, dbinstance):
        found = None
        for item in self.serverDbCache:
            if dbinstance == self.serverDbCache[item]:
                found = item
                break
        if found:
            instance = self.serverDbCache.pop(found)
            instance.closeDB()

    def get_available_repositories(self):
        return etpConst['server_repositories'].copy()

    def switch_default_repository(self, repoid, save = None, handle_uninitialized = True):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return

        if save == None:
            save = self.do_save_repository
        if repoid not in etpConst['server_repositories']:
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (
                        repoid,
                        _("repository not configured"),
                    )
            )
        self.close_server_databases()
        etpConst['officialserverrepositoryid'] = repoid
        self.default_repository = repoid
        self.setup_services()
        if save:
            self.save_default_repository(repoid)

        self.setup_community_repositories_settings()
        self.show_interface_status()
        if handle_uninitialized:
            self.handle_uninitialized_repository(repoid)

    def setup_community_repositories_settings(self):
        if self.community_repo:
            for repoid in etpConst['server_repositories']:
                etpConst['server_repositories'][repoid]['community'] = True


    def handle_uninitialized_repository(self, repoid):
        if not self.is_repository_initialized(repoid):
            mytxt = blue("%s.") % (_("Your default repository is not initialized"),)
            self.updateProgress(
                "[%s:%s] %s" % (
                        brown("repo"),
                        purple(repoid),
                        mytxt,
                ),
                importance = 1,
                type = "warning",
                header = darkred(" !!! ")
            )
            answer = self.askQuestion(_("Do you want to initialize your default repository ?"))
            if answer == "No":
                mytxt = red("%s.") % (_("You have taken the risk to continue with an uninitialized repository"),)
                self.updateProgress(
                    "[%s:%s] %s" % (
                            brown("repo"),
                            purple(repoid),
                            mytxt,
                    ),
                    importance = 1,
                    type = "warning",
                    header = darkred(" !!! ")
                )
            else:
                # move empty database for security sake
                dbfile = self.get_local_database_file(repoid)
                if os.path.isfile(dbfile):
                    shutil.move(dbfile,dbfile+".backup")
                self.initialize_server_database(empty = True, repo = repoid, warnings = False)


    def show_interface_status(self):
        type_txt = _("server-side repository")
        if self.community_repo:
            type_txt = _("community repository")
        mytxt = _("Entropy Server Interface Instance on repository") # ..on repository: <repository_name>
        self.updateProgress(
            blue("%s: %s, %s: %s (%s: %s)" % (
                    mytxt,
                    red(self.default_repository),
                    _("current branch"),
                    darkgreen(etpConst['branch']),
                    purple(_("type")),
                    bold(type_txt),
                )
            ),
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )
        repos = etpConst['server_repositories'].keys()
        mytxt = blue("%s:") % (_("Currently configured repositories"),) # ...: <list>
        self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red(" @@ ")
        )
        for repo in repos:
            self.updateProgress(
                darkgreen(repo),
                importance = 0,
                type = "info",
                header = brown("   # ")
            )


    def save_default_repository(self, repoid):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return

        if os.path.isfile(etpConst['serverconf']):
            f = open(etpConst['serverconf'],"r")
            content = f.readlines()
            f.close()
            content = [x.strip() for x in content]
            found = False
            new_content = []
            for line in content:
                if line.strip().startswith("officialserverrepositoryid|"):
                    line = "officialserverrepositoryid|%s" % (repoid,)
                    found = True
                new_content.append(line)
            if not found:
                new_content.append("officialserverrepositoryid|%s" % (repoid,))
            f = open(etpConst['serverconf']+".save_default_repo_tmp","w")
            for line in new_content:
                f.write(line+"\n")
            f.flush()
            f.close()
            shutil.move(etpConst['serverconf']+".save_default_repo_tmp",etpConst['serverconf'])
        else:
            f = open(etpConst['serverconf'],"w")
            f.write("officialserverrepositoryid|%s\n" % (repoid,))
            f.flush()
            f.close()

    def toggle_repository(self, repoid, enable = True):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return False

        if not os.path.isfile(etpConst['serverconf']):
            return None
        f = open(etpConst['serverconf'])
        tmpfile = etpConst['serverconf']+".switch"
        mycontent = [x.strip() for x in f.readlines()]
        f.close()
        f = open(tmpfile,"w")
        st = "repository|%s" % (repoid,)
        status = False
        for line in mycontent:
            if enable:
                if (line.find(st) != -1) and line.startswith("#") and (len(line.split("|")) == 5):
                    line = line[1:]
                    status = True
            else:
                if (line.find(st) != -1) and not line.startswith("#") and (len(line.split("|")) == 5):
                    line = "#"+line
                    status = True
            f.write(line+"\n")
        f.flush()
        f.close()
        shutil.move(tmpfile,etpConst['serverconf'])
        if status:
            self.close_server_databases()
            const_readServerSettings()
            self.setup_services()
            self.show_interface_status()
        return status

    def backup_entropy_settings(self):
        for setting in self.settings_to_backup:
            self.ClientService.backup_setting(setting)

    def is_repository_initialized(self, repo):

        def do_validate(dbc):
            try:
                dbc.validateDatabase()
                return True
            except exceptionTools.SystemDatabaseError:
                return False

        dbc = self.openServerDatabase(just_reading = True, repo = repo)
        valid = do_validate(dbc)
        self.close_server_database(dbc)
        if not valid: # check online?
            dbc = self.openServerDatabase(read_only = False, no_upload = True, repo = repo, is_new = True)
            valid = do_validate(dbc)
            self.close_server_database(dbc)

        return valid

    def doServerDatabaseSyncLock(self, repo, no_upload):

        if repo == None:
            repo = self.default_repository

        # check if the database is locked locally
        lock_file = self.MirrorsService.get_database_lockfile(repo)
        if os.path.isfile(lock_file):
            self.updateProgress(
                red(_("Entropy database is already locked by you :-)")),
                importance = 1,
                type = "info",
                header = red(" * ")
            )
        else:
            # check if the database is locked REMOTELY
            mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
            self.updateProgress(
                red(mytxt),
                importance = 1,
                type = "info",
                header = red(" * "),
                back = True
            )
            for uri in self.get_remote_mirrors(repo):
                given_up = self.MirrorsService.mirror_lock_check(uri, repo = repo)
                if given_up:
                    crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
                    mytxt = "%s:" % (_("Mirrors status table"),)
                    self.updateProgress(
                        darkgreen(mytxt),
                        importance = 1,
                        type = "info",
                        header = brown(" * ")
                    )
                    dbstatus = self.MirrorsService.get_mirrors_lock(repo = repo)
                    for db in dbstatus:
                        db[1] = green(_("Unlocked"))
                        if (db[1]):
                            db[1] = red(_("Locked"))
                        db[2] = green(_("Unlocked"))
                        if (db[2]):
                            db[2] = red(_("Locked"))

                        crippled_uri = self.entropyTools.extractFTPHostFromUri(db[0])
                        self.updateProgress(
                            "%s: [%s: %s] [%s: %s]" % (
                                bold(crippled_uri),
                                brown(_("database")),
                                db[1],
                                brown(_("download")),
                                db[2],
                            ),
                            importance = 1,
                            type = "info",
                            header = "\t"
                        )

                    raise exceptionTools.OnlineMirrorError("OnlineMirrorError: %s %s" % (
                            _("cannot lock mirror"),
                            crippled_uri,
                        )
                    )

            # if we arrive here, it is because all the mirrors are unlocked
            self.MirrorsService.lock_mirrors(True, repo = repo)
            self.MirrorsService.sync_databases(no_upload, repo = repo)

    def openServerDatabase(
            self,
            read_only = True,
            no_upload = True,
            just_reading = False,
            repo = None,
            indexing = True,
            warnings = True,
            do_cache = True,
            use_branch = None,
            lock_remote = True,
            is_new = False
        ):

        if repo == None:
            repo = self.default_repository

        if repo == etpConst['clientserverrepoid'] and self.community_repo:
            return self.ClientService.clientDbconn

        if just_reading:
            read_only = True
            no_upload = True

        t_ident = 1 # thread.get_ident() disabled for now
        local_dbfile = self.get_local_database_file(repo, use_branch)
        if do_cache:
            cached = self.serverDbCache.get(
                            (   etpConst['systemroot'],
                                local_dbfile,
                                read_only,
                                no_upload,
                                just_reading,
                                repo,
                                t_ident,
                                use_branch,
                                lock_remote,
                            )
            )
            if cached != None:
                return cached

        if not os.path.isdir(os.path.dirname(local_dbfile)):
            os.makedirs(os.path.dirname(local_dbfile))

        if (not read_only) and (lock_remote):
            self.doServerDatabaseSyncLock(repo, no_upload)

        conn = EntropyDatabaseInterface(
            readOnly = read_only,
            dbFile = local_dbfile,
            noUpload = no_upload,
            OutputInterface = self,
            ServiceInterface = self,
            dbname = etpConst['serverdbid']+repo,
            useBranch = use_branch,
            lockRemote = lock_remote
        )

        valid = True
        try:
            conn.validateDatabase()
        except exceptionTools.SystemDatabaseError:
            valid = False

        # verify if we need to update the database to sync
        # with portage updates, we just ignore being readonly in the case
        if (repo not in etpConst['server_treeupdatescalled']) and (not just_reading):
            # sometimes, when filling a new server db, we need to avoid tree updates
            if valid:
                conn.serverUpdatePackagesData()
            elif warnings and not is_new:
                mytxt = _( "Entropy database is probably corrupted! I won't stop you here btw...")
                self.updateProgress(
                    darkred(mytxt),
                    importance = 1,
                    type = "warning",
                    header = bold(" !!! ")
                )

        if not read_only and valid and indexing:

            self.updateProgress(
                "[repo:%s|%s] %s" % (
                            blue(repo),
                            red(_("database")),
                            blue(_("indexing database")),
                    ),
                importance = 1,
                type = "info",
                header = brown(" @@ "),
                back = True
            )
            conn.createAllIndexes()

        if do_cache:
            # !!! also cache just_reading otherwise there will be
            # real issues if the connection is opened several times
            self.serverDbCache[(
                etpConst['systemroot'],
                local_dbfile,
                read_only,
                no_upload,
                just_reading,
                repo,
                t_ident,
                use_branch,
                lock_remote
            )] = conn

        # auto-update package sets
        if (not read_only) and (not is_new):
            cur_sets = conn.retrievePackageSets()
            sys_sets = self.get_configured_package_sets(repo)
            if cur_sets != sys_sets:
                self.update_database_package_sets(repo, dbconn = conn)
            conn.commitChanges()

        return conn

    def deps_tester(self):

        server_repos = etpConst['server_repositories'].keys()
        installed_packages = set()
        for repo in server_repos:
            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
            installed_packages |= set([(x,repo) for x in dbconn.listAllIdpackages()])


        deps_not_satisfied = set()
        length = str((len(installed_packages)))
        count = 0
        mytxt = _("Checking")
        for pkgdata in installed_packages:
            count += 1
            idpackage = pkgdata[0]
            repo = pkgdata[1]
            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
            atom = dbconn.retrieveAtom(idpackage)
            if atom == None:
                continue
            self.updateProgress(
                darkgreen(mytxt)+" "+bold(atom),
                importance = 0,
                type = "info",
                back = True,
                count = (count,length),
                header = darkred(" @@  ")
            )

            xdeps = dbconn.retrieveDependencies(idpackage)
            for xdep in xdeps:
                xmatch = self.atomMatch(xdep)
                if xmatch[0] == -1:
                    deps_not_satisfied.add(xdep)

        return deps_not_satisfied

    def dependencies_test(self):

        mytxt = "%s %s" % (blue(_("Running dependencies test")),red("..."))
        self.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )

        server_repos = etpConst['server_repositories'].keys()
        deps_not_matched = self.deps_tester()

        if deps_not_matched:

            crying_atoms = {}
            for atom in deps_not_matched:
                for repo in server_repos:
                    dbconn = self.openServerDatabase(just_reading = True, repo = repo)
                    riddep = dbconn.searchDependency(atom)
                    if riddep == -1:
                        continue
                    if riddep != -1:
                        ridpackages = dbconn.searchIdpackageFromIddependency(riddep)
                        for i in ridpackages:
                            iatom = dbconn.retrieveAtom(i)
                            if not crying_atoms.has_key(atom):
                                crying_atoms[atom] = set()
                            crying_atoms[atom].add((iatom,repo))

            mytxt = blue("%s:") % (_("These are the dependencies not found"),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
            mytxt = "%s:" % (_("Needed by"),)
            for atom in deps_not_matched:
                self.updateProgress(
                    red(atom),
                    importance = 1,
                    type = "info",
                    header = blue("   # ")
                )
                if crying_atoms.has_key(atom):
                    self.updateProgress(
                        red(mytxt),
                        importance = 0,
                        type = "info",
                        header = blue("      # ")
                    )
                    for x , myrepo in crying_atoms[atom]:
                        self.updateProgress(
                            "[%s:%s] %s" % (blue(_("by repo")),darkred(myrepo),darkgreen(x),),
                            importance = 0,
                            type = "info",
                            header = blue("      # ")
                        )
        else:

            mytxt = blue(_("Every dependency is satisfied. It's all fine."))
            self.updateProgress(
                mytxt,
                importance = 2,
                type = "info",
                header = red(" @@ ")
            )

        return deps_not_matched

    def libraries_test(self, get_files = False, repo = None):

        # load db
        dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
        packagesMatched, brokenexecs, status = self.ClientService.libraries_test(dbconn = dbconn, broken_symbols = False)
        if status != 0:
            return 1,None

        if get_files:
            return 0,brokenexecs

        if (not brokenexecs) and (not packagesMatched):
            mytxt = "%s." % (_("System is healthy"),)
            self.updateProgress(
                blue(mytxt),
                importance = 2,
                type = "info",
                header = red(" @@ ")
            )
            return 0,None

        mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
        self.updateProgress(
            blue(mytxt),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        packages = self.SpmService.query_belongs_multiple(brokenexecs)

        if packages:
            mytxt = "%s:" % (_("These are the matched packages"),)
            self.updateProgress(
                red(mytxt),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
            for package_slot in packages:
                self.updateProgress(
                    blue(unicode(package_slot)),
                    importance = 0,
                    type = "info",
                    header = red("     # ")
                )
                for filename in sorted(list(packages[package_slot])):
                    self.updateProgress(
                        blue(filename),
                        importance = 0,
                        type = "info",
                        header = brown("       => ")
                    )
            # print string
            pkgstring = ' '.join(["%s:%s" % (self.entropyTools.dep_getkey(x[0]),x[1],) for x in sorted(packages.keys())])
            mytxt = "%s: %s" % (darkgreen(_("Packages string")),pkgstring,)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
        else:
            self.updateProgress(
                red(_("No matched packages")),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )

        return 0,packages

    def orphaned_spm_packages_test(self):

        mytxt = "%s %s" % (blue(_("Running orphaned SPM packages test")),red("..."),)
        self.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )
        installed_packages, length = self.SpmService.get_installed_packages()
        not_found = {}
        count = 0
        for installed_package in installed_packages:
            count += 1
            self.updateProgress(
                "%s: %s" % (darkgreen(_("Scanning package")),brown(installed_package),),
                importance = 0,
                type = "info",
                back = True,
                count = (count,length),
                header = darkred(" @@ ")
            )
            key, slot = self.entropyTools.dep_getkey(installed_package),self.SpmService.get_installed_package_slot(installed_package)
            pkg_atom = "%s:%s" % (key,slot,)
            tree_atom = self.SpmService.get_best_atom(pkg_atom)
            if not tree_atom:
                not_found[installed_package] = pkg_atom
                self.updateProgress(
                    "%s: %s" % (blue(pkg_atom),darkred(_("not found anymore")),),
                    importance = 0,
                    type = "warning",
                    count = (count,length),
                    header = darkred(" @@ ")
                )

        if not_found:
            not_found_list = ' '.join([not_found[x] for x in sorted(not_found.keys())])
            self.updateProgress(
                "%s: %s" % (blue(_("Packages string")),not_found_list,),
                importance = 0,
                type = "warning",
                count = (count,length),
                header = darkred(" @@ ")
            )

        return not_found

    def depends_table_initialize(self, repo = None):
        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        dbconn.regenerateDependsTable()
        dbconn.taintDatabase()
        dbconn.commitChanges()

    def create_empty_database(self, dbpath = None, repo = None):
        if dbpath == None:
            dbpath = self.get_local_database_file(repo)

        dbdir = os.path.dirname(dbpath)
        if not os.path.isdir(dbdir):
            os.makedirs(dbdir)

        mytxt = red("%s ...") % (_("Initializing an empty database file with Entropy structure"),)
        self.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = darkgreen(" * "),
            back = True
        )
        dbconn = self.ClientService.openGenericDatabase(dbpath)
        dbconn.initializeDatabase()
        dbconn.commitChanges()
        dbconn.closeDB()
        mytxt = "%s %s %s." % (red(_("Entropy database file")),bold(dbpath),red(_("successfully initialized")),)
        self.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = darkgreen(" * ")
        )

    def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

        # check package_tag "no spaces"

        try:
            package_tag = str(package_tag)
            if " " in package_tag: raise ValueError
        except (UnicodeDecodeError,UnicodeEncodeError,ValueError,):
            self.updateProgress(
                "%s: %s" % (
                    blue(_("Invalid tag specified")),
                    package_tag,
                ),
                importance = 1, type = "error", header = darkred(" !! ")
            )
            return 1, package_tag

        if repo == None: repo = self.default_repository

        # sanity check
        invalid_atoms = []
        dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
        for idpackage in idpackages:
            ver_tag = dbconn.retrieveVersionTag(idpackage)
            if ver_tag:
                invalid_atoms.append(dbconn.retrieveAtom(idpackage))

        if invalid_atoms:
            self.updateProgress(
                "%s: %s" % (
                    blue(_("These are the packages already tagged, cannot re-tag, action aborted")),
                    ', '.join([darkred(unicode(x)) for x in invalid_atoms]),
                ),
                importance = 1, type = "error", header = darkred(" !! ")
            )
            return 2, invalid_atoms

        matches = [(x,repo) for x in idpackages]
        status = 0
        data = self.move_packages(
            matches, to_repo = repo, from_repo = repo, ask = ask,
            do_copy = True, new_tag = package_tag
        )
        return status, data


    def move_packages(self, matches, to_repo, from_repo = None, ask = True, do_copy = False, new_tag = None):

        if from_repo == None: from_repo = self.default_repository
        switched = set()

        # avoid setting __default__ as default server repo
        if etpConst['clientserverrepoid'] in (to_repo,from_repo):
            self.updateProgress(
                "%s: %s" % (
                        blue(_("You cannot switch packages from/to your system database")),
                        red(etpConst['clientserverrepoid']),
                ),
                importance = 2, type = "warning", header = darkred(" @@ ")
            )
            return switched

        if not matches and from_repo:
            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = from_repo)
            matches = set( \
                [(x,from_repo) for x in \
                    dbconn.listAllIdpackages()]
            )

        mytxt = _("Preparing to move selected packages to")
        if do_copy:
            mytxt = _("Preparing to copy selected packages to")
        self.updateProgress(
            "%s %s:" % (
                    blue(mytxt),
                    red(to_repo),
            ),
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )
        self.updateProgress(
            "%s: %s" % (
                    bold(_("Note")),
                    red(_("all the old packages with conflicting scope will be removed from the destination repo unless injected")),
            ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        new_tag_string = ''
        if new_tag != None: new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),brown(new_tag),)
        for match in matches:
            repo = match[1]
            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
            self.updateProgress(
                "[%s=>%s|%s] %s " % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(etpConst['branch']),
                        blue(dbconn.retrieveAtom(match[0])),
                ) + new_tag_string,
                importance = 0,
                type = "info",
                header = brown("    # ")
            )


        if ask:
            rc = self.askQuestion(_("Would you like to continue ?"))
            if rc == "No":
                return switched

        for idpackage, repo in matches:
            dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
            match_branch = dbconn.retrieveBranch(idpackage)
            match_atom = dbconn.retrieveAtom(idpackage)
            package_filename = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
            self.updateProgress(
                "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(etpConst['branch']),
                        blue(_("switching")),
                        darkgreen(match_atom),
                ),
                importance = 0,
                type = "info",
                header = red(" @@ "),
                back = True
            )
            # move binary file
            from_file = os.path.join(self.get_local_packages_directory(repo),match_branch,package_filename)
            if not os.path.isfile(from_file):
                from_file = os.path.join(self.get_local_upload_directory(repo),match_branch,package_filename)
            if not os.path.isfile(from_file):
                self.updateProgress(
                    "[%s=>%s|%s] %s: %s -> %s" % (
                            darkgreen(repo),
                            darkred(to_repo),
                            brown(etpConst['branch']),
                            bold(_("cannot switch, package not found, skipping")),
                            darkgreen(),
                            red(from_file),
                    ),
                    importance = 1,
                    type = "warning",
                    header = darkred(" !!! ")
                )
                continue

            if new_tag != None:
                match_category = dbconn.retrieveCategory(idpackage)
                match_name = dbconn.retrieveName(idpackage)
                match_version = dbconn.retrieveVersion(idpackage)
                tagged_package_filename = self.entropyTools.create_package_filename(match_category, match_name, match_version, new_tag)
                to_file = os.path.join(self.get_local_upload_directory(to_repo),match_branch,tagged_package_filename)
            else:
                to_file = os.path.join(self.get_local_upload_directory(to_repo),match_branch,package_filename)
            if not os.path.isdir(os.path.dirname(to_file)):
                os.makedirs(os.path.dirname(to_file))

            copy_data = [
                            (from_file,to_file,),
                            (from_file+etpConst['packageshashfileext'],to_file+etpConst['packageshashfileext'],),
                            (from_file+etpConst['packagesexpirationfileext'],to_file+etpConst['packagesexpirationfileext'],)
                        ]

            for from_item,to_item in copy_data:
                self.updateProgress(
                        "[%s=>%s|%s] %s: %s" % (
                                darkgreen(repo),
                                darkred(to_repo),
                                brown(etpConst['branch']),
                                blue(_("moving file")),
                                darkgreen(os.path.basename(from_item)),
                        ),
                        importance = 0,
                        type = "info",
                        header = red(" @@ "),
                        back = True
                )
                if os.path.isfile(from_item):
                    shutil.copy2(from_item,to_item)

            self.updateProgress(
                "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(etpConst['branch']),
                        blue(_("loading data from source database")),
                        darkgreen(repo),
                ),
                importance = 0,
                type = "info",
                header = red(" @@ "),
                back = True
            )
            # install package into destination db
            data = dbconn.getPackageData(idpackage)
            if new_tag != None:
                data['versiontag'] = new_tag

            todbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = to_repo)

            self.updateProgress(
                "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(etpConst['branch']),
                        blue(_("injecting data to destination database")),
                        darkgreen(to_repo),
                ),
                importance = 0,
                type = "info",
                header = red(" @@ "),
                back = True
            )
            new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
            del data
            todbconn.commitChanges()

            if not do_copy:
                self.updateProgress(
                    "[%s=>%s|%s] %s: %s" % (
                            darkgreen(repo),
                            darkred(to_repo),
                            brown(etpConst['branch']),
                            blue(_("removing entry from source database")),
                            darkgreen(repo),
                    ),
                    importance = 0,
                    type = "info",
                    header = red(" @@ "),
                    back = True
                )

                # remove package from old db
                dbconn.removePackage(idpackage)
                dbconn.commitChanges()

            self.updateProgress(
                "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(etpConst['branch']),
                        blue(_("successfully handled atom")),
                        darkgreen(match_atom),
                ),
                importance = 0,
                type = "info",
                header = blue(" @@ ")
            )
            switched.add(match)

        return switched


    def package_injector(self, package_file, inject = False, repo = None):

        if repo == None:
            repo = self.default_repository

        upload_dir = os.path.join(self.get_local_upload_directory(repo),etpConst['branch'])
        if not os.path.isdir(upload_dir):
            os.makedirs(upload_dir)

        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        self.updateProgress(
            red("[repo: %s] %s: %s" % (
                        darkgreen(repo),
                        _("adding package"),
                        bold(os.path.basename(package_file)),
                    )
            ),
            importance = 1,
            type = "info",
            header = brown(" * "),
            back = True
        )
        mydata = self.ClientService.extract_pkg_metadata(package_file, etpBranch = etpConst['branch'], inject = inject)
        idpackage, revision, mydata = dbconn.handlePackage(mydata)

        # set trashed counters
        trashing_counters = set()
        myserver_repos = etpConst['server_repositories'].keys()
        for myrepo in myserver_repos:
            mydbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = myrepo)
            mylist = mydbconn.retrieve_packages_to_remove(
                    mydata['name'],
                    mydata['category'],
                    mydata['slot'],
                    mydata['injected']
            )
            for myitem in mylist:
                trashing_counters.add(mydbconn.retrieveCounter(myitem))

        for mycounter in trashing_counters:
            dbconn.setTrashedCounter(mycounter)

        # add package info to our current server repository
        dbconn.removePackageFromInstalledTable(idpackage)
        dbconn.addPackageToInstalledTable(idpackage,repo)
        atom = dbconn.retrieveAtom(idpackage)

        self.updateProgress(
            "[repo:%s] %s: %s %s: %s" % (
                        darkgreen(repo),
                        blue(_("added package")),
                        darkgreen(atom),
                        blue(_("rev")), # as in revision
                        bold(str(revision)),
                ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        download_url = self._setup_repository_package_filename(idpackage, repo = repo)
        downloadfile = os.path.basename(download_url)
        destination_path = os.path.join(upload_dir,downloadfile)
        shutil.move(package_file,destination_path)

        dbconn.commitChanges()
        return idpackage,destination_path

    # this function changes the final repository package filename
    def _setup_repository_package_filename(self, idpackage, repo = None):

        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)

        downloadurl = dbconn.retrieveDownloadURL(idpackage)
        packagerev = dbconn.retrieveRevision(idpackage)
        downloaddir = os.path.dirname(downloadurl)
        downloadfile = os.path.basename(downloadurl)
        # add revision
        downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,etpConst['packagesext'],)
        downloadurl = os.path.join(downloaddir,downloadfile)

        # update url
        dbconn.setDownloadURL(idpackage,downloadurl)

        return downloadurl

    def add_packages_to_repository(self, packages_data, ask = True, repo = None):

        if repo == None:
            repo = self.default_repository

        mycount = 0
        maxcount = len(packages_data)
        idpackages_added = set()
        to_be_injected = set()
        myQA = self.QA()
        missing_deps_taint = False
        for package_filepath, inject in packages_data:

            mycount += 1
            self.updateProgress(
                "[repo:%s] %s: %s" % (
                            darkgreen(repo),
                            blue(_("adding package")),
                            darkgreen(os.path.basename(package_filepath)),
                        ),
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                count = (mycount,maxcount,)
            )

            try:
                # add to database
                idpackage, destination_path = self.package_injector(
                    package_filepath,
                    inject = inject
                )
                idpackages_added.add(idpackage)
                to_be_injected.add((idpackage,destination_path))
            except Exception, e:
                self.updateProgress(
                    "[repo:%s] %s: %s" % (
                                darkgreen(repo),
                                darkred(_("Exception caught, running injection and RDEPEND check before raising")),
                                darkgreen(unicode(e)),
                            ),
                    importance = 1,
                    type = "error",
                    header = bold(" !!! "),
                    count = (mycount,maxcount,)
                )
                # reinit depends table
                self.depends_table_initialize(repo)
                if idpackages_added:
                    dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
                    missing_deps_taint = myQA.scan_missing_dependencies(
                        idpackages_added,
                        dbconn,
                        ask = ask,
                        repo = repo,
                        self_check = True,
                        black_list = self.get_missing_dependencies_blacklist(repo = repo),
                        black_list_adder = self.add_missing_dependencies_blacklist_items
                    )
                    myQA.test_depends_linking(idpackages_added, dbconn, repo = repo)
                if to_be_injected:
                    self.inject_database_into_packages(to_be_injected, repo = repo)
                # reinit depends table
                if missing_deps_taint:
                    self.depends_table_initialize(repo)
                self.close_server_databases()
                raise

        # reinit depends table
        self.depends_table_initialize(repo)

        if idpackages_added:
            dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
            missing_deps_taint = myQA.scan_missing_dependencies(
                idpackages_added,
                dbconn,
                ask = ask,
                repo = repo,
                self_check = True,
                black_list = self.get_missing_dependencies_blacklist(repo = repo),
                black_list_adder = self.add_missing_dependencies_blacklist_items
            )
            myQA.test_depends_linking(idpackages_added, dbconn, repo = repo)

        # reinit depends table
        if missing_deps_taint:
            self.depends_table_initialize(repo)

        # inject database into packages
        self.inject_database_into_packages(to_be_injected, repo = repo)

        return idpackages_added


    def inject_database_into_packages(self, injection_data, repo = None):

        if repo == None:
            repo = self.default_repository

        # now inject metadata into tbz2 packages
        self.updateProgress(
            "[repo:%s] %s:" % (
                        darkgreen(repo),
                        blue(_("Injecting entropy metadata into built packages")),
                    ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        for idpackage,package_path in injection_data:
            self.updateProgress(
                "[repo:%s|%s] %s: %s" % (
                            darkgreen(repo),
                            brown(str(idpackage)),
                            blue(_("injecting entropy metadata")),
                            darkgreen(os.path.basename(package_path)),
                        ),
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )
            data = dbconn.getPackageData(idpackage)
            treeupdates_actions = dbconn.listAllTreeUpdatesActions()
            dbpath = self.ClientService.inject_entropy_database_into_package(package_path, data, treeupdates_actions)
            digest = self.entropyTools.md5sum(package_path)
            # update digest
            dbconn.setDigest(idpackage,digest)
            self.entropyTools.createHashFile(package_path)
            # remove garbage
            os.remove(dbpath)
            self.updateProgress(
                "[repo:%s|%s] %s: %s" % (
                            darkgreen(repo),
                            brown(str(idpackage)),
                            blue(_("injection complete")),
                            darkgreen(os.path.basename(package_path)),
                        ),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
            dbconn.commitChanges()

    def check_config_file_updates(self):
        self.updateProgress(
            "[%s] %s" % (
                red(_("config files")), # something short please
                blue(_("checking system")),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ "),
            back = True
        )
        # scanning for config files not updated
        scandata = self.ClientService.FileUpdates.scanfs(dcache = False)
        if scandata:
            self.updateProgress(
                "[%s] %s" % (
                    red(_("config files")), # something short please
                    blue(_("there are configuration files not updated yet")),
                ),
                importance = 1,
                type = "error",
                header = darkred(" @@ ")
            )
            for x in scandata:
                self.updateProgress(
                    "%s" % ( brown(etpConst['systemroot']+scandata[x]['destination']) ),
                    importance = 1,
                    type = "info",
                    header = "\t"
                )
            return True
        return False

    def quickpkg(self, atom, storedir):
        return self.SpmService.quickpkg(atom,storedir)


    def remove_packages(self, idpackages, repo = None):

        if repo == None:
            repo = self.default_repository

        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        for idpackage in idpackages:
            atom = dbconn.retrieveAtom(idpackage)
            self.updateProgress(
                "[repo:%s] %s: %s" % (
                        darkgreen(repo),
                        blue(_("removing package")),
                        darkgreen(atom),
                ),
                importance = 1,
                type = "info",
                header = brown(" @@ ")
            )
            dbconn.removePackage(idpackage)
        self.close_server_database(dbconn)
        self.updateProgress(
            "[repo:%s] %s" % (
                        darkgreen(repo),
                        blue(_("removal complete")),
                ),
            importance = 1,
            type = "info",
            header = brown(" @@ ")
        )


    def bump_database(self, repo = None):
        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        dbconn.taintDatabase()
        self.close_server_database(dbconn)

    def get_remote_mirrors(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['mirrors'][:]

    def get_remote_packages_relative_path(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['packages_relative_path']

    def get_remote_database_relative_path(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['database_relative_path']

    def get_local_database_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabasefile'])

    def get_local_store_directory(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['store_dir']

    def get_local_upload_directory(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['upload_dir']

    def get_local_packages_directory(self, repo = None):
        if repo == None:
            repo = self.default_repository
        return etpConst['server_repositories'][repo]['packages_dir']

    def get_local_database_taint_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabasetaintfile'])

    def get_local_database_revision_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabaserevisionfile'])

    def get_local_database_ca_cert_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabasecacertfile'])

    def get_local_database_server_cert_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabaseservercertfile'])

    def get_local_database_mask_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabasemaskfile'])

    def get_local_database_system_mask_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabasesytemmaskfile'])

    def get_local_database_confl_tagged_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabaseconflictingtaggedfile'])

    def get_local_database_licensewhitelist_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabaselicwhitelistfile'])

    def get_local_database_rss_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['rss-name'])

    def get_local_database_rsslight_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['rss-light-name'])

    def get_local_database_notice_board_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['rss-notice-board'])

    def get_local_database_treeupdates_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['etpdatabaseupdatefile'])

    def get_local_database_sets_dir(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        return os.path.join(self.get_local_database_dir(repo, branch),etpConst['confsetsdirname'])

    def get_local_database_dir(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        if branch == None:
            branch = etpConst['branch']
        return os.path.join(etpConst['server_repositories'][repo]['database_dir'],branch)

    def get_missing_dependencies_blacklist_file(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        if branch == None:
            branch = etpConst['branch']
        return os.path.join(etpConst['server_repositories'][repo]['database_dir'],branch,etpConst['etpdatabasemissingdepsblfile'])

    def get_missing_dependencies_blacklist(self, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        if branch == None:
            branch = etpConst['branch']
        wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
        wl_data = []
        if os.path.isfile(wl_file) and os.access(wl_file,os.R_OK):
            f = open(wl_file,"r")
            wl_data = [x.strip() for x in f.readlines() if x.strip() and not x.strip().startswith("#")]
            f.close()
        return set(wl_data)

    def add_missing_dependencies_blacklist_items(self, items, repo = None, branch = None):
        if repo == None:
            repo = self.default_repository
        if branch == None:
            branch = etpConst['branch']
        wl_file = self.get_missing_dependencies_blacklist_file(repo, branch)
        wl_dir = os.path.dirname(wl_file)
        if not (os.path.isdir(wl_dir) and os.access(wl_dir,os.W_OK)):
            return
        if os.path.isfile(wl_file) and not os.access(wl_file,os.W_OK):
            return
        f = open(wl_file,"a+")
        f.write('\n'.join(items)+'\n')
        f.flush()
        f.close()

    def get_local_database_revision(self, repo = None):

        if repo == None:
            repo = self.default_repository

        dbrev_file = self.get_local_database_revision_file(repo)
        if os.path.isfile(dbrev_file):
            f = open(dbrev_file)
            rev = f.readline().strip()
            f.close()
            try:
                rev = int(rev)
            except ValueError:
                self.updateProgress(
                    "[repo:%s] %s: %s - %s" % (
                            darkgreen(repo),
                            blue(_("invalid database revision")),
                            bold(rev),
                            blue(_("defaulting to 0")),
                        ),
                    importance = 2,
                    type = "error",
                    header = darkred(" !!! ")
                )
                rev = 0
            return rev
        else:
            return 0

    def get_remote_database_revision(self, repo = None):

        if repo == None:
            repo = self.default_repository

        remote_status =  self.MirrorsService.get_remote_databases_status(repo)
        if not [x for x in remote_status if x[1]]:
            remote_revision = 0
        else:
            remote_revision = max([x[1] for x in remote_status])

        return remote_revision

    def get_branch_from_download_relative_uri(self, mypath):
        return self.ClientService.get_branch_from_download_relative_uri(mypath)

    def packageSetList(self, *args, **kwargs):
        repos = etpConst['server_repositories'].keys()
        kwargs['server_repos'] = repos
        kwargs['serverInstance'] = self
        return self.ClientService.packageSetList(*args,**kwargs)

    def packageSetSearch(self, *args, **kwargs):
        repos = etpConst['server_repositories'].keys()
        kwargs['server_repos'] = repos
        kwargs['serverInstance'] = self
        return self.ClientService.packageSetSearch(*args,**kwargs)

    def packageSetMatch(self, *args, **kwargs):
        repos = etpConst['server_repositories'].keys()
        kwargs['server_repos'] = repos
        kwargs['serverInstance'] = self
        return self.ClientService.packageSetMatch(*args,**kwargs)

    def atomMatch(self, *args, **kwargs):
        repos = etpConst['server_repositories'].keys()
        kwargs['server_repos'] = repos
        kwargs['serverInstance'] = self
        return self.ClientService.atomMatch(*args,**kwargs)

    def scan_package_changes(self):

        installed_packages = self.SpmService.get_installed_packages_counter()
        installed_counters = set()
        toBeAdded = set()
        toBeRemoved = set()
        toBeInjected = set()

        server_repos = etpConst['server_repositories'].keys()

        # packages to be added
        for spm_atom,spm_counter in installed_packages:
            found = False
            for server_repo in server_repos:
                installed_counters.add(spm_counter)
                server_dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = server_repo)
                counter = server_dbconn.isCounterAvailable(spm_counter, branch = etpConst['branch'])
                if counter:
                    found = True
                    break
            if not found:
                toBeAdded.add((spm_atom,spm_counter,))

        # packages to be removed from the database
        database_counters = {}
        for server_repo in server_repos:
            server_dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = server_repo)
            database_counters[server_repo] = server_dbconn.listAllCounters(branch = etpConst['branch'])

        ordered_counters = set()
        for server_repo in database_counters:
            for data in database_counters[server_repo]:
                ordered_counters.add((data,server_repo))
        database_counters = ordered_counters

        for (counter,idpackage,),xrepo in database_counters:

            if counter < 0:
                continue # skip packages without valid counter

            if counter in installed_counters:
                continue

            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = xrepo)

            dorm = True
            # check if the package is in toBeAdded
            if toBeAdded:

                dorm = False
                atom = dbconn.retrieveAtom(idpackage)
                atomkey = self.entropyTools.dep_getkey(atom)
                atomtag = self.entropyTools.dep_gettag(atom)
                atomslot = dbconn.retrieveSlot(idpackage)

                add = True
                for spm_atom, spm_counter in toBeAdded:
                    addslot = self.SpmService.get_installed_package_slot(spm_atom)
                    addkey = self.entropyTools.dep_getkey(spm_atom)
                    # workaround for ebuilds not having slot
                    if addslot == None:
                        addslot = '0'                                              # handle tagged packages correctly
                    if (atomkey == addkey) and ((str(atomslot) == str(addslot)) or (atomtag != None)):
                        # do not add to toBeRemoved
                        add = False
                        break

                if not add:
                    continue
                dorm = True

            if dorm:
                trashed = self.is_counter_trashed(counter)
                if trashed:
                    # search into portage then
                    try:
                        key, slot = dbconn.retrieveKeySlot(idpackage)
                        trashed = self.SpmService.get_installed_atom(key+":"+slot)
                    except TypeError: # referred to retrieveKeySlot
                        trashed = True
                if not trashed:
                    dbtag = dbconn.retrieveVersionTag(idpackage)
                    if dbtag != '':
                        is_injected = dbconn.isInjected(idpackage)
                        if not is_injected:
                            toBeInjected.add((idpackage,xrepo))
                    else:
                        toBeRemoved.add((idpackage,xrepo))

        return toBeAdded, toBeRemoved, toBeInjected

    def is_counter_trashed(self, counter):
        server_repos = etpConst['server_repositories'].keys()
        for repo in server_repos:
            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
            if dbconn.isCounterTrashed(counter):
                return True
        return False

    def transform_package_into_injected(self, idpackage, repo = None):
        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        counter = dbconn.getNewNegativeCounter()
        dbconn.setCounter(idpackage,counter)
        dbconn.setInjected(idpackage)

    def initialize_server_database(self, empty = True, repo = None, warnings = True):

        if repo == None: repo = self.default_repository

        self.close_server_databases()
        revisions_match = {}
        treeupdates_actions = []
        injected_packages = set()
        idpackages = set()
        idpackages_added = set()

        mytxt = red("%s ...") % (_("Initializing Entropy database"),)
        self.updateProgress(
            mytxt, importance = 1,
            type = "info", header = darkgreen(" * "),
            back = True
        )

        if os.path.isfile(self.get_local_database_file(repo)):

            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo, warnings = warnings)

            if dbconn.doesTableExist("baseinfo") and dbconn.doesTableExist("extrainfo"):
                idpackages = dbconn.listAllIdpackages()

            if dbconn.doesTableExist("treeupdatesactions"):
                treeupdates_actions = dbconn.listAllTreeUpdatesActions()

            # save list of injected packages
            if dbconn.doesTableExist("injected") and dbconn.doesTableExist("extrainfo"):
                injected_packages = dbconn.listAllInjectedPackages(justFiles = True)
                injected_packages = set([os.path.basename(x) for x in injected_packages])

            for idpackage in idpackages:
                package = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
                branch = dbconn.retrieveBranch(idpackage)
                revision = dbconn.retrieveRevision(idpackage)
                revisions_match[package] = (branch,revision,)

            self.close_server_database(dbconn)

            mytxt = "%s: %s: %s" % (
                bold(_("WARNING")),
                red(_("database already exists")),
                self.get_local_database_file(repo),
            )
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = darkred(" !!! ")
            )

            rc = self.askQuestion(_("Do you want to continue ?"))
            if rc == "No": return
            try:
                os.remove(self.get_local_database_file(repo))
            except OSError:
                pass


        # initialize
        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo, is_new = True)
        dbconn.initializeDatabase()

        if not empty:

            revisions_file = "/entropy-revisions-dump.txt"
            # dump revisions - as a backup
            if revisions_match:
                self.updateProgress(
                    "%s: %s" % (
                        red(_("Dumping current revisions to file")),
                        darkgreen(revisions_file),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                f = open(revisions_file,"w")
                f.write(str(revisions_match))
                f.flush()
                f.close()

            # dump treeupdates - as a backup
            treeupdates_file = "/entropy-treeupdates-dump.txt"
            if treeupdates_actions:
                self.updateProgress(
                    "%s: %s" % (
                        red(_("Dumping current 'treeupdates' actions to file")), # do not translate treeupdates
                        bold(treeupdates_file),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                f = open(treeupdates_file,"w")
                f.write(str(treeupdates_actions))
                f.flush()
                f.close()

            rc = self.askQuestion(_("Would you like to sync packages first (important if you don't have them synced) ?"))
            if rc == "Yes":
                self.MirrorsService.sync_packages(repo = repo)

            # fill tree updates actions
            if treeupdates_actions:
                dbconn.bumpTreeUpdatesActions(treeupdates_actions)

            # now fill the database
            pkg_branch_dir = os.path.join(self.get_local_packages_directory(repo),etpConst['branch'])
            pkglist = os.listdir(pkg_branch_dir)
            # filter .md5 and .expired packages
            pkglist = [x for x in pkglist if x[-5:] == etpConst['packagesext'] and not \
                os.path.isfile(os.path.join(pkg_branch_dir,x+etpConst['packagesexpirationfileext']))]

            if pkglist:
                self.updateProgress(
                    "%s '%s' %s %s" % (
                        red(_("Reinitializing Entropy database for branch")),
                        bold(etpConst['branch']),
                        red(_("using Packages in the repository")),
                        red("..."),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )

            counter = 0
            maxcount = len(pkglist)
            for pkg in pkglist:
                counter += 1

                self.updateProgress(
                    "[repo:%s|%s] %s: %s" % (
                            darkgreen(repo),
                            brown(etpConst['branch']),
                            blue(_("analyzing")),
                            bold(pkg),
                        ),
                    importance = 1,
                    type = "info",
                    header = " ",
                    back = True,
                    count = (counter,maxcount,)
                )

                doinject = False
                if pkg in injected_packages:
                    doinject = True

                pkg_path = os.path.join(self.get_local_packages_directory(repo),etpConst['branch'],pkg)
                mydata = self.ClientService.extract_pkg_metadata(pkg_path, etpConst['branch'], inject = doinject)

                # get previous revision
                revision_avail = revisions_match.get(pkg)
                addRevision = 0
                if (revision_avail != None):
                    if etpConst['branch'] == revision_avail[0]:
                        addRevision = revision_avail[1]

                idpackage, revision, mydata_upd = dbconn.addPackage(mydata, revision = addRevision)
                idpackages_added.add(idpackage)

                self.updateProgress(
                    "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
                                repo,
                                brown(etpConst['branch']),
                                darkgreen(str(counter)),
                                blue(str(maxcount)),
                                red(_("added package")),
                                darkgreen(pkg),
                                red(_("revision")),
                                brown(str(revision)),
                        ),
                    importance = 1,
                    type = "info",
                    header = " ",
                    back = True
                )

            self.depends_table_initialize(repo)

            myQA = self.QA()

            if idpackages_added:
                dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
                myQA.scan_missing_dependencies(
                    idpackages_added, dbconn, ask = True,
                    repo = repo, self_check = True,
                    black_list = self.get_missing_dependencies_blacklist(repo = repo),
                    black_list_adder = self.add_missing_dependencies_blacklist_items
                )

        dbconn.commitChanges()
        self.close_server_databases()

        return 0

    def match_packages(self, packages, repo = None):

        dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
        if ("world" in packages) or not packages:
            return dbconn.listAllIdpackages(),True
        else:
            idpackages = set()
            for package in packages:
                matches = dbconn.atomMatch(package, multiMatch = True)
                if matches[1] == 0:
                    idpackages |= matches[0]
                else:
                    mytxt = "%s: %s: %s" % (red(_("Attention")),blue(_("cannot match")),bold(package),)
                    self.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "warning",
                        header = darkred(" !!! ")
                    )
            return idpackages,False

    def get_remote_package_checksum(self, repo, filename, branch):

        if not etpConst['server_repositories'][repo].has_key('handler'):
            return None
        url = etpConst['server_repositories'][repo]['handler']

        # does the package has "#" (== tag) ? hackish thing that works
        filename = filename.replace("#","%23")
        # "+"
        filename = filename.replace("+","%2b")
        request = os.path.join(url,etpConst['handlers']['md5sum'])
        request += filename+"&branch="+branch

        # now pray the server
        try:
            mydict = {}
            if etpConst['proxy']['ftp']:
                mydict['ftp'] = etpConst['proxy']['ftp']
            if etpConst['proxy']['http']:
                mydict['http'] = etpConst['proxy']['http']
            if mydict:
                mydict['username'] = etpConst['proxy']['username']
                mydict['password'] = etpConst['proxy']['password']
                self.entropyTools.add_proxy_opener(urllib2, mydict)
            else:
                # unset
                urllib2._opener = None
            item = urllib2.urlopen(request)
            result = item.readline().strip()
            item.close()
            del item
            return result
        except: # no HTTP support?
            return None

    def verify_remote_packages(self, packages, ask = True, repo = None):

        if repo == None:
            repo = self.default_repository

        self.updateProgress(
            "[%s] %s:" % (
                red("remote"),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )

        idpackages, world = self.match_packages(packages)
        dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)

        if world:
            self.updateProgress(
                blue(_("All the packages in the Entropy Packages repository will be checked.")),
                importance = 1,
                type = "info",
                header = "    "
            )
        else:
            mytxt = red("%s:") % (_("This is the list of the packages that would be checked"),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = "    "
            )
            for idpackage in idpackages:
                pkgatom = dbconn.retrieveAtom(idpackage)
                pkgfile = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
                self.updateProgress(
                    red(pkgatom)+" -> "+bold(os.path.join(etpConst['branch'],pkgfile)),
                    importance = 1,
                    type = "info",
                    header = darkgreen("   - ")
                )

        if ask:
            rc = self.askQuestion(_("Would you like to continue ?"))
            if rc == "No":
                return set(),set(),{}

        match = set()
        not_match = set()
        broken_packages = {}

        for uri in self.get_remote_mirrors(repo):

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            self.updateProgress(
                "[repo:%s] %s: %s" % (
                        darkgreen(repo),
                        blue(_("Working on mirror")),
                        brown(crippled_uri),
                    ),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )


            totalcounter = len(idpackages)
            currentcounter = 0
            for idpackage in idpackages:

                currentcounter += 1
                pkgfile = dbconn.retrieveDownloadURL(idpackage)
                orig_branch = self.get_branch_from_download_relative_uri(pkgfile)

                self.updateProgress(
                    "[%s] %s: %s" % (
                            brown(crippled_uri),
                            blue(_("checking hash")),
                            darkgreen(pkgfile),
                    ),
                    importance = 1,
                    type = "info",
                    header = blue(" @@ "),
                    back = True,
                    count = (currentcounter,totalcounter,)
                )

                ckOk = False
                ck = self.get_remote_package_checksum(repo, os.path.basename(pkgfile), orig_branch)
                if ck == None:
                    self.updateProgress(
                        "[%s] %s: %s %s" % (
                            brown(crippled_uri),
                            blue(_("digest verification of")),
                            bold(pkgfile),
                            blue(_("not supported")),
                        ),
                        importance = 1,
                        type = "info",
                        header = blue(" @@ "),
                        count = (currentcounter,totalcounter,)
                    )
                elif len(ck) == 32:
                    pkghash = dbconn.retrieveDigest(idpackage)
                    if ck == pkghash: ckOk = True
                else:
                    self.updateProgress(
                        "[%s] %s: %s %s" % (
                            brown(crippled_uri),
                            blue(_("digest verification of")),
                            bold(pkgfile),
                            blue(_("failed for unknown reasons")),
                        ),
                        importance = 1,
                        type = "info",
                        header = blue(" @@ "),
                        count = (currentcounter,totalcounter,)
                    )

                if ckOk:
                    match.add(idpackage)
                else:
                    not_match.add(idpackage)
                    self.updateProgress(
                        "[%s] %s: %s %s" % (
                            brown(crippled_uri),
                            blue(_("package")),
                            bold(pkgfile),
                            red(_("NOT healthy")),
                        ),
                        importance = 1,
                        type = "warning",
                        header = darkred(" !!! "),
                        count = (currentcounter,totalcounter,)
                    )
                    if not broken_packages.has_key(crippled_uri):
                        broken_packages[crippled_uri] = []
                    broken_packages[crippled_uri].append(pkgfile)

            if broken_packages:
                mytxt = blue("%s:") % (_("This is the list of broken packages"),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = red(" * ")
                )
                for mirror in broken_packages.keys():
                    mytxt = "%s: %s" % (brown(_("Mirror")),bold(mirror),)
                    self.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "info",
                        header = red("   <> ")
                    )
                    for bp in broken_packages[mirror]:
                        self.updateProgress(
                            blue(bp),
                            importance = 1,
                            type = "info",
                            header = red("      - ")
                        )

            self.updateProgress(
                "%s:" % (
                    blue(_("Statistics")),
                ),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
            self.updateProgress(
                "[%s] %s:\t%s" % (
                    red(crippled_uri),
                    brown(_("Number of checked packages")),
                    brown(str(len(match)+len(not_match))),
                ),
                importance = 1,
                type = "info",
               header = brown("   # ")
            )
            self.updateProgress(
                "[%s] %s:\t%s" % (
                    red(crippled_uri),
                    darkgreen(_("Number of healthy packages")),
                    darkgreen(str(len(match))),
                ),
                importance = 1,
                type = "info",
               header = brown("   # ")
            )
            self.updateProgress(
                "[%s] %s:\t%s" % (
                    red(crippled_uri),
                    darkred(_("Number of broken packages")),
                    darkred(str(len(not_match))),
                ),
                importance = 1,
                type = "info",
                header = brown("   # ")
            )

        return match, not_match, broken_packages


    def verify_local_packages(self, packages, ask = True, repo = None):

        if repo == None:
            repo = self.default_repository

        self.updateProgress(
            "[%s] %s:" % (
                red(_("local")),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            type = "info",
            header = darkgreen(" * ")
        )

        idpackages, world = self.match_packages(packages)
        dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)

        if world:
            self.updateProgress(
                blue(_("All the packages in the Entropy Packages repository will be checked.")),
                importance = 1,
                type = "info",
                header = "    "
            )

        to_download = set()
        available = set()
        for idpackage in idpackages:

            pkgatom = dbconn.retrieveAtom(idpackage)
            pkg_path = dbconn.retrieveDownloadURL(idpackage)
            pkg_rel_path = '/'.join(pkg_path.split("/")[2:])

            bindir_path = os.path.join(self.get_local_packages_directory(repo),pkg_rel_path)
            uploaddir_path = os.path.join(self.get_local_upload_directory(repo),pkg_rel_path)

            if os.path.isfile(bindir_path):
                if not world:
                    self.updateProgress(
                        "[%s] %s :: %s" % (
                                darkgreen(_("available")),
                                blue(pkgatom),
                                darkgreen(pkg_rel_path),
                        ),
                        importance = 0,
                        type = "info",
                        header = darkgreen("   # ")
                    )
                available.add(idpackage)
            elif os.path.isfile(uploaddir_path):
                if not world:
                    self.updateProgress(
                        "[%s] %s :: %s" % (
                                darkred(_("upload/ignored")),
                                blue(pkgatom),
                                darkgreen(pkg_rel_path),
                        ),
                        importance = 0,
                        type = "info",
                        header = darkgreen("   # ")
                    )
            else:
                self.updateProgress(
                    "[%s] %s :: %s" % (
                            brown(_("download")),
                            blue(pkgatom),
                            darkgreen(pkg_rel_path),
                    ),
                    importance = 0,
                    type = "info",
                    header = darkgreen("   # ")
                )
                to_download.add((idpackage,pkg_path,))

        if ask:
            rc = self.askQuestion(_("Would you like to continue ?"))
            if rc == "No":
                return set(),set(),set(),set()


        fine = set()
        failed = set()
        downloaded_fine = set()
        downloaded_errors = set()

        if to_download:

            not_downloaded = set()
            mytxt = blue("%s ...") % (_("Starting to download missing files"),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "info",
                header = "   "
            )
            for uri in self.get_remote_mirrors(repo):

                if not_downloaded:
                    mytxt = blue("%s ...") % (_("Trying to search missing or broken files on another mirror"),)
                    self.updateProgress(
                        mytxt,
                        importance = 1,
                        type = "info",
                        header = "   "
                    )
                    to_download = not_downloaded.copy()
                    not_downloaded = set()

                for idpackage, pkg_path in to_download: # idpackage, pkgfile, branch
                    rc = self.MirrorsService.download_package(uri, pkg_path, repo = repo)
                    if rc:
                        downloaded_fine.add(idpackage)
                        available.add(idpackage)
                    else:
                        not_downloaded.add(pkg_path)

                if not not_downloaded:
                    self.updateProgress(
                        red(_("All the binary packages have been downloaded successfully.")),
                        importance = 1,
                        type = "info",
                        header = "   "
                    )
                    break

            if not_downloaded:
                mytxt = blue("%s:") % (_("These are the packages that cannot be found online"),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = "   "
                )
                for pkg_path in not_downloaded:
                    downloaded_errors.add(pkg_path)
                    self.updateProgress(
                            brown(pkg_path),
                            importance = 1,
                            type = "warning",
                            header = red("    * ")
                    )
                downloaded_errors |= not_downloaded
                mytxt = "%s." % (_("They won't be checked"),)
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = "   "
                )

        totalcounter = str(len(available))
        currentcounter = 0
        for idpackage in available:
            currentcounter += 1
            pkg_path = dbconn.retrieveDownloadURL(idpackage)
            orig_branch = self.get_branch_from_download_relative_uri(pkg_path)
            pkgfile = os.path.basename(pkg_path)

            self.updateProgress(
                "[branch:%s] %s %s" % (
                        brown(orig_branch),
                        blue(_("checking hash of")),
                        darkgreen(pkgfile),
                ),
                importance = 1,
                type = "info",
                header = "   ",
                back = True,
                count = (currentcounter,totalcounter,)
            )

            storedmd5 = dbconn.retrieveDigest(idpackage)
            pkgpath = os.path.join(self.get_local_packages_directory(repo),orig_branch,pkgfile)
            result = self.entropyTools.compareMd5(pkgpath,storedmd5)
            if result:
                fine.add(idpackage)
            else:
                failed.add(idpackage)
                self.updateProgress(
                    "[branch:%s] %s %s %s: %s" % (
                            brown(orig_branch),
                            blue(_("package")),
                            darkgreen(pkg_path),
                            blue(_("is corrupted, stored checksum")), # package -blah- is corrupted...
                            brown(storedmd5),
                    ),
                    importance = 1,
                    type = "info",
                    header = "   ",
                    count = (currentcounter,totalcounter,)
                )

        if failed:
            mytxt = blue("%s:") % (_("This is the list of broken packages"),)
            self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header =  darkred("  # ")
            )
            for idpackage in failed:
                atom = dbconn.retrieveAtom(idpackage)
                dp = dbconn.retrieveDownloadURL(idpackage)
                self.updateProgress(
                        blue("[atom:%s] %s" % (atom,dp,)),
                        importance = 0,
                        type = "warning",
                        header =  brown("    # ")
                )

        # print stats
        self.updateProgress(
            red("Statistics:"),
            importance = 1,
            type = "info",
            header = blue(" * ")
        )
        self.updateProgress(
            brown("%s:\t\t%s" % (
                    _("Number of checked packages"),
                    len(fine)+len(failed),
                )
            ),
            importance = 0,
            type = "info",
            header = brown("   # ")
        )
        self.updateProgress(
            darkgreen("%s:\t\t%s" % (
                    _("Number of healthy packages"),
                    len(fine),
                )
            ),
            importance = 0,
            type = "info",
            header = brown("   # ")
        )
        self.updateProgress(
            darkred("%s:\t\t%s" % (
                    _("Number of broken packages"),
                    len(failed),
                )
            ),
            importance = 0,
            type = "info",
            header = brown("   # ")
        )
        self.updateProgress(
            blue("%s:\t\t%s" % (
                    _("Number of downloaded packages"),
                    len(downloaded_fine),
                )
            ),
            importance = 0,
            type = "info",
            header = brown("   # ")
        )
        self.updateProgress(
            bold("%s:\t\t%s" % (
                    _("Number of failed downloads"),
                    len(downloaded_errors),
                )
            ),
            importance = 0,
            type = "info",
            header = brown("   # ")
        )

        self.close_server_database(dbconn)
        return fine, failed, downloaded_fine, downloaded_errors


    def switch_packages_branch(self, idpackages, from_branch, to_branch, repo = None):

        if repo == None:
            repo = self.default_repository

        if to_branch != etpConst['branch']:
            mytxt = "%s: %s %s" % (blue(_("Please setup your branch to")),bold(to_branch),blue(_("and retry")),)
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "error",
                header = darkred(" !! ")
            )
            return None

        mytxt = red("%s ...") % (_("Moving database (if not exists)"),)
        self.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = darkgreen(" @@ ")
        )
        branch_dbdir = self.get_local_database_dir(repo)
        old_branch_dbdir = self.get_local_database_dir(repo, from_branch)
        if (not os.path.isdir(branch_dbdir)) and os.path.isdir(old_branch_dbdir):
            shutil.copytree(old_branch_dbdir,branch_dbdir)

        mytxt = red("%s ...") % (_("Switching packages"),)
        self.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = darkgreen(" @@ ")
        )
        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo, lock_remote = False)

        already_switched = set()
        not_found = set()
        switched = set()
        ignored = set()
        no_checksum = set()

        maxcount = len(idpackages)
        count = 0
        for idpackage in idpackages:
            count += 1

            cur_branch = dbconn.retrieveBranch(idpackage)
            atom = dbconn.retrieveAtom(idpackage)
            if cur_branch == to_branch:
                already_switched.add(idpackage)
                self.updateProgress(
                    red("%s %s, %s %s" % (
                            _("Ignoring"),
                            bold(atom),
                            _("already in branch"),
                            cur_branch,
                        )
                    ),
                    importance = 0,
                    type = "info",
                    header = darkgreen(" @@ "),
                    count = (count,maxcount,)
                )
                ignored.add(idpackage)
                continue

            mytxt = blue("%s ...") % (_("configuring package information"),)
            self.updateProgress(
                "[%s=>%s] %s, %s" % (
                    brown(cur_branch),
                    bold(to_branch),
                    darkgreen(atom),
                    mytxt,
                ),
                importance = 0,
                type = "info",
                header = darkgreen(" @@ "),
                back = True,
                count = (count,maxcount,)
            )
            switch_status = dbconn.switchBranch(idpackage,to_branch)
            if not switch_status:
                # remove idpackage
                dbconn.removePackage(idpackage)
            dbconn.commitChanges()
            switched.add(idpackage)

        dbconn.commitChanges()

        # now migrate counters
        dbconn.moveCountersToBranch(to_branch)

        self.close_server_database(dbconn)
        mytxt = blue("%s.") % (_("migration loop completed"),)
        self.updateProgress(
            "[%s=>%s] %s" % (
                    brown(from_branch),
                    bold(to_branch),
                    mytxt,
            ),
            importance = 1,
            type = "info",
            header = darkgreen(" * ")
        )

        return switched, already_switched, ignored, not_found, no_checksum

    def get_entropy_sets(self, repo = None, branch = etpConst['branch']):

        if repo == None: repo = self.default_repository

        sets_dir = self.get_local_database_sets_dir(repo, branch)
        if not (os.path.isdir(sets_dir) and os.access(sets_dir,os.R_OK)):
            return {}

        mydata = {}
        items = os.listdir(sets_dir)
        for item in items:

            try:
                item_clean = str(item)
            except (UnicodeEncodeError,UnicodeDecodeError,):
                continue
            item_path = os.path.join(sets_dir,item)
            if not (os.path.isfile(item_path) and os.access(item_path,os.R_OK)):
                continue
            item_elements = self.entropyTools.extract_packages_from_set_file(item_path)
            if item_elements:
                mydata[item_clean] = item_elements.copy()

        return mydata

    def get_configured_package_sets(self, repo = None, branch = etpConst['branch'], validate = True):

        if repo == None: repo = self.default_repository

        # portage sets
        sets_data = self.SpmService.get_sets_expanded(builtin_sets = False)
        sets_data.update(self.get_entropy_sets(repo, branch))

        if validate:
            invalid_sets = set()
            # validate
            for setname in sets_data:
                good = True
                for atom in sets_data[setname]:
                    dbconn = self.openServerDatabase(just_reading = True, repo = repo)
                    match = dbconn.atomMatch(atom)
                    if match[0] == -1:
                        good = False
                        break
                if not good: invalid_sets.add(setname)
            for invalid_set in invalid_sets:
                del sets_data[invalid_set]

        return sets_data

    def update_database_package_sets(self, repo = None, dbconn = None):

        if repo == None: repo = self.default_repository
        package_sets = self.get_configured_package_sets(repo)
        if dbconn == None: dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        dbconn.clearPackageSets()
        if package_sets: dbconn.insertPackageSets(package_sets)
        dbconn.commitChanges()


class RemoteDbSkelInterface:

    def __init__(self):
        self.dbconn = None
        self.cursor = None
        self.plain_cursor = None
        self.connection_data = {}
        try:
            import MySQLdb, _mysql_exceptions
            from MySQLdb.constants import FIELD_TYPE
            from MySQLdb.converters import conversions
        except ImportError:
            raise exceptionTools.LibraryNotFound('LibraryNotFound: dev-python/mysql-python not found')
        self.mysql = MySQLdb
        self.mysql_exceptions = _mysql_exceptions
        self.FIELD_TYPE = FIELD_TYPE
        self.conversion_dict = conversions.copy()
        self.conversion_dict[self.FIELD_TYPE.DECIMAL] = int
        self.conversion_dict[self.FIELD_TYPE.LONG] = int
        self.conversion_dict[self.FIELD_TYPE.FLOAT] = float
        self.conversion_dict[self.FIELD_TYPE.NEWDECIMAL] = float

    def check_connection(self):
        if self.dbconn == None:
            raise exceptionTools.ConnectionError('ConnectionError: %s' % (_("not connected to database"),))
        self._check_needed_reconnect()

    def _check_needed_reconnect(self):
        if self.dbconn == None:
            return
        try:
            self.dbconn.ping()
        except self.mysql_exceptions.OperationalError, e:
            if e[0] != 2006:
                raise
            else:
               self.connect()
               return True
        return False

    def _raise_not_implemented_error(self):
        raise exceptionTools.NotImplementedError('NotImplementedError: %s' % (_('method not implemented'),))

    def set_connection_data(self, data):
        self.connection_data = data.copy()
        if not self.connection_data.has_key('converters') and self.conversion_dict:
            self.connection_data['converters'] = self.conversion_dict.copy()

    def check_connection_data(self):
        if not self.connection_data:
            raise exceptionTools.PermissionDenied('ConnectionError: %s' % (_("no connection data"),))

    def connect(self):
        kwargs = {}
        keys = [
            ('host',"hostname"),
            ('user',"username"),
            ('passwd',"password"),
            ('db',"dbname"),
            ('port',"port"),
            ('conv',"converters"), # mysql type converter dict
        ]
        for ckey, dkey in keys:
            if not self.connection_data.has_key(dkey):
                continue
            kwargs[ckey] = self.connection_data.get(dkey)

        try:
            self.dbconn = self.mysql.connect(**kwargs)
        except self.mysql_exceptions.OperationalError, e:
            raise exceptionTools.ConnectionError('ConnectionError: %s' % (e,))
        self.plain_cursor = self.dbconn.cursor()
        self.cursor = self.mysql.cursors.DictCursor(self.dbconn)
        return True

    def disconnect(self):
        self.check_connection()
        if hasattr(self.cursor,'close'):
            self.cursor.close()
        if hasattr(self.dbconn,'close'):
            self.dbconn.close()
        self.dbconn = None
        self.cursor = None
        self.plain_cursor = None
        self.connection_data.clear()
        return True

    def commit(self):
        self.check_connection()
        return self.dbconn.commit()

    def execute_script(self, myscript):
        pty = None
        for line in myscript.split(";"):
            line = line.strip()
            if not line:
                continue
            pty = self.cursor.execute(line)
        return pty

    def execute_query(self, *args):
        return self.cursor.execute(*args)

    def execute_many(self, query, myiter):
        return self.cursor.executemany(query, myiter)

    def fetchone(self):
        return self.cursor.fetchone()

    def fetchall(self):
        return self.cursor.fetchall()

    def fetchmany(self, *args, **kwargs):
        return self.cursor.fetchmany(*args,**kwargs)

    def lastrowid(self):
        return self.cursor.lastrowid

    def table_exists(self, table):
        self.check_connection()
        self.cursor.execute("show tables like %s", (table,))
        rslt = self.cursor.fetchone()
        if rslt:
            return True
        return False

    def column_in_table_exists(self, table, column):
        t_ex = self.table_exists(table)
        if not t_ex:
            return False
        self.cursor.execute("show columns from "+table)
        data = self.cursor.fetchall()
        for row in data:
            if row['Field'] == column:
                return True
        return False

    def fetchall2set(self, item):
        mycontent = set()
        for x in item:
            mycontent |= set(x)
        return mycontent

    def fetchall2list(self, item):
        content = []
        for x in item:
            content += list(x)
        return content

    def fetchone2list(self, item):
        return list(item)

    def fetchone2set(self, item):
        return set(item)

    def _generate_sql(self, action, table, data, where = ''):
        sql = u''
        keys = sorted(data.keys())
        if action == "update":
            sql += 'UPDATE %s SET ' % (self.dbconn.escape_string(table),)
            keys_data = []
            for key in keys:
                keys_data.append("%s = '%s'" % (
                        self.dbconn.escape_string(key),
                        self.dbconn.escape_string(unicode(data[key]).encode('utf-8')).decode('utf-8')
                    )
                )
            sql += ', '.join(keys_data)
            sql += ' WHERE %s' % (where,)
        elif action == "insert":
            sql = u'INSERT INTO %s (%s) VALUES (%s)' % (
                self.dbconn.escape_string(table),
                u', '.join([self.dbconn.escape_string(x) for x in keys]),
                u', '.join([u"'"+self.dbconn.escape_string(unicode(data[x]).encode('utf-8')).decode('utf-8')+"'" for x in keys])
            )
        return sql

class DistributionUGCInterface(RemoteDbSkelInterface):

    SQL_TABLES = {
        'entropy_base': """
            CREATE TABLE `entropy_base` (
            `idkey` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
            `key` VARCHAR( 255 )  collate utf8_bin NOT NULL,
            KEY `key` (`key`)
            );
        """,
        'entropy_votes': """
            CREATE TABLE `entropy_votes` (
            `idvote` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
            `idkey` INT UNSIGNED NOT NULL,
            `userid` INT UNSIGNED NOT NULL,
            `vdate` DATE NOT NULL,
            `vote` TINYINT NOT NULL,
            `ts` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
            FOREIGN KEY  (`idkey`) REFERENCES `entropy_base` (`idkey`)
            );
        """,
        'entropy_user_scores': """
            CREATE TABLE `entropy_user_scores` (
            `userid` INT UNSIGNED NOT NULL PRIMARY KEY,
            `score` INT UNSIGNED NOT NULL DEFAULT 0
            );
        """,
        'entropy_downloads': """
            CREATE TABLE `entropy_downloads` (
            `iddownload` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
            `idkey` INT UNSIGNED NOT NULL,
            `ddate` DATE NOT NULL,
            `count` INT UNSIGNED NULL DEFAULT '0',
            KEY `idkey` (`idkey`,`ddate`),
            KEY `idkey_2` (`idkey`),
            FOREIGN KEY  (`idkey`) REFERENCES `entropy_base` (`idkey`)
            );
        """,
        'entropy_downloads_data': """
            CREATE TABLE `entropy_downloads_data` (
            `iddownload` INT UNSIGNED NOT NULL,
            `ip_address` VARCHAR(40) NULL DEFAULT '',
            `location` TINYINT NULL DEFAULT 0,
            FOREIGN KEY  (`iddownload`) REFERENCES `entropy_downloads` (`iddownload`)
            );
        """,
        'entropy_docs': """
            CREATE TABLE `entropy_docs` (
            `iddoc` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
            `idkey` INT UNSIGNED NOT NULL,
            `userid` INT UNSIGNED NOT NULL,
            `username` VARCHAR( 255 ),
            `iddoctype` TINYINT NOT NULL,
            `ddata` TEXT NOT NULL,
            `title` VARCHAR( 512 ),
            `description` VARCHAR( 4000 ),
            `ts` TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
            KEY `idkey` (`idkey`),
            KEY `userid` (`userid`),
            KEY `idkey_2` (`idkey`,`userid`,`iddoctype`),
            KEY `title` (`title`(333)),
            KEY `description` (`description`(333)),
            FOREIGN KEY  (`idkey`) REFERENCES `entropy_base` (`idkey`)
            );
        """,
        'entropy_doctypes': """
            CREATE TABLE `entropy_doctypes` (
            `iddoctype` TINYINT NOT NULL PRIMARY KEY,
            `description` TEXT NOT NULL
            );
        """,
        'entropy_docs_keywords': """
            CREATE TABLE `entropy_docs_keywords` (
            `iddoc` INT UNSIGNED NOT NULL,
            `keyword` VARCHAR( 100 ),
            KEY `keyword` (`keyword`),
            FOREIGN KEY  (`iddoc`) REFERENCES `entropy_docs` (`iddoc`)
            );
        """,
    }
    VOTE_RANGE = etpConst['ugc_voterange'] # [1, 2, 3, 4, 5]
    VIRUS_CHECK_EXEC = '/usr/bin/clamscan'
    VIRUS_CHECK_ARGS = []
    gdata = None
    YouTube = None
    YouTubeService = None
    entropy_docs_title_len = 512
    entropy_docs_description_len = 4000
    entropy_docs_keyword_len = 100
    COMMENTS_SCORE_WEIGHT = 5
    DOCS_SCORE_WEIGHT = 10
    VOTES_SCORE_WEIGHT = 2
    DOWNLOADS_STORE_LOCATION = 0

    '''
        dependencies:
            dev-python/gdata
    '''
    def __init__(self, connection_data, store_path, store_url = ''):
        import entropyTools, dumpTools
        self.entropyTools, self.dumpTools = entropyTools, dumpTools
        self.store_url = store_url
        self.FLOOD_INTERVAL = 30
        self.DOC_TYPES = etpConst['ugc_doctypes'].copy()
        self.UPLOADED_DOC_TYPES = [
            self.DOC_TYPES['image'],
            self.DOC_TYPES['generic_file']
        ]
        RemoteDbSkelInterface.__init__(self)
        self.set_connection_data(connection_data)
        self.connect()
        self.initialize_tables()
        self.initialize_doctypes()
        self.setup_store_path(store_path)
        self.system_name = etpConst['systemname']
        from datetime import datetime
        self.datetime = datetime
        try:
            import gdata
            import gdata.youtube
            import gdata.youtube.service
            self.gdata = gdata
            self.YouTube = gdata.youtube
            self.YouTubeService = gdata.youtube.service
        except ImportError:
            pass

        self.cached_results = {
            #'get_ugc_allvotes': (self.get_ugc_allvotes, [], {}, 86400),
            'get_ugc_alldownloads': (self.get_ugc_alldownloads, [], {}, 86400),
            #'get_users_scored_count': (self.get_users_scored_count, [], {}, 86400),
            'get_total_downloads_count': (self.get_total_downloads_count, [], {}, 7200),
        }

    def get_current_time(self):
        return int(time.time())

    def cache_results(self):
        for cache_item in self.cached_results:
            fdata = self.cached_results.get(cache_item)
            if fdata == None: return
            func, args, kwargs, exp_time = fdata
            key = self.get_cache_item_key(cache_item)
            r = func(*args,**kwargs)
            self.dumpTools.dumpobj(key, r)

    def get_cache_item_key(self, cache_item):
        return os.path.join(etpCache['ugc_srv_cache'],cache_item)

    def cache_result(self, cache_item, r):
        if not self.cached_results.get(cache_item): return None
        key = self.get_cache_item_key(cache_item)
        self.dumpTools.dumpobj(key, r)

    # expired get_ugc_alldownloads 0 86400 1228577077
    def get_cached_result(self, cache_item):
        fdata = self.cached_results.get(cache_item)
        if fdata == None: return None
        func, args, kwargs, exp_time = fdata

        key = self.get_cache_item_key(cache_item)
        cur_time = self.get_current_time()
        cache_time = self.dumpTools.getobjmtime(key)
        if (cache_time + exp_time) < cur_time:
            # expired
            return None
        return self.dumpTools.loadobj(key)

    def setup_store_path(self, path):
        path = os.path.realpath(path)
        if not os.path.isabs(path):
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_("not a valid directory path"),))
        if not os.path.isdir(path):
            try:
                os.makedirs(path)
            except OSError, e:
                raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (e,))
            if etpConst['entropygid'] != None:
                const_setup_perms(path,etpConst['entropygid'])
        self.STORE_PATH = path

    def initialize_tables(self):
        notable = False
        for table in self.SQL_TABLES:
            if self.table_exists(table):
                continue
            notable = True
            self.execute_script(self.SQL_TABLES[table])
        if notable:
            self.commit()

    def initialize_doctypes(self):
        for mydoctype in self.DOC_TYPES:
            if self.is_iddoctype_available(self.DOC_TYPES[mydoctype]):
                continue
            self.insert_iddoctype(self.DOC_TYPES[mydoctype],mydoctype)

    def is_iddoctype_available(self, iddoctype):
        self.check_connection()
        rows = self.execute_query('SELECT `iddoctype` FROM entropy_doctypes WHERE `iddoctype` = %s', (iddoctype,))
        if rows:
            return True
        return False

    def is_pkgkey_available(self, key):
        self.check_connection()
        rows = self.execute_query('SELECT `idkey` FROM entropy_base WHERE `key` = %s', (key,))
        if rows:
            return True
        return False

    def is_iddoc_available(self, iddoc):
        self.check_connection()
        rows = self.execute_query('SELECT `iddoc` FROM entropy_docs WHERE `iddoc` = %s', (iddoc,))
        if rows:
            return True
        return False

    def insert_iddoctype(self, iddoctype, description, do_commit = False):
        self.check_connection()
        self.execute_query('INSERT INTO entropy_doctypes VALUES (%s,%s)', (iddoctype,description,))
        if do_commit: self.commit()

    def insert_pkgkey(self, key, do_commit = False):
        self.check_connection()
        self.execute_query('INSERT INTO entropy_base VALUES (%s,%s)', (None,key,))
        myid = self.lastrowid()
        if do_commit: self.commit()
        return myid

    def insert_download(self, key, ddate, count = 0, do_commit = False):
        self.check_connection()
        idkey = self.handle_pkgkey(key)
        self.execute_query('INSERT INTO entropy_downloads VALUES (%s,%s,%s,%s)', (None,idkey,ddate,count))
        myid = self.lastrowid()
        if do_commit: self.commit()
        return myid

    def update_download(self, iddownload, key, ddate, incr, do_commit = False):
        self.check_connection()
        self.execute_query('SELECT `count` FROM entropy_downloads WHERE `iddownload` = %s', (iddownload,))
        data = self.fetchone()
        if not data: # !?!?
            return self.insert_download(key, ddate, count = 1)
        count = data['count']+incr
        # now update
        self.execute_query('UPDATE entropy_downloads SET `count` = %s WHERE `iddownload` = %s', (count,iddownload,))
        if do_commit: self.commit()
        return iddownload

    def store_download_data(self, iddownload, ip_addr, do_commit = False):
        self.execute_query('INSERT INTO entropy_downloads_data VALUES (%s,%s,%s)', (iddownload,ip_addr,self.DOWNLOADS_STORE_LOCATION))
        if do_commit: self.commit()

    def get_date(self):
        mytime = time.time()
        mydate = self.datetime.fromtimestamp(mytime)
        mydate = self.datetime(mydate.year,mydate.month,mydate.day)
        return mydate

    def get_iddownload(self, key, ddate):
        self.check_connection()
        idkey = self.handle_pkgkey(key)
        self.execute_query('SELECT `iddownload` FROM entropy_downloads WHERE `idkey` = %s AND `ddate` = %s', (idkey,ddate,))
        data = self.fetchone()
        if data:
            return data['iddownload']
        return -1

    def get_idkey(self, key):
        self.check_connection()
        self.execute_query('SELECT `idkey` FROM entropy_base WHERE `key` = %s', (key,))
        data = self.fetchone()
        if data: return data['idkey']
        return -1

    def get_iddoctype(self, iddoc):
        self.check_connection()
        self.execute_query('SELECT `iddoctype` FROM entropy_docs WHERE `iddoc` = %s', (iddoc,))
        data = self.fetchone()
        if data: return data['iddoctype']
        return -1

    def get_pkgkey(self, idkey):
        self.check_connection()
        self.execute_query('SELECT `key` FROM entropy_base WHERE `idkey` = %s', (idkey,))
        data = self.fetchone()
        if data: return data['key']

    def get_ugc_metadata(self, pkgkey):
        self.check_connection()
        metadata = {
            'vote': 0.0,
            'downloads': 0,
        }
        self.execute_query('SELECT * FROM entropy_docs,entropy_base WHERE entropy_base.`idkey` = entropy_docs.`idkey` AND entropy_base.`key` = %s', (pkgkey,))
        metadata['docs'] = self.fetchall()
        for mydict in metadata['docs']:
            mydict = self._get_ugc_extra_metadata(mydict)
        metadata['vote'] = self.get_ugc_vote(pkgkey)
        metadata['downloads'] = self.get_ugc_downloads(pkgkey)
        return metadata

    def get_ugc_keywords(self, iddoc):
        self.execute_query('SELECT `keyword` FROM entropy_docs_keywords WHERE `iddoc` = %s order by `keyword`', (iddoc,))
        data = self.fetchall()
        if not data: return []
        mykeys = []
        for mydict in data:
            if not mydict.has_key('keyword'):
                continue
            mykeys.append(mydict['keyword'])
        return mykeys

    def get_ugc_metadata_doctypes(self, pkgkey, typeslist):
        self.check_connection()
        metadata = []
        self.execute_query('SELECT * FROM entropy_docs,entropy_base WHERE entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_base.`key` = %s AND entropy_docs.`iddoctype` IN %s ORDER BY entropy_docs.`ts` ASC', (pkgkey,typeslist,))
        metadata = self.fetchall()
        for mydict in metadata:
            mydict = self._get_ugc_extra_metadata(mydict)
        return metadata

    def get_ugc_metadata_doctypes_by_identifiers(self, identifiers, typeslist):
        self.check_connection()
        identifiers = list(identifiers)
        if len(identifiers) < 2:
            identifiers += [0]
        typeslist = list(typeslist)
        if len(typeslist) < 2:
            typeslist += [0]
        self.execute_query('SELECT * FROM entropy_docs WHERE `iddoc` IN %s AND `iddoctype` IN %s', (identifiers,typeslist,))
        metadata = self.fetchall()
        for mydict in metadata:
            mydict = self._get_ugc_extra_metadata(mydict)
        return metadata

    def get_ugc_metadata_by_identifiers(self, identifiers):
        self.check_connection()
        identifiers = list(identifiers)
        if len(identifiers) < 2:
            identifiers += [0]
        self.execute_query('SELECT * FROM entropy_docs WHERE `iddoc` IN %s', (identifiers,))
        metadata = self.fetchall()
        for mydict in metadata:
            mydict = self._get_ugc_extra_metadata(mydict)
        return metadata

    def _get_ugc_extra_metadata(self, mydict):
        mydict['store_url'] = None
        mydict['keywords'] = self.get_ugc_keywords(mydict['iddoc'])
        if mydict.has_key("key"): mydict['pkgkey'] = mydict['key']
        else: mydict['pkgkey'] = self.get_pkgkey(mydict['idkey'])
        # for binary files, get size too
        mydict['size'] = 0
        if mydict['iddoctype'] in self.UPLOADED_DOC_TYPES:
            myfilename = mydict['ddata']
            if not isinstance(myfilename,basestring):
                myfilename = myfilename.tostring()
            mypath = os.path.join(self.STORE_PATH,myfilename)
            if os.path.isfile(mypath) and os.access(mypath,os.R_OK):
                try:
                    mydict['size'] = self.entropyTools.get_file_size(mypath)
                except OSError:
                    pass
            mydict['store_url'] = os.path.join(self.store_url,myfilename)
        else:
            mydata = mydict['ddata']
            if not isinstance(mydata,basestring):
                mydata = mydata.tostring()
            try:
                mydict['size'] = len(mydata)
            except:
                pass
        return mydict

    def get_ugc_vote(self, pkgkey):
        self.check_connection()
        vote = 0.0
        self.execute_query('SELECT avg(entropy_votes.`vote`) as avg_vote FROM entropy_votes,entropy_base WHERE entropy_base.`key` = %s AND entropy_base.idkey = entropy_votes.idkey', (pkgkey,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data.get('avg_vote'):
                return data['avg_vote']
        return vote

    def get_ugc_allvotes(self):

        # cached?
        cache_item = 'get_ugc_allvotes'
        cached = self.get_cached_result(cache_item)
        if cached != None: return cached

        self.check_connection()
        vote_data = {}
        self.execute_query('SELECT entropy_base.`key` as `vkey`,avg(entropy_votes.vote) as `avg_vote` FROM entropy_votes,entropy_base WHERE entropy_votes.`idkey` = entropy_base.`idkey` GROUP BY entropy_base.`key`')
        data = self.fetchall()
        for d_dict in data:
            vote_data[d_dict['vkey']] = d_dict['avg_vote']

        # do cache
        self.cache_result(cache_item, vote_data)
        return vote_data

    def get_ugc_downloads(self, pkgkey):
        self.check_connection()
        downloads = 0
        self.execute_query('SELECT SQL_CACHE sum(entropy_downloads.`count`) as `tot_downloads` FROM entropy_downloads,entropy_base WHERE entropy_base.key = %s AND entropy_base.idkey = entropy_downloads.idkey', (pkgkey,))
        data = self.fetchone()
        if data['tot_downloads'] != None:
            downloads = data['tot_downloads']
        return downloads

    def get_ugc_alldownloads(self):

        # cached?
        cache_item = 'get_ugc_alldownloads'
        cached = self.get_cached_result(cache_item)
        if cached != None: return cached

        self.check_connection()
        down_data = {}
        self.execute_query('SELECT SQL_CACHE entropy_base.`key` as `vkey`,sum(entropy_downloads.`count`) as `tot_downloads` FROM entropy_downloads,entropy_base WHERE entropy_downloads.`idkey` = entropy_base.`idkey` GROUP BY entropy_base.`idkey`')
        data = self.fetchall()
        for d_dict in data:
            down_data[d_dict['vkey']] = d_dict['tot_downloads']

        # do cache
        self.cache_result(cache_item, down_data)
        return down_data

    def get_iddoc_userid(self, iddoc):
        self.check_connection()
        self.execute_query('SELECT `userid` FROM entropy_docs WHERE `iddoc` = %s', (iddoc,))
        data = self.fetchone()
        if not data:
            return None
        elif not data.has_key('userid'):
            return None
        return data['userid']

    def get_total_comments_count(self):
        self.check_connection()
        self.execute_query('SELECT count(`iddoc`) as comments FROM entropy_docs WHERE `iddoctype` = %s', (self.DOC_TYPES['comments'],))
        data = self.fetchone()
        if isinstance(data,dict):
            if data['comments']: return data['comments']
        return 0

    def get_total_documents_count(self):
        self.check_connection()
        self.execute_query('SELECT count(`iddoc`) as comments FROM entropy_docs WHERE `iddoctype` != %s', (self.DOC_TYPES['comments'],))
        data = self.fetchone()
        if isinstance(data,dict):
            if data['comments']: return data['comments']
        return 0

    def get_total_votes_count(self):
        self.check_connection()
        self.execute_query('SELECT count(`idvote`) as votes FROM entropy_votes')
        data = self.fetchone()
        if isinstance(data,dict):
            if data['votes']: return data['votes']
        return 0

    def get_total_downloads_count(self):

        # cached?
        cache_item = 'get_total_downloads_count'
        cached = self.get_cached_result(cache_item)
        if cached != None: return cached

        self.check_connection()
        self.execute_query('SELECT SQL_CACHE sum(entropy_downloads.`count`) as downloads FROM entropy_downloads')
        data = self.fetchone()
        r = 0
        if isinstance(data,dict):
            if data['downloads']: r = int(data['downloads'])

        # do cache
        self.cache_result(cache_item, r)
        return r

    def get_user_score_ranking(self, userid):
        self.check_connection()
        self.execute_query('SET @row = 0')
        self.execute_query('SELECT Row, col_a FROM (SELECT @row := @row + 1 AS Row, userid AS col_a FROM entropy_user_scores ORDER BY score DESC) As derived1 WHERE col_a = %s', (userid,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data.get('Row'): return data['Row']
        return 0

    def get_users_scored_count(self):

        # cached?
        cache_item = 'get_users_scored_count'
        cached = self.get_cached_result(cache_item)
        if cached != None: return cached

        self.check_connection()
        self.execute_query('SELECT SQL_CACHE count(`userid`) as mycount FROM entropy_user_scores')
        data = self.fetchone()
        r = 0
        if isinstance(data,dict):
            if data.get('mycount'): r = data['mycount']

        # do cache
        self.cache_result(cache_item, r)
        return r

    def is_user_score_available(self, userid):
        self.check_connection()
        rows = self.execute_query('SELECT `userid` FROM entropy_user_scores WHERE `userid` = %s', (userid,))
        if rows:
            return True
        return False

    def calculate_user_score(self, userid):
        comments = self.get_user_comments_count(userid)
        docs = self.get_user_docs_count(userid)
        votes = self.get_user_votes_count(userid)
        return (comments*self.COMMENTS_SCORE_WEIGHT)+(docs*self.DOCS_SCORE_WEIGHT)+(votes*self.VOTES_SCORE_WEIGHT)

    def update_user_score(self, userid):
        self.check_connection()
        avail = self.is_user_score_available(userid)
        myscore = self.calculate_user_score(userid)
        if avail:
            self.execute_query('UPDATE entropy_user_scores SET score = %s WHERE `userid` = %s', (myscore,userid,))
        else:
            self.execute_query('INSERT INTO entropy_user_scores VALUES (%s,%s)', (userid,myscore,))
        return myscore

    def get_user_score(self, userid):
        self.check_connection()
        myscore = None
        self.execute_query('SELECT score FROM entropy_user_scores WHERE userid = %s',(userid,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data.has_key('score'): myscore = data.get('score')
        if myscore == None: myscore = self.update_user_score(userid)
        return myscore

    def get_users_score_ranking(self, offset = 0, count = 0):
        self.check_connection()
        limit_string = ''
        if count:
            limit_string = ' LIMIT %s,%s' % (offset,count,)
        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *,userid,score FROM entropy_user_scores ORDER BY score DESC'+limit_string)
        data = self.fetchall()

        self.execute_query('SELECT FOUND_ROWS() as count')
        rdata = self.fetchone()
        found_rows = 0
        if isinstance(rdata,dict):
            if rdata.has_key('count'):
                found_rows = rdata.get('count')

        return found_rows, data

    def get_user_votes_average(self, userid):
        self.check_connection()
        self.execute_query('SELECT avg(`vote`) as vote_avg FROM entropy_votes WHERE `userid` = %s', (userid,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data['vote_avg']: return round(float(data['vote_avg']),2)
        return 0.0

    def get_user_alldocs(self, userid):
        self.check_connection()
        self.execute_query('SELECT * FROM entropy_docs,entropy_base WHERE entropy_docs.`userid` = %s AND entropy_base.idkey = entropy_docs.idkey ORDER BY entropy_base.`key`', (userid,))
        return self.fetchall()

    def get_user_docs(self, userid):
        return self.get_user_generic_doctype(userid, self.DOC_TYPES['comments'], doctype_sql_cmp = "!=")

    def get_user_comments(self, userid):
        return self.get_user_generic_doctype(userid, self.DOC_TYPES['comments'], doctype_sql_cmp = "=")

    def get_user_votes(self, userid):
        self.check_connection()
        self.execute_query('SELECT * FROM entropy_votes WHERE `userid` = %s', (userid,))
        return self.fetchall()

    def get_user_generic_doctype(self, userid, doctype, doctype_sql_cmp = "="):
        self.check_connection()
        self.execute_query('SELECT * FROM entropy_docs WHERE `userid` = %s AND `iddoctype` '+doctype_sql_cmp+' %s', (userid,doctype,))
        return self.fetchall()

    def get_user_generic_doctype_count(self, userid, doctype, doctype_sql_cmp = "="):
        self.check_connection()
        self.execute_query('SELECT count(`iddoc`) as docs FROM entropy_docs WHERE `userid` = %s AND `iddoctype` '+doctype_sql_cmp+' %s', (userid,doctype,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data['docs']: return data['docs']
        return 0

    def get_user_comments_count(self, userid):
        return self.get_user_generic_doctype_count(userid, self.DOC_TYPES['comments'])

    def get_user_docs_count(self, userid):
        return self.get_user_generic_doctype_count(userid, self.DOC_TYPES['comments'], doctype_sql_cmp = "!=")

    def get_user_images_count(self, userid):
        return self.get_user_generic_doctype_count(userid, self.DOC_TYPES['image'])

    def get_user_files_count(self, userid):
        return self.get_user_generic_doctype_count(userid, self.DOC_TYPES['generic_file'])

    def get_user_yt_videos_count(self, userid):
        return self.get_user_generic_doctype_count(userid, self.DOC_TYPES['youtube_video'])

    def get_user_votes_count(self, userid):
        self.check_connection()
        self.execute_query('SELECT count(`idvote`) as votes FROM entropy_votes WHERE `userid` = %s', (userid,))
        data = self.fetchone()
        if isinstance(data,dict):
            if data['votes']: return data['votes']
        return 0

    def get_user_stats(self, userid):
        mydict = {}
        mydict['comments'] = self.get_user_comments_count(userid)
        mydict['docs'] = self.get_user_docs_count(userid)
        mydict['images'] = self.get_user_images_count(userid)
        mydict['files'] = self.get_user_files_count(userid)
        mydict['yt_videos'] = self.get_user_yt_videos_count(userid)
        mydict['votes'] = self.get_user_votes_count(userid)
        mydict['votes_avg'] = self.get_user_votes_average(userid)
        mydict['total_docs'] = mydict['comments'] + mydict['docs']
        mydict['score'] = self.get_user_score(userid)
        mydict['ranking'] = self.get_user_score_ranking(userid)
        return mydict

    def get_distribution_stats(self):
        self.check_connection()
        mydict = {}
        mydict['comments'] = self.get_total_comments_count()
        mydict['documents'] = self.get_total_documents_count()
        mydict['votes'] = self.get_total_votes_count()
        mydict['downloads'] = self.get_total_downloads_count()
        return mydict

    def search_pkgkey_items(self, pkgkey_string, iddoctypes = None, results_offset = 0, results_limit = 30, order_by = None):
        self.check_connection()

        if iddoctypes == None:
            iddoctypes = [self.DOC_TYPES[x] for x in self.DOC_TYPES]
        iddoctypes = "("+', '.join([str(self.DOC_TYPES[x]) for x in self.DOC_TYPES])+")"
        myterm = "%"+pkgkey_string+"%"

        search_params = [myterm,results_offset,results_limit]

        order_by_string = ''
        if order_by == "key":
            order_by_string = 'ORDER BY entropy_base.`key`'
        elif order_by == "username":
            order_by_string = 'ORDER BY entropy_docs.`username`'
        elif order_by == "vote":
            order_by_string = 'ORDER BY avg_vote DESC'
        elif order_by == "downloads":
            order_by_string = 'ORDER BY tot_downloads DESC'

        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *, avg(entropy_votes.`vote`) as avg_vote, sum(entropy_downloads.`count`) as `tot_downloads`, entropy_user_scores.`score` as `score` FROM entropy_docs,entropy_base,entropy_votes,entropy_downloads,entropy_user_scores WHERE entropy_base.`key` LIKE %s AND entropy_docs.`iddoctype` IN '+iddoctypes+' AND entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_votes.`idkey` = entropy_base.`idkey` AND entropy_downloads.`idkey` = entropy_base.`idkey` AND entropy_docs.`userid` = entropy_user_scores.`userid` GROUP BY entropy_docs.`iddoc` '+order_by_string+' LIMIT %s,%s', search_params)

        results = self.fetchall()
        self.execute_query('SELECT FOUND_ROWS() as count')
        data = self.fetchone()
        found_rows = 0
        if isinstance(data,dict):
            if data.has_key('count'):
                found_rows = data.get('count')
        return results, found_rows

    def search_username_items(self, pkgkey_string, iddoctypes = None, results_offset = 0, results_limit = 30, order_by = None):
        self.check_connection()

        if iddoctypes == None:
            iddoctypes = [self.DOC_TYPES[x] for x in self.DOC_TYPES]
        iddoctypes = "("+', '.join([str(self.DOC_TYPES[x]) for x in self.DOC_TYPES])+")"
        myterm = "%"+pkgkey_string+"%"

        search_params = [myterm,results_offset,results_limit]

        order_by_string = ''
        if order_by == "key":
            order_by_string = 'ORDER BY entropy_base.`key`'
        elif order_by == "username":
            order_by_string = 'ORDER BY entropy_docs.`username`'
        elif order_by == "vote":
            order_by_string = 'ORDER BY avg_vote DESC'
        elif order_by == "downloads":
            order_by_string = 'ORDER BY tot_downloads DESC'

        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *, avg(entropy_votes.`vote`) as avg_vote, sum(entropy_downloads.`count`) as `tot_downloads`, entropy_user_scores.`score` as `score` FROM entropy_docs,entropy_base,entropy_votes,entropy_downloads,entropy_user_scores WHERE entropy_docs.`username` LIKE %s AND entropy_docs.`iddoctype` IN '+iddoctypes+' AND entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_votes.`idkey` = entropy_base.`idkey` AND entropy_downloads.`idkey` = entropy_base.`idkey` AND entropy_docs.`userid` = entropy_user_scores.`userid` GROUP BY entropy_docs.`iddoc` '+order_by_string+' LIMIT %s,%s', search_params)

        results = self.fetchall()
        self.execute_query('SELECT FOUND_ROWS() as count')
        data = self.fetchone()
        found_rows = 0
        if isinstance(data,dict):
            if data.has_key('count'):
                found_rows = data.get('count')
        return results, found_rows

    def search_content_items(self, pkgkey_string, iddoctypes = None, results_offset = 0, results_limit = 30, order_by = None):
        self.check_connection()

        if iddoctypes == None:
            iddoctypes = [self.DOC_TYPES[x] for x in self.DOC_TYPES]
        iddoctypes = "("+', '.join([str(self.DOC_TYPES[x]) for x in self.DOC_TYPES])+")"
        myterm = "%"+pkgkey_string+"%"
        search_params = [myterm,myterm,myterm,results_offset,results_limit]

        order_by_string = ''
        if order_by == "key":
            order_by_string = 'ORDER BY entropy_base.`key`'
        elif order_by == "username":
            order_by_string = 'ORDER BY entropy_docs.`username`'
        elif order_by == "vote":
            order_by_string = 'ORDER BY avg_vote DESC'
        elif order_by == "downloads":
            order_by_string = 'ORDER BY tot_downloads DESC'

        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *, avg(entropy_votes.`vote`) as avg_vote, sum(entropy_downloads.`count`) as `tot_downloads`, entropy_user_scores.`score` as `score` FROM entropy_docs,entropy_base,entropy_votes,entropy_downloads,entropy_user_scores WHERE (entropy_docs.`title` LIKE %s OR entropy_docs.`description` LIKE %s OR entropy_docs.`ddata` LIKE %s) AND entropy_docs.`iddoctype` IN '+iddoctypes+' AND entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_votes.`idkey` = entropy_base.`idkey` AND entropy_downloads.`idkey` = entropy_base.`idkey` AND entropy_docs.`userid` = entropy_user_scores.`userid` GROUP BY entropy_docs.`iddoc` '+order_by_string+' LIMIT %s,%s', search_params)

        results = self.fetchall()
        self.execute_query('SELECT FOUND_ROWS() as count')
        data = self.fetchone()
        found_rows = 0
        if isinstance(data,dict):
            if data.has_key('count'):
                found_rows = data.get('count')
        return results, found_rows

    def search_keyword_items(self, keyword_string, iddoctypes = None, results_offset = 0, results_limit = 30, order_by = None):
        self.check_connection()

        if iddoctypes == None:
            iddoctypes = [self.DOC_TYPES[x] for x in self.DOC_TYPES]
        iddoctypes = "("+', '.join([str(self.DOC_TYPES[x]) for x in self.DOC_TYPES])+")"
        myterm = "%"+keyword_string+"%"

        search_params = [myterm,results_offset,results_limit]

        order_by_string = ''
        if order_by == "key":
            order_by_string = 'ORDER BY entropy_base.`key`'
        elif order_by == "username":
            order_by_string = 'ORDER BY entropy_docs.`username`'
        elif order_by == "vote":
            order_by_string = 'ORDER BY avg_vote DESC'
        elif order_by == "downloads":
            order_by_string = 'ORDER BY tot_downloads DESC'

        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *, avg(entropy_votes.`vote`) as avg_vote, sum(entropy_downloads.`count`) as `tot_downloads`, entropy_user_scores.`score` as `score` FROM entropy_docs,entropy_base,entropy_docs_keywords,entropy_votes,entropy_downloads,entropy_user_scores WHERE entropy_docs_keywords.`keyword` LIKE %s AND entropy_docs.`iddoctype` IN '+iddoctypes+' AND entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_docs_keywords.`iddoc` = entropy_docs.`iddoc` AND entropy_votes.`idkey` = entropy_base.`idkey` AND entropy_downloads.`idkey` = entropy_base.`idkey` AND entropy_docs.`userid` = entropy_user_scores.`userid` GROUP BY entropy_docs.`iddoc` '+order_by_string+' LIMIT %s,%s', search_params)

        results = self.fetchall()
        self.execute_query('SELECT FOUND_ROWS() as count')
        data = self.fetchone()
        found_rows = 0
        if isinstance(data,dict):
            if data.has_key('count'):
                found_rows = data.get('count')
        return results, found_rows

    def search_iddoc_item(self, iddoc_string, iddoctypes = None, results_offset = 0, results_limit = 30, order_by = None):
        self.check_connection()

        if iddoctypes == None:
            iddoctypes = [self.DOC_TYPES[x] for x in self.DOC_TYPES]
        iddoctypes = "("+', '.join([str(self.DOC_TYPES[x]) for x in self.DOC_TYPES])+")"
        try:
            myterm = int(iddoc_string)
        except ValueError:
            return [],0

        search_params = [myterm,results_offset,results_limit]

        order_by_string = ''
        if order_by == "key":
            order_by_string = 'ORDER BY entropy_base.`key`'
        elif order_by == "username":
            order_by_string = 'ORDER BY entropy_docs.`username`'
        elif order_by == "vote":
            order_by_string = 'ORDER BY avg_vote DESC'
        elif order_by == "downloads":
            order_by_string = 'ORDER BY tot_downloads DESC'

        self.execute_query('SELECT SQL_CALC_FOUND_ROWS *, avg(entropy_votes.`vote`) as avg_vote, sum(entropy_downloads.`count`) as `tot_downloads`, entropy_user_scores.`score` as `score` FROM entropy_docs,entropy_base,entropy_votes,entropy_downloads,entropy_user_scores WHERE entropy_docs.`iddoc` = %s AND entropy_docs.`iddoctype` IN '+iddoctypes+' AND entropy_docs.`idkey` = entropy_base.`idkey` AND entropy_votes.`idkey` = entropy_base.`idkey` AND entropy_downloads.`idkey` = entropy_base.`idkey` AND entropy_docs.`userid` = entropy_user_scores.`userid` GROUP BY entropy_docs.`iddoc` '+order_by_string+' LIMIT %s,%s', search_params)

        results = self.fetchall()
        self.execute_query('SELECT FOUND_ROWS() as count')
        data = self.fetchone()
        found_rows = 0
        if isinstance(data,dict):
            if data.has_key('count'):
                found_rows = data.get('count')
        return results, found_rows

    def handle_pkgkey(self, key):
        idkey = self.get_idkey(key)
        if idkey == -1: return self.insert_pkgkey(key)
        return idkey

    def insert_flood_control_check(self, userid):
        self.check_connection()
        self.execute_query('SELECT max(`ts`) as ts FROM entropy_docs WHERE `userid` = %s', (userid,))
        data = self.fetchone()
        if not data:
            return False
        elif not data.has_key('ts'):
            return False
        elif data['ts'] == None:
            return False
        delta = self.datetime.fromtimestamp(time.time()) - data['ts']
        if (delta.days == 0) and (delta.seconds <= self.FLOOD_INTERVAL):
            return True
        return False

    def insert_generic_doc(self, idkey, userid, username, doc_type, data, title, description, keywords, do_commit = False):
        self.check_connection()

        title = title[:self.entropy_docs_title_len]
        description = description[:self.entropy_docs_description_len]

        # flood control
        flood_risk = self.insert_flood_control_check(userid)
        if flood_risk: return 'flooding detected'

        self.execute_query('INSERT INTO entropy_docs VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',(
                None,
                idkey,
                userid,
                username,
                doc_type,
                data,
                title,
                description,
                None,
            )
        )
        if do_commit: self.commit()
        iddoc = self.lastrowid()
        self.insert_keywords(iddoc, keywords)
        self.update_user_score(userid)
        return iddoc

    def insert_keywords(self, iddoc, keywords):
        keywords = keywords.split(",")
        clean_keys = []
        for key in keywords:
            try:
                key = key.strip().split()[0]
            except IndexError:
                continue
            if not key: continue
            if not key.isalnum(): continue
            key = key[:self.entropy_docs_keyword_len]
            if key in clean_keys: continue
            clean_keys.append(key)

        if not clean_keys:
            return

        mydata = []
        for key in clean_keys:
            mydata.append((iddoc, key,))

        self.execute_many('INSERT INTO entropy_docs_keywords VALUES (%s,%s)', mydata)

    def remove_keywords(self, iddoc):
        self.execute_query('DELETE FROM entropy_docs_keywords WHERE `iddoc` = %s',(iddoc,))

    def insert_comment(self, pkgkey, userid, username, comment, title, keywords, do_commit = False):
        self.check_connection()
        idkey = self.handle_pkgkey(pkgkey)
        iddoc = self.insert_generic_doc(idkey, userid, username, self.DOC_TYPES['comments'], comment, title, '', keywords)
        if isinstance(iddoc,basestring):
            return False, iddoc
        if do_commit: self.commit()
        return True, iddoc

    def edit_comment(self, iddoc, new_comment, new_title, new_keywords, do_commit = False):
        self.check_connection()
        if not self.is_iddoc_available(iddoc):
            return False
        new_title = new_title[:self.entropy_docs_title_len]
        self.execute_query('UPDATE entropy_docs SET `ddata` = %s, `title` = %s WHERE `iddoc` = %s AND `iddoctype` = %s',(
                new_comment,
                new_title,
                iddoc,
                self.DOC_TYPES['comments'],
            )
        )
        self.remove_keywords(iddoc)
        self.insert_keywords(iddoc, new_keywords)
        if do_commit: self.commit()
        return True, iddoc

    def remove_comment(self, iddoc):
        self.check_connection()
        userid = self.get_iddoc_userid(iddoc)
        self.remove_keywords(iddoc)
        self.execute_query('DELETE FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                iddoc,
                self.DOC_TYPES['comments'],
            )
        )
        if userid: self.update_user_score(userid)
        return True, iddoc

    # give a vote to an app
    def do_vote(self, pkgkey, userid, vote, do_commit = False):
        self.check_connection()
        idkey = self.handle_pkgkey(pkgkey)
        vote = int(vote)
        if vote not in self.VOTE_RANGE: # weird
            vote = 3 # avg?
        mydate = self.get_date()
        if self.has_user_already_voted(pkgkey, userid):
            return False
        self.execute_query('INSERT INTO entropy_votes VALUES (%s,%s,%s,%s,%s,%s)',(
                None,
                idkey,
                userid,
                mydate,
                vote,
                None,
            )
        )
        if do_commit: self.commit()
        self.update_user_score(userid)
        return True

    def has_user_already_voted(self, pkgkey, userid):
        self.check_connection()
        idkey = self.handle_pkgkey(pkgkey)
        self.execute_query('SELECT `idvote` FROM entropy_votes WHERE `idkey` = %s AND `userid` = %s', (idkey,userid,))
        data = self.fetchone()
        if data:
            return True
        return False

    # increment +1 the number of downloads
    def do_download(self, pkgkey, ip_addr = None, do_commit = False):
        self.check_connection()
        mydate = self.get_date()
        iddownload = self.get_iddownload(pkgkey, mydate)
        if iddownload == -1:
            iddownload = self.insert_download(pkgkey, mydate, count = 1)
        else:
            self.update_download(iddownload, pkgkey, mydate, 1)
        if do_commit: self.commit()
        if (iddownload > 0) and isinstance(ip_addr,basestring):
            self.store_download_data(iddownload, ip_addr)
        return True

    def do_downloads(self, pkgkeys, ip_addr = None, do_commit = False):
        self.check_connection()
        mydate = self.get_date()
        for pkgkey in pkgkeys:
            iddownload = self.get_iddownload(pkgkey, mydate)
            if iddownload == -1:
                iddownload = self.insert_download(pkgkey, mydate, count = 1)
            else:
                self.update_download(iddownload, pkgkey, mydate, 1)
            if (iddownload > 0) and isinstance(ip_addr,basestring):
                self.store_download_data(iddownload, ip_addr)
        if do_commit: self.commit()
        return True

    def insert_document(self, pkgkey, userid, username, text, title, description, keywords, doc_type = None, do_commit = False):
        self.check_connection()
        idkey = self.handle_pkgkey(pkgkey)
        if doc_type == None: doc_type = self.DOC_TYPES['bbcode_doc']
        iddoc = self.insert_generic_doc(idkey, userid, username, doc_type, text, title, description, keywords)
        if isinstance(iddoc,basestring):
            return False, iddoc
        if do_commit: self.commit()
        return True, iddoc

    def edit_document(self, iddoc, new_document, new_title, new_keywords, do_commit = False):
        self.check_connection()
        if not self.is_iddoc_available(iddoc):
            return False

        new_title = new_title[:self.entropy_docs_title_len]

        self.execute_query('UPDATE entropy_docs SET `ddata` = %s, `title` = %s  WHERE `iddoc` = %s AND `iddoctype` = %s',(
                new_document,
                new_title,
                iddoc,
                self.DOC_TYPES['bbcode_doc'],
            )
        )
        self.remove_keywords(iddoc)
        self.insert_keywords(iddoc, new_keywords)
        if do_commit: self.commit()
        return True, iddoc

    def remove_document(self, iddoc):
        self.check_connection()
        userid = self.get_iddoc_userid(iddoc)
        self.remove_keywords(iddoc)
        self.execute_query('DELETE FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                iddoc,
                self.DOC_TYPES['bbcode_doc'],
            )
        )
        if userid: self.update_user_score(userid)
        return True, iddoc

    def scan_for_viruses(self, filepath):

        if not os.access(filepath,os.R_OK):
            return False,None

        args = [self.VIRUS_CHECK_EXEC]
        args += self.VIRUS_CHECK_ARGS
        args += [filepath]
        f = open("/dev/null","w")
        p = subprocess.Popen(args, stdout = f, stderr = f)
        rc = p.wait()
        f.close()
        if rc == 1:
            return True,None
        return False,None

    def insert_generic_file(self, pkgkey, userid, username, file_path, file_name, doc_type, title, description, keywords):
        self.check_connection()
        file_path = os.path.realpath(file_path)

        # do a virus check?
        virus_found, virus_type = self.scan_for_viruses(file_path)
        if virus_found:
            os.remove(file_path)
            return False, None

        # flood control
        flood_risk = self.insert_flood_control_check(userid)
        if flood_risk: return False, 'flooding detected'

        # validity check
        if doc_type == self.DOC_TYPES['image']:
            valid = False
            if os.path.isfile(file_path) and os.access(file_path,os.R_OK):
                valid = self.entropyTools.is_supported_image_file(file_path)
            if not valid:
                return False, 'not a valid image'

        dest_path = os.path.join(self.STORE_PATH,file_name)

        # create dir if not exists
        dest_dir = os.path.dirname(dest_path)
        if not os.path.isdir(dest_dir):
            try:
                os.makedirs(dest_dir)
            except OSError, e:
                raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (e,))
            if etpConst['entropygid'] != None:
                const_setup_perms(dest_dir,etpConst['entropygid'])

        orig_dest_path = dest_path
        dcount = 0
        while os.path.isfile(dest_path):
            dcount += 1
            dest_path_name = "%s_%s" % (dcount,os.path.basename(orig_dest_path),)
            dest_path = os.path.join(os.path.dirname(orig_dest_path),dest_path_name)

        if os.path.dirname(file_path) != dest_dir:
            shutil.move(file_path,dest_path)
        if etpConst['entropygid'] != None:
            const_setup_file(dest_path, etpConst['entropygid'], 0664)

        title = title[:self.entropy_docs_title_len]
        description = description[:self.entropy_docs_description_len]

        # now store in db
        idkey = self.handle_pkgkey(pkgkey)
        self.execute_query('INSERT INTO entropy_docs VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',(
                None,
                idkey,
                userid,
                username,
                doc_type,
                file_name,
                title,
                description,
                None,
            )
        )
        iddoc = self.lastrowid()
        self.insert_keywords(iddoc, keywords)
        store_url = os.path.basename(dest_path)
        if self.store_url:
            store_url = os.path.join(self.store_url,store_url)
        self.update_user_score(userid)
        return True, (iddoc, store_url)

    def insert_image(self, pkgkey, userid, username, image_path, file_name, title, description, keywords):
        return self.insert_generic_file(pkgkey, userid, username, image_path, file_name, self.DOC_TYPES['image'], title, description, keywords)

    def insert_file(self, pkgkey, userid, username, file_path, file_name, title, description, keywords):
        return self.insert_generic_file(pkgkey, userid, username, file_path, file_name, self.DOC_TYPES['generic_file'], title, description, keywords)

    def delete_image(self, iddoc):
        return self.delete_generic_file(iddoc, self.DOC_TYPES['image'])

    def delete_file(self, iddoc):
        return self.delete_generic_file(iddoc, self.DOC_TYPES['generic_file'])

    def delete_generic_file(self, iddoc, doc_type):
        self.check_connection()

        userid = self.get_iddoc_userid(iddoc)
        self.execute_query('SELECT `ddata` FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                iddoc,
                doc_type,
            )
        )
        data = self.fetchone()
        if data != None:
            mypath = data.get('ddata')
            if not isinstance(mypath,basestring):
                mypath = mypath.tostring()
            if mypath != None:
                mypath = os.path.join(self.STORE_PATH,mypath)
                if os.path.isfile(mypath) and os.access(mypath,os.W_OK):
                    os.remove(mypath)

        self.remove_keywords(iddoc)
        self.execute_query('DELETE FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                iddoc,
                doc_type,
            )
        )
        if userid: self.update_user_score(userid)
        return True, (iddoc, None)

    def insert_youtube_video(self, pkgkey, userid, username, video_path, file_name, title, description, keywords):
        self.check_connection()
        if not self.gdata:
            return False, None

        idkey = self.handle_pkgkey(pkgkey)
        video_path = os.path.realpath(video_path)
        if not (os.access(video_path,os.R_OK) and os.path.isfile(video_path)):
            return False
        virus_found, virus_type = self.scan_for_viruses(video_path)
        if virus_found:
            os.remove(video_path)
            return False, None

        new_video_path = video_path
        if isinstance(file_name,basestring):
            # move file to the new filename
            new_video_path = os.path.join(os.path.dirname(video_path),os.path.basename(file_name)) # force basename
            scount = 0
            while os.path.lexists(new_video_path):
                scount += 1
                bpath = "%s.%s" % (unicode(scount),os.path.basename(file_name),)
                new_video_path = os.path.join(os.path.dirname(video_path),bpath)
            shutil.move(video_path,new_video_path)

        yt_service = self.get_youtube_service()
        if yt_service == None:
            return False, None

        mykeywords = ', '.join([x.strip().strip(',') for x in keywords.split()+["sabayon"] if (x.strip() and x.strip(",") and (len(x.strip()) > 4))])
        gd_keywords = self.gdata.media.Keywords(text = mykeywords)

        mydescription = "%s: %s" % (pkgkey,description,)
        mytitle = "%s: %s" % (self.system_name,title,)
        my_media_group = self.gdata.media.Group(
            title = self.gdata.media.Title(text = mytitle),
            description = self.gdata.media.Description(
                description_type = 'plain',
                text = mydescription
            ),
            keywords = gd_keywords,
            category = self.gdata.media.Category(
                text = 'Tech',
                scheme = 'http://gdata.youtube.com/schemas/2007/categories.cat',
                label = 'Tech'
            ),
            player = None
        )
        video_entry = self.gdata.youtube.YouTubeVideoEntry(media = my_media_group)
        new_entry = yt_service.InsertVideoEntry(video_entry, new_video_path)
        if not isinstance(new_entry,self.gdata.youtube.YouTubeVideoEntry):
            return False, None
        video_url = new_entry.GetSwfUrl()
        video_id = os.path.basename(video_url)

        iddoc = self.insert_generic_doc(idkey, userid, username, self.DOC_TYPES['youtube_video'], video_id, title, description, keywords)
        if isinstance(iddoc,basestring):
            return False, (iddoc, None,)
        return True, (iddoc, video_id,)

    def remove_youtube_video(self, iddoc):
        self.check_connection()
        if not self.gdata:
            return False, None
        if not self.is_iddoc_available(iddoc):
            return False, None
        userid = self.get_iddoc_userid(iddoc)

        yt_service = self.get_youtube_service()
        if yt_service == None:
            return False, None

        def do_remove():
            self.remove_keywords(iddoc)
            self.execute_query('DELETE FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                    iddoc,
                    self.DOC_TYPES['youtube_video'],
                )
            )

        self.execute_query('SELECT `ddata` FROM entropy_docs WHERE `iddoc` = %s AND `iddoctype` = %s',(
                iddoc,
                self.DOC_TYPES['youtube_video'],
            )
        )
        data = self.fetchone()
        if data == None:
            do_remove()
            return False, None
        elif not data.has_key('ddata'):
            do_remove()
            return False, None

        video_id = data.get('ddata')
        try:
            video_entry = yt_service.GetYouTubeVideoEntry(video_id = video_id)
            deleted = yt_service.DeleteVideoEntry(video_entry)
        except:
            deleted = True

        if deleted:
            do_remove()
        if userid: self.update_user_score(userid)
        return deleted, (iddoc, video_id,)

    def get_youtube_service(self):
        self.check_connection()
        if not self.gdata:
            return None
        keywords = ['google_email', 'google_password']
        for keyword in keywords:
            if not self.connection_data.has_key(keyword):
                return None
        # note: your google account must be linked with the YouTube one
        srv = self.YouTubeService.YouTubeService()
        srv.email = self.connection_data['google_email']
        srv.password = self.connection_data['google_password']
        if self.connection_data.has_key('google_developer_key'):
            srv.developer_key = self.connection_data['google_developer_key']
        if self.connection_data.has_key('google_client_id'):
            srv.client_id = self.connection_data['google_client_id']
        srv.source = 'Entropy'
        srv.ProgrammaticLogin()
        return srv

class DistributionUGCCommands(SocketCommandsSkel):

    import dumpTools, entropyTools
    def __init__(self, HostInterface, connection_data, store_path, store_url):

        SocketCommandsSkel.__init__(self, HostInterface, inst_name = "ugc-commands")
        self.connection_data = connection_data.copy()
        self.store_path = store_path
        self.store_url = store_url
        self.DOC_TYPES = etpConst['ugc_doctypes'].copy()
        self.SUPPORTED_DOCFILE_TYPES = [
            self.DOC_TYPES['image'],
            self.DOC_TYPES['generic_file'],
            self.DOC_TYPES['youtube_video'],
        ]
        self.raw_commands = ['ugc:add_comment', 'ugc:edit_comment', 'ugc:register_stream']

        self.valid_commands = {
            'ugc:get_comments':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_comments,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the comments of the provided package key",
                'syntax': "<SESSION_ID> ugc:get_comments app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:get_comments_by_identifiers':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_comments_by_identifiers,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the comments belonging to the provided identifiers",
                'syntax': "<SESSION_ID> ugc:get_comments_by_identifiers <identifier1> <identifier2> <identifier3>",
                'from': unicode(self), # from what class
            },
            'ugc:get_documents_by_identifiers':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_documents_by_identifiers,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the documents belonging to the provided identifiers",
                'syntax': "<SESSION_ID> ugc:get_documents_by_identifiers <identifier1> <identifier2> <identifier3>",
                'from': unicode(self), # from what class
            },
            'ugc:get_vote':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_vote,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the vote of the provided package key",
                'syntax': "<SESSION_ID> ugc:get_vote app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:get_downloads':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_downloads,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the number of downloads of the provided package key",
                'syntax': "<SESSION_ID> ugc:get_downloads app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:get_textdocs':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_textdocs,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the text documents belonging to the provided package key",
                'syntax': "<SESSION_ID> ugc:get_textdocs app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:get_textdocs_by_identifiers':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_textdocs_by_identifiers,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the text documents belonging to the provided identifiers",
                'syntax': "<SESSION_ID> ugc:get_textdocs_by_identifiers <identifier1> <identifier2> <identifier3>",
                'from': unicode(self), # from what class
            },
            'ugc:get_alldocs':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_alldocs,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get the all the documents belonging to the provided package key",
                'syntax': "<SESSION_ID> ugc:get_alldocs app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:get_allvotes':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_allvotes,
                'args': [],
                'as_user': False,
                'desc': "get vote information for every available package key",
                'syntax': "<SESSION_ID> ugc:get_allvotes",
                'from': unicode(self), # from what class
            },
            'ugc:get_alldownloads':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_get_alldownloads,
                'args': [],
                'as_user': False,
                'desc': "get download information for every available package key",
                'syntax': "<SESSION_ID> ugc:get_alldownloads",
                'from': unicode(self), # from what class
            },
            'ugc:do_vote':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_do_vote,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "vote the specified application (from 0 to 5)",
                'syntax': "<SESSION_ID> ugc:do_vote app-foo/foo <0..5>",
                'from': unicode(self), # from what class
            },
            'ugc:do_download':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_do_download,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "inform the system of a downloaded application",
                'syntax': "<SESSION_ID> ugc:do_download app-foo/foo",
                'from': unicode(self), # from what class
            },
            'ugc:do_downloads':    {
                'auth': False,
                'built_in': False,
                'cb': self.docmd_do_downloads,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "inform the system of downloaded applications",
                'syntax': "<SESSION_ID> ugc:do_downloads app-foo/foo1 app-foo/foo2 <...>",
                'from': unicode(self), # from what class
            },
            'ugc:add_comment':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_add_comment,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "insert a comment related to a package key",
                'syntax': "<SESSION_ID> ugc:add_comment app-foo/foo <valid xml formatted data>",
                'from': unicode(self), # from what class
            },
            'ugc:remove_comment':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_comment,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "remove a comment (you need its iddoc and mod/admin privs)",
                'syntax': "<SESSION_ID> ugc:remove_comment <iddoc>",
                'from': unicode(self), # from what class
            },
            'ugc:edit_comment':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_edit_comment,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "edit a comment related to a package key (you need its iddoc, mod/admin privs or being the author)",
                'syntax': "<SESSION_ID> ugc:edit_comment <iddoc> <valid xml formatted data>",
                'from': unicode(self), # from what class
            },
            'ugc:register_stream':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_register_stream,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "register an uploaded file (through stream cmd) to the relative place (image, file, videos)",
                'syntax': "<SESSION_ID> ugc:register_stream app-foo/foo <valid xml formatted data>",
                'from': unicode(self), # from what class
            },
            'ugc:remove_image':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_image,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "remove an image (you need its iddoc and mod/admin privs)",
                'syntax': "<SESSION_ID> ugc:remove_image <iddoc>",
                'from': unicode(self), # from what class
            },
            'ugc:remove_file':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_file,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "remove a file (you need its iddoc and mod/admin privs)",
                'syntax': "<SESSION_ID> ugc:remove_file <iddoc>",
                'from': unicode(self), # from what class
            },
            'ugc:remove_youtube_video':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_youtube_video,
                'args': ["authenticator","myargs"],
                'as_user': False,
                'desc': "remove a youtube video (you need its iddoc and mod/admin privs)",
                'syntax': "<SESSION_ID> ugc:remove_youtube_video <iddoc>",
                'from': unicode(self), # from what class
            },
        }

    def _load_ugc_interface(self):
        return DistributionUGCInterface(self.connection_data, self.store_path, self.store_url)

    def _get_userid(self, authenticator):
        session_data = self.HostInterface.sessions.get(authenticator.session)
        if not session_data:
            return False
        elif not session_data.has_key('auth_uid'):
            return False
        return session_data['auth_uid']

    def _get_username(self, authenticator):
        return authenticator.get_username()

    def _get_session_file(self, authenticator):
        session_data = self.HostInterface.sessions.get(authenticator.session)
        if not session_data:
            return False
        elif not session_data.has_key('stream_path'):
            return False
        elif not session_data['stream_path']:
            return False
        mypath = session_data['stream_path']
        if not (os.path.isfile(mypath) and os.access(mypath,os.R_OK)):
            return False
        return mypath

    def _get_session_ip_address(self, authenticator):
        session_data = self.HostInterface.sessions.get(authenticator.session)
        if not session_data:
            return None
        elif not session_data.has_key('ip_address'):
            return None
        return session_data['ip_address']

    def docmd_register_stream(self, authenticator, myargs):

        if len(myargs) < 3:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        xml_string = ' '.join(myargs[1:])
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('doc_type') \
                and mydict.has_key('title') \
                and mydict.has_key('description') \
                and mydict.has_key('keywords') \
                and mydict.has_key('file_name') ):
            return None,'wrong dict arguments, xml must have 5 items with attr value -> doc_type, title, description, keywords, file_name'
        doc_type = mydict.get('doc_type')
        title = mydict.get('title')
        description = mydict.get('description')
        keywords = mydict.get('keywords')
        file_name = mydict.get('file_name')
        real_filename = mydict.get('real_filename')

        try:
            doc_type = int(doc_type)
        except (ValueError,):
            return None,'wrong arguments (doc_type)'
        if doc_type not in self.SUPPORTED_DOCFILE_TYPES:
            return None,'unsupported doc type (SUPPORTED_DOCFILE_TYPES)'

        if not title: title = 'No title'
        if not description: description = 'No description'
        if not keywords: keywords = ''

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'
        username = self._get_username(authenticator)

        # get file path
        stream_path = self._get_session_file(authenticator)
        if not stream_path:
            return False,'no stream path available'
        orig_stream_path = os.path.dirname(stream_path)
        new_stream_path = orig_stream_path

        scount = -1
        while os.path.lexists(new_stream_path):
            scount += 1
            b_name = os.path.basename(stream_path)
            b_name = "%s.%s" % (scount,b_name,)
            new_stream_path = os.path.join(os.path.dirname(orig_stream_path),b_name)
            if scount > 1000000:
                return False,'while loop interrupted while looking for new_stream_path'
        shutil.move(stream_path,new_stream_path)
        stream_path = new_stream_path

        ugc = self._load_ugc_interface()

        rslt = None, 'invalid doc type'
        if doc_type == self.DOC_TYPES['image']:
            rslt = ugc.insert_image(pkgkey, userid, username, stream_path, file_name, title, description, keywords)
        elif doc_type == self.DOC_TYPES['generic_file']:
            rslt = ugc.insert_file(pkgkey, userid, username, stream_path, file_name, title, description, keywords)
        elif doc_type == self.DOC_TYPES['youtube_video']:
            rslt = ugc.insert_youtube_video(pkgkey, userid, username, stream_path, real_filename, title, description, keywords)
        return rslt

    def docmd_add_comment(self, authenticator, myargs):

        if len(myargs) < 2:
            return None,'wrong arguments'
        pkgkey = myargs[0]
        xml_string = ' '.join(myargs[1:])
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('comment') and mydict.has_key('title') and mydict.has_key('keywords')):
            return None,'wrong dict arguments, xml must have 3 items with attr value -> comment, title, keywords'
        comment = mydict.get('comment')
        title = mydict.get('title')
        keywords = mydict.get('keywords')

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'
        username = self._get_username(authenticator)

        ugc = self._load_ugc_interface()
        status, iddoc = ugc.insert_comment(pkgkey, userid, username, comment, title, keywords)
        if not status:
            t = 'unable to add comment'
            if isinstance(iddoc,basestring):
                t = iddoc
            return False,t
        return iddoc,'ok'

    def docmd_remove_comment(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'
        try:
            iddoc = int(myargs[0])
        except (ValueError,):
            return False,'not a valid iddoc'

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'

        ugc = self._load_ugc_interface()
        iddoc_userid = ugc.get_iddoc_userid(iddoc)
        if iddoc_userid == None:
            return False,'document not available'

        # check if admin/mod or author
        if authenticator.is_user() and (userid != iddoc_userid):
            return False,'permission denied'

        ugc = self._load_ugc_interface()
        status, iddoc = ugc.remove_comment(iddoc)
        if not status:
            return False,'document not removed or not available'

        return iddoc,'ok'

    def docmd_edit_comment(self, authenticator, myargs):

        if len(myargs) < 2:
            return None,'wrong arguments'
        try:
            iddoc = int(myargs[0])
        except (ValueError,):
            return False,'not a valid iddoc'

        xml_string = ' '.join(myargs[1:])
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('comment') and mydict.has_key('title') and mydict.has_key('keywords')):
            return None,'wrong dict arguments, xml must have two item with attr value -> comment, title'
        new_comment = mydict.get('comment')
        new_title = mydict.get('title')
        new_keywords = mydict.get('keywords')

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'

        ugc = self._load_ugc_interface()
        iddoc_userid = ugc.get_iddoc_userid(iddoc)
        if iddoc_userid == None:
            return False,'document not available'

        # check if admin/mod or author
        if authenticator.is_user() and (userid != iddoc_userid):
            return False,'permission denied'

        status, iddoc = ugc.edit_comment(iddoc, new_comment, new_title, new_keywords)
        if not status:
            return False,'document not removed or not available'

        return iddoc,'ok'

    def docmd_remove_image(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'
        try:
            iddoc = int(myargs[0])
        except (ValueError,):
            return False,'not a valid iddoc'

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'

        ugc = self._load_ugc_interface()
        iddoc_userid = ugc.get_iddoc_userid(iddoc)
        if iddoc_userid == None:
            return False,'document not available'

        # check if admin/mod or author
        if authenticator.is_user() and (userid != iddoc_userid):
            return False,'permission denied'

        ugc = self._load_ugc_interface()
        status, iddoc = ugc.delete_image(iddoc)
        if not status:
            return False,'document not removed or not available'

        return iddoc,'ok'

    def docmd_remove_file(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'
        try:
            iddoc = int(myargs[0])
        except (ValueError,):
            return False,'not a valid iddoc'

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'

        ugc = self._load_ugc_interface()
        iddoc_userid = ugc.get_iddoc_userid(iddoc)
        if iddoc_userid == None:
            return False,'document not available'

        # check if admin/mod or author
        if authenticator.is_user() and (userid != iddoc_userid):
            return False,'permission denied'

        ugc = self._load_ugc_interface()
        status, iddoc = ugc.delete_file(iddoc)
        if not status:
            return False,'document not removed or not available'

        return iddoc,'ok'

    def docmd_remove_youtube_video(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'
        try:
            iddoc = int(myargs[0])
        except (ValueError,):
            return False,'not a valid iddoc'

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return False,'no session data available'

        ugc = self._load_ugc_interface()
        iddoc_userid = ugc.get_iddoc_userid(iddoc)
        if iddoc_userid == None:
            return False,'document not available'

        # check if admin/mod or author
        if authenticator.is_user() and (userid != iddoc_userid):
            return False,'permission denied'

        ugc = self._load_ugc_interface()
        status, iddoc = ugc.remove_youtube_video(iddoc)
        if not status:
            return False,'document not removed or not available'

        return iddoc,'ok'

    def docmd_do_vote(self, authenticator, myargs):

        if len(myargs) < 2:
            return None,'wrong arguments'
        pkgkey = myargs[0]
        vote = myargs[1]

        userid = self._get_userid(authenticator)
        if userid == None:
            return False,'no session userid available'
        elif isinstance(userid,bool) and not userid:
            return userid,'no session data available'

        ugc = self._load_ugc_interface()
        voted = ugc.do_vote(pkgkey, userid, vote)
        if not voted:
            return voted,'already voted'
        return voted,'ok'

    def docmd_do_download(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        ip_addr = self._get_session_ip_address(authenticator)
        ugc = self._load_ugc_interface()
        done = ugc.do_download(pkgkey, ip_addr = ip_addr)
        if not done:
            return done,'download not stored'
        return done,'ok'

    def docmd_do_downloads(self, authenticator, myargs):

        if not myargs:
            return None,'wrong arguments'

        ip_addr = self._get_session_ip_address(authenticator)
        ugc = self._load_ugc_interface()
        done = ugc.do_downloads(myargs, ip_addr = ip_addr)
        if not done:
            return done,'download not stored'
        return done,'ok'

    def _get_generic_doctypes(self, pkgkey, doctypes):
        ugc = self._load_ugc_interface()
        metadata = ugc.get_ugc_metadata_doctypes(pkgkey, doctypes)
        if not metadata:
            return None
        return metadata

    def _get_generic_doctypes_by_identifiers(self, identifiers, doctypes):
        ugc = self._load_ugc_interface()
        metadata = ugc.get_ugc_metadata_doctypes_by_identifiers(identifiers, doctypes)
        if not metadata:
            return None
        return metadata

    def _get_generic_documents_by_identifiers(self, identifiers):
        ugc = self._load_ugc_interface()
        metadata = ugc.get_ugc_metadata_by_identifiers(identifiers)
        if not metadata:
            return None
        return metadata

    def docmd_get_comments(self, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        metadata = self._get_generic_doctypes(pkgkey, [self.DOC_TYPES['comments']])
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'

    def docmd_get_comments_by_identifiers(self, myargs):

        if not myargs:
            return None,'wrong arguments'

        identifiers = []
        for myarg in myargs:
            try:
                identifiers.append(int(myarg))
            except ValueError:
                pass

        if not identifiers:
            return None,'wrong arguments'

        metadata = self._get_generic_doctypes_by_identifiers(identifiers, [self.DOC_TYPES['comments']])
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'

    def docmd_get_documents_by_identifiers(self, myargs):

        if not myargs:
            return None,'wrong arguments'

        identifiers = []
        for myarg in myargs:
            try:
                identifiers.append(int(myarg))
            except ValueError:
                pass

        if not identifiers:
            return None,'wrong arguments'

        metadata = self._get_generic_documents_by_identifiers(identifiers)
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'

    def docmd_get_allvotes(self):
        ugc = self._load_ugc_interface()
        metadata = ugc.get_ugc_allvotes()
        if not metadata:
            return None,'no metadata available'
        return metadata,'ok'

    def docmd_get_alldownloads(self):
        ugc = self._load_ugc_interface()
        metadata = ugc.get_ugc_alldownloads()
        if not metadata:
            return None,'no metadata available'
        return metadata,'ok'

    def docmd_get_vote(self, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        ugc = self._load_ugc_interface()
        vote = ugc.get_ugc_vote(pkgkey)
        return vote,'ok'

    def docmd_get_downloads(self, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        ugc = self._load_ugc_interface()
        downloads = ugc.get_ugc_downloads(pkgkey)
        return downloads,'ok'

    def docmd_get_textdocs(self, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        metadata = self._get_generic_doctypes(pkgkey, [self.DOC_TYPES['comments'],self.DOC_TYPES['bbcode_doc']])
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'

    def docmd_get_textdocs_by_identifiers(self, myargs):

        if not myargs:
            return None,'wrong arguments'

        identifiers = []
        for myarg in myargs:
            try:
                identifiers.append(int(myarg))
            except ValueError:
                pass

        if not identifiers:
            return None,'wrong arguments'

        metadata = self._get_generic_doctypes_by_identifiers(identifiers, [self.DOC_TYPES['comments'],self.DOC_TYPES['bbcode_doc']])
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'

    def docmd_get_alldocs(self, myargs):

        if not myargs:
            return None,'wrong arguments'
        pkgkey = myargs[0]

        metadata = self._get_generic_doctypes(pkgkey, [self.DOC_TYPES[x] for x in self.DOC_TYPES])
        if metadata == None:
            return None,'no metadata available'

        return metadata,'ok'


class DistributionAuthInterface:
    """just a reference class, methods must be reimplemented"""

    def __init__(self):
        self.login_data = {}
        self.logged_in = False

    def check_login(self):
        if not self.logged_in:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_("not logged in"),))

    def set_login_data(self, data):
        self.login_data = data.copy()

    def check_login_data(self):
        if not self.login_data:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_("no login data"),))

    def check_logged_in(self):
        if not self.is_logged_in():
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_("not logged in"),))

    def _raise_not_implemented_error(self):
        raise exceptionTools.NotImplementedError('NotImplementedError: %s' % (_('method not implemented'),))

    def check_connection(self):
        pass

    def login(self):
        self.check_connection()
        self.check_login_data()
        self._raise_not_implemented_error()
        self.logged_in = True
        return True

    def logout(self):
        self.check_connection()
        self.check_login_data()
        self._raise_not_implemented_error()
        return True

    def is_developer(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return True

    def is_administrator(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return True

    def is_moderator(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return True

    def is_user(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return True

    def is_user_banned(self, user):
        self.check_connection()
        self._raise_not_implemented_error()
        return False

    def is_in_group(self, group):
        mygroup = group
        del mygroup
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return True

    def get_user_groups(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return {}

    def get_user_group(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return -1

    def get_user_id(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return -1

    def is_logged_in(self):
        return self.logged_in

    def get_permission_data(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return {}

    def get_user_data(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return {}

    def get_username(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self._raise_not_implemented_error()
        return {}


class phpBB3AuthInterface(DistributionAuthInterface,RemoteDbSkelInterface):

    import entropyTools
    def __init__(self):
        DistributionAuthInterface.__init__(self)
        RemoteDbSkelInterface.__init__(self)
        self.itoa64 = './0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
        self.USER_NORMAL = 0
        self.USER_INACTIVE = 1
        self.USER_IGNORE = 2
        self.USER_FOUNDER = 3
        self.REGISTERED_USERS_GROUP = 7895
        self.ADMIN_GROUPS = [7893, 7898]
        self.MODERATOR_GROUPS = [484]
        self.DEVELOPER_GROUPS = [7900]
        self.USERNAME_LENGTH_RANGE = range(3,21)
        self.PASSWORD_LENGTH_RANGE = range(6,31)
        self.PRIVMSGS_NO_BOX = -3
        self.NOTIFY_EMAIL = 0
        self.FAKE_USERNAME = 'already_authed'
        self.USER_AGENT = "Entropy/%s (compatible; %s; %s: %s %s %s)" % (
                                        etpConst['entropyversion'],
                                        "Entropy",
                                        "UGC",
                                        os.uname()[0],
                                        os.uname()[4],
                                        os.uname()[2],
        )
        self.TABLE_PREFIX = 'phpbb_'
        self.do_update_session_table = True

    def validate_username_regex(self, username):
        allow_name_chars = self._get_config_value("allow_name_chars")
        if allow_name_chars == "USERNAME_CHARS_ANY":
            regex = '.+'
        elif allow_name_chars == "USERNAME_ALPHA_ONLY":
            regex = '[A-Za-z0-9]+'
        elif allow_name_chars == "USERNAME_ALPHA_SPACERS":
            regex = '[A-Za-z0-9-[\]_+ ]+'
        elif allow_name_chars == "USERNAME_LETTER_NUM":
            regex = '[a-zA-Z0-9]+'
        elif allow_name_chars == "USERNAME_LETTER_NUM_SPACERS":
            regex = '[-\]_+ [a-zA-Z0-9]+'
        else: # USERNAME_ASCII
            regex = '[\x01-\x7F]+'
        regex = "^%s$" % (regex,)
        import re
        myreg = re.compile(regex)
        if myreg.match(username):
            del myreg
            return True
        return False

    def does_username_exist(self, username, username_clean):
        self.check_connection()
        self.cursor.execute('SELECT user_id FROM '+self.TABLE_PREFIX+'users WHERE `username_clean` = %s OR LOWER(`username`) = %s', (username_clean,username.lower(),))
        data = self.cursor.fetchone()
        if not data: return False
        if not isinstance(data,dict): return False
        if not data.has_key('user_id'): return False
        return True

    def does_email_exist(self, email):
        self.check_connection()
        self.cursor.execute('SELECT user_id FROM '+self.TABLE_PREFIX+'users WHERE `user_email` = %s', (email,))
        data = self.cursor.fetchone()
        if not data: return False
        if not isinstance(data,dict): return False
        if not data.has_key('user_id'): return False
        return True

    def is_username_allowed(self, username):
        self.check_connection()
        self.cursor.execute('SELECT disallow_id FROM '+self.TABLE_PREFIX+'disallow WHERE `disallow_username` = %s', (username,))
        data = self.cursor.fetchone()
        if not data: return True
        if not isinstance(data,dict): return True
        if not data.has_key('disallow_id'): return True
        return False

    def validate_username_string(self, username, username_clean):

        try:
            x = unicode(username.encode('utf-8'),'raw_unicode_escape')
            del x
        except (UnicodeDecodeError,UnicodeEncodeError,):
            return False,'Invalid username'
        if ("&quot;" in username) or ("'" in username) or ('"' in username):
            return False,'Invalid username'

        try:
            valid = self.validate_username_regex(username)
        except:
            return False,'Username contains bad characters'
        if not valid:
            return False,'Invalid username'

        exists = self.does_username_exist(username, username_clean)
        if exists: return False,'Username already taken'

        allowed = self.is_username_allowed(username)
        if not allowed: return False,'Username not allowed'

        return True,'All fine'

    def _generate_email_hash(self, email):
        import binascii
        return str(binascii.crc32(email.lower())) + str(len(email))

    def activate_user(self, user_id):
        self.check_connection()
        self.cursor.execute('UPDATE '+self.TABLE_PREFIX+'users SET user_type = %s WHERE `user_id` = %s', (self.USER_NORMAL,user_id,))
        return True, user_id

    def generate_username_clean(self, username):
        import re
        username_clean = username.lower()
        username_clean = re.sub(r'(?:[\x00-\x1F\x7F]+|(?:\xC2[\x80-\x9F])+)', '', username_clean)
        username_clean = re.sub(r' {2,}',' ',username_clean)
        username_clean = username_clean.strip()
        return username_clean

    def register_user(self, username, password, email, activate = False):

        if len(username) not in self.USERNAME_LENGTH_RANGE:
            return False,'Username not in range'
        if len(password) not in self.PASSWORD_LENGTH_RANGE:
            return False,'Password not in range'
        valid = self.entropyTools.is_valid_email(email)
        if not valid:
            return False,'Invalid email'

        # create the clean one
        username_clean = self.generate_username_clean(username)

        # check username validity
        status, err_msg = self.validate_username_string(username, username_clean)
        if not status: return False,err_msg

        # check email
        exists = self.does_email_exist(email)
        if exists: return False,'Email already in use'

        # now cross fingers
        user_id = self.__register(username, username_clean, password, email, activate)

        return True, user_id


    def __register(self, username, username_clean, password, email, activate):

        email_hash = self._generate_email_hash(email)
        password_hash = self._get_password_hash(password.encode('utf-8'))
        time_now = int(time.time())

        user_type = self.USER_INACTIVE
        if activate: user_type = self.USER_NORMAL

        registration_data = {
            'username': username,
            'username_clean': username_clean,
            'user_password': password_hash,
            'user_pass_convert': 0,
            'user_email': email.lower(),
            'user_email_hash': email_hash,
            'group_id': self.REGISTERED_USERS_GROUP,
            'user_type': user_type,
            'user_permissions': '',
            'user_timezone': self._get_config_value('board_timezone'),
            'user_dateformat': self._get_config_value('default_dateformat'),
            'user_lang': self._get_config_value('default_lang'),
            'user_style': self._get_config_value('default_style'),
            'user_actkey': '',
            'user_ip': '',
            'user_regdate': time_now,
            'user_passchg': time_now,
            'user_options': 895, # ? don't ask me
            'user_inactive_reason': 0,
            'user_inactive_time': 0,
            'user_lastmark': time_now,
            'user_lastvisit': 0,
            'user_lastpost_time': 0,
            'user_lastpage': '',
            'user_posts': 0,
            'user_dst': self._get_config_value('board_dst'),
            'user_colour': '',
            'user_occ': '',
            'user_interests': '',
            'user_avatar': '',
            'user_avatar_type': 0,
            'user_avatar_width': 0,
            'user_avatar_height': 0,
            'user_new_privmsg': 0,
            'user_unread_privmsg': 0,
            'user_last_privmsg': 0,
            'user_message_rules': 0,
            'user_full_folder': self.PRIVMSGS_NO_BOX,
            'user_emailtime': 0,
            'user_notify': 0,
            'user_notify_pm': 1,
            'user_notify_type': self.NOTIFY_EMAIL,
            'user_allow_pm': 1,
            'user_allow_viewonline': 1,
            'user_allow_viewemail': 1,
            'user_allow_massemail': 1,
            'user_sig': '',
            'user_sig_bbcode_uid': '',
            'user_sig_bbcode_bitfield': '',
            'user_form_salt': self._get_unique_id(),
        }

        sql = self._generate_sql('insert', self.TABLE_PREFIX+'users', registration_data)
        self.cursor.execute(sql)
        user_id = self.cursor.lastrowid

        # now insert into the default group
        group_data = {
            'user_id': user_id,
            'group_id': self.REGISTERED_USERS_GROUP,
            'user_pending': 0,
        }
        sql = self._generate_sql('insert', self.TABLE_PREFIX+'user_group', group_data)
        self.cursor.execute(sql)

        # set some misc config shit
        self._set_config_value('newest_user_id',user_id)
        self._set_config_value('newest_username',username)
        self._set_config_value('num_users',int(self._get_config_value('num_users'))+1)
        self.cursor.execute('SELECT group_colour FROM '+self.TABLE_PREFIX+'groups WHERE group_id = %s', (group_data['group_id'],))
        data = self.cursor.fetchone()
        gcolor = None
        if isinstance(data,dict):
            if data.has_key('group_colour'):
                gcolor = data['group_colour']
        if gcolor: self._set_config_value('newest_user_colour',gcolor)

        return user_id


    def login(self):
        self.check_connection()
        self.check_login_data()

        if not self.login_data.has_key('username'):
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('no username specified'),))
        elif not self.login_data.has_key('password'):
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('no password specified'),))

        if not self.login_data['password']:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('empty password'),))
        elif not self.login_data['username']:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('empty username'),))

        self.cursor.execute('SELECT * FROM '+self.TABLE_PREFIX+'users WHERE username = %s', (self.login_data['username'],))
        data = self.cursor.fetchone()
        if not data:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('user not found'),))

        if data['user_pass_convert']:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (
                    _('you need to login on the website to update your password format'),
                )
            )

        valid = self._phpbb3_check_hash(self.login_data['password'], data['user_password'])
        if not valid:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('wrong password'),))

        user_type = data['user_type']
        if (user_type == self.USER_INACTIVE) or (user_type == self.USER_IGNORE):
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('user inactive'),))

        banned = self.is_user_banned(data['user_id'])
        if banned:
            raise exceptionTools.PermissionDenied('PermissionDenied: %s' % (_('user banned'),))

        self.login_data.update(data)
        self.logged_in = True
        return self.logged_in

    def disconnect(self):
        if self.is_logged_in():
            self.logout()
        RemoteDbSkelInterface.disconnect(self)

    def logout(self):
        self.check_connection()
        self.check_login_data()
        self.logged_in = False
        self.login_data.clear()
        return True

    def get_user_data(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT * FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        return self.cursor.fetchone()

    def get_user_birthday(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT user_birthday FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        bday = self.cursor.fetchone()
        if not bday:
            return None
        elif not bday.has_key('user_birthday'):
            return None
        return bday['user_birthday']

    def get_username(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT username_clean FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        data = self.cursor.fetchone()
        if not data:
            return ''
        elif not data.has_key('username_clean'):
            return ''
        return data['username_clean']

    def is_developer(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        # search into phpbb_groups
        groups = self.get_user_groups()
        for group in groups:
            if group in self.DEVELOPER_GROUPS:
                return True

        return False

    def is_administrator(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT user_type FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        data = self.cursor.fetchone()
        if data:
            if data['user_type'] == self.USER_FOUNDER:
                return True

        # search into phpbb_groups
        groups = self.get_user_groups()
        for group in groups:
            if group in self.ADMIN_GROUPS:
                return True

        return False

    def is_moderator(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        # search into phpbb_groups
        groups = self.get_user_groups()
        for group in groups:
            if group in self.MODERATOR_GROUPS:
                return True

        return False

    def is_user(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        if self.is_moderator():
            return False
        elif self.is_administrator():
            return False
        elif self.is_developer():
            return False

        self.cursor.execute('SELECT user_type,user_id FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        data = self.cursor.fetchone()
        if not data:
            return False
        if self.is_user_banned(data['user_id']):
            return False
        elif data['user_type'] in [self.USER_NORMAL]:
            return True

        return False

    # user == user_id
    def is_user_banned(self, user):
        self.check_connection()
        self.cursor.execute('SELECT ban_userid FROM '+self.TABLE_PREFIX+'banlist WHERE ban_userid = %s', (user,))
        data = self.cursor.fetchone()
        if data:
            return True
        return False

    def is_in_group(self, group):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        groups = self.get_user_groups()
        if isinstance(group,int):
            if group in groups:
                return True
        elif isinstance(group,basestring):
            self.cursor.execute('SELECT group_id FROM '+self.TABLE_PREFIX+'groups WHERE group_name = %s', (group,))
            data = self.cursor.fetchone()
            if not data:
                return False
            elif data['group_id'] in groups:
                return True

        return False

    def get_user_groups(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT '+self.TABLE_PREFIX+'user_group.group_id,'+self.TABLE_PREFIX+'groups.group_name FROM '+self.TABLE_PREFIX+'user_group,'+self.TABLE_PREFIX+'users,'+self.TABLE_PREFIX+'groups WHERE '+self.TABLE_PREFIX+'users.user_id = %s and '+self.TABLE_PREFIX+'users.user_id = '+self.TABLE_PREFIX+'user_group.user_id and '+self.TABLE_PREFIX+'user_group.group_id = '+self.TABLE_PREFIX+'groups.group_id', (self.login_data['user_id'],))
        data = self.cursor.fetchall()
        mydata = {}
        for mydict in data:
            mydata[mydict['group_id']] = mydict['group_name']

        return mydata

    def get_user_group(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        self.cursor.execute('SELECT group_id FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        data = self.cursor.fetchone()
        if data:
            if data.has_key('group_id'):
                return data['group_id']

        return -1

    def get_user_id(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        return self.login_data['user_id']

    def get_permission_data(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self.cursor.execute('SELECT user_permissions FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        return self.cursor.fetchone()

    def update_email(self, email):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        email_hash = self._generate_email_hash(email)
        mydata = {
            'user_email_hash': email_hash,
            'user_email': email.lower(),
        }

        try:
            sql = self._generate_sql("update",self.TABLE_PREFIX+'users', mydata, 'user_id = %s' % (self.login_data['user_id'],))
            self.cursor.execute(sql)
            return True
        except Exception:
            return False

    def update_password_hash(self, password_hash):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        mydata = {
            'user_password': password_hash,
        }

        try:
            sql = self._generate_sql("update",self.TABLE_PREFIX+'users', mydata, 'user_id = %s' % (self.login_data['user_id'],))
            self.cursor.execute(sql)
            return True
        except Exception:
            return False

    def get_email(self):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()
        self.cursor.execute('SELECT user_email FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (self.login_data['user_id'],))
        data = self.cursor.fetchone()
        if not data:
            return ''
        elif not data.has_key('user_email'):
            return ''
        return data['user_email']

    def update_user_id_profile(self, profile_data):
        self.check_connection()
        self.check_login_data()
        self.check_logged_in()

        # filter valid params
        valid_params = [
            "user_icq","user_yim","user_msnm",
            "user_jabber","user_website","user_from",
            "user_interests","user_occ","user_birthday",
            "user_sig"
        ]

        my_params = {}
        for param in valid_params:
            d = profile_data.get(param)
            if d == None: continue
            my_params[param] = d

        if not my_params:
            return False,'no parameters'


        # validate parameters
        b_day = my_params.get('user_birthday')
        if isinstance(b_day,basestring):
            import re
            myre = re.compile("(0[1-9]|[12][0-9]|3[01])[-](0[1-9]|1[012])[-](19|20)\d\d")
            if not myre.match(b_day):
                del my_params['user_birthday']

        try:
            sql = self._generate_sql("update",self.TABLE_PREFIX+'users', my_params, 'user_id = %s' % (self.login_data['user_id'],))
            self.cursor.execute(sql)
            return True, None
        except Exception, e:
            return False, unicode(e)


    def _set_config_value(self, config_name, data):
        self.cursor.execute('UPDATE '+self.TABLE_PREFIX+'config SET config_value = %s WHERE config_name = %s',(data,config_name,))

    def _get_config_value(self, config_name):
        self.check_connection()
        self.cursor.execute('SELECT config_value FROM '+self.TABLE_PREFIX+'config WHERE config_name = %s',(config_name,))
        myconfig = self.cursor.fetchone()
        if isinstance(myconfig,dict):
            if myconfig.has_key('config_value'):
                return myconfig['config_value']
        return None

    def _update_session_table(self, user_id, ip_address):
        self.check_connection()
        time_now = int(time.time())
        autologin = self._get_config_value("allow_autologin")
        self.cursor.execute('SELECT user_allow_viewonline FROM '+self.TABLE_PREFIX+'users WHERE user_id = %s', (user_id,))
        myuserprefs = self.cursor.fetchone()
        session_admin = 0
        session_data = {
            'session_id': None,
            'session_user_id': user_id,
            'session_last_visit': time_now,
            'session_start': time_now,
            'session_time': time_now,
            'session_ip': ip_address,
            'session_browser': self.USER_AGENT,
            'session_forwarded_for': '',
            'session_page': 'index.php',
            'session_viewonline': myuserprefs['user_allow_viewonline'],
            'session_autologin': autologin,
            'session_admin': session_admin,
            'session_forum_id': 0,
        }
        import hashlib
        m = hashlib.md5()
        m.update(str(user_id)+str(time_now)+str(self.USER_AGENT)+str(ip_address)+str(autologin)+str(myuserprefs['user_allow_viewonline']))
        session_data['session_id'] = m.hexdigest()

        self.cursor.execute('SELECT * FROM '+self.TABLE_PREFIX+'sessions WHERE session_user_id = %s', (user_id,))
        mydata = self.cursor.fetchone()
        do_update = False
        if mydata:
            do_update = True
            # update
            session_data['session_id'] = mydata['session_id']
            session_data['session_viewonline'] = mydata['session_viewonline']
            session_data['session_autologin'] = mydata['session_autologin']
            session_data['session_forwarded_for'] = mydata['session_forwarded_for']
            session_data['session_forum_id'] = mydata['session_forum_id']
            session_data['session_page'] = mydata['session_page']
            session_data['session_browser'] = mydata['session_browser']
            session_data['session_admin'] = mydata['session_admin']

        if do_update:
            where = "session_id = '%s'" % (session_data['session_id'],)
            del session_data['session_id']
            sql = self._generate_sql('update', self.TABLE_PREFIX+'sessions', session_data, where)
        else:
            sql = self._generate_sql('insert', self.TABLE_PREFIX+'sessions', session_data)
        if sql:
            self.cursor.execute(sql)
            self.dbconn.commit()


    def _is_ip_banned(self, ip):
        self.check_connection()
        self.cursor.execute('SELECT ban_ip FROM '+self.TABLE_PREFIX+'banlist WHERE ban_ip = %s', (ip,))
        data = self.cursor.fetchone()
        if data:
            return True
        return False

    def _get_unique_id(self):
        import hashlib
        m = hashlib.md5()
        rnd = str(abs(hash(os.urandom(20))))
        m.update(rnd)
        x = m.hexdigest()[:-16]
        del m
        return x

    def _get_random_number(self):
        myrand = 0
        while (myrand < 100000) or (myrand > 999999):
            myrand = hash(os.urandom(50))%999999
        return myrand

    def _get_password_hash(self, password):

        #random_state = self._get_unique_id()
        myrandom = str(self._get_random_number())

        myhash = self._hash_crypt_private(password, self._hash_gensalt_private(myrandom))

        if len(myhash) == 34:
            return myhash

        import hashlib
        m = hashlib.md5()
        m.update(myhash)
        return m.hexdigest()


    def _hash_gensalt_private(self, myinput, iteration_count_log2 = 6):

        if (iteration_count_log2 < 4) or (iteration_count_log2 > 31):
            iteration_count_log2 = 8

        myoutput = '$H$'
        myoutput += self.itoa64[min(iteration_count_log2 + 5,30)]
        myoutput += self._hash_encode64(myinput, 6)

        return myoutput

    def _hash_crypt_private(self, password, setting):

        myoutput = '*'
        # Check for correct hash
        if setting[:3] != '$H$':
            return myoutput

        count_log2 = self.itoa64.find(setting[3])
        if count_log2 == -1: count_log2 = 0

        if (count_log2 < 7) or (count_log2 > 30):
            return myoutput

        count = 1 << count_log2
        salt = setting[4:12]

        if len(salt) != 8:
            return myoutput

        import hashlib
        m = hashlib.md5()
        m.update(salt+password)
        myhash = m.digest()
        while count:
            m = hashlib.md5()
            m.update(myhash+password)
            myhash = m.digest()
            count -= 1

        myoutput = setting[:12]
        myoutput += self._hash_encode64(myhash, 16)

        return myoutput

    def _hash_encode64(self, myinput, count):

        output = ''
        i = 0
        while i < count:

            value = ord(myinput[i])
            i += 1
            output += self.itoa64[value & 0x3f]
            if i < count:
                value |= ord(myinput[i]) << 8

            output += self.itoa64[(value >> 6) & 0x3f]

            if i >= count:
                break
            i += 1

            if i < count:
                value |= ord(myinput[i]) << 16

            output += self.itoa64[(value >> 12) & 0x3f]

            if (i >= count):
                break
            i += 1

            output += self.itoa64[(value >> 18) & 0x3f]

        return output

    def _phpbb3_check_hash(self, password, myhash):

        if len(myhash) == 34:
            return self._hash_crypt_private(password, myhash) == myhash

        import hashlib
        m = hashlib.md5()
        m.update(password)
        rhash = m.hexdigest()
        return rhash == myhash

# Authenticator that can be used by SocketHostInterface based instances
class phpbb3Authenticator(phpBB3AuthInterface,SocketAuthenticatorSkel):
    import entropyTools

    def __init__(self, HostInterface, *args, **kwargs):
        SocketAuthenticatorSkel.__init__(self, HostInterface)
        phpBB3AuthInterface.__init__(self)
        self.set_connection_data(kwargs)
        self.connect()

    def set_session(self, session):
        self.session = session
        session_data = self.HostInterface.sessions.get(self.session)
        if not session_data:
            return
        auth_id = session_data['auth_uid']
        if auth_id:
            self.logged_in = True
            # fill login_data with fake information
            self.login_data = {'username': self.FAKE_USERNAME, 'password': 'look elsewhere, this is not a password', 'user_id': auth_id}
            ip_address = session_data.get('ip_address')
            if ip_address and self.do_update_session_table:
                self._update_session_table(auth_id, ip_address)

    def docmd_login(self, arguments):

        # filter n00bs
        if not arguments or (len(arguments) != 2):
            return False,None,None,'wrong arguments'

        ip_address = None
        session_data = self.HostInterface.sessions.get(self.session)
        if session_data:
            ip_address = session_data.get('ip_address')
        user = arguments[0]
        password = arguments[1]

        if ip_address:
            if self._is_ip_banned(ip_address):
                return False,user,None,"banned IP"

        login_data = {'username': user, 'password': password}
        self.set_login_data(login_data)
        rc = False
        try:
            rc = self.login()
        except exceptionTools.PermissionDenied, e:
            return rc,user,None,e.value

        if rc:
            uid = self.get_user_id()
            is_admin = self.is_administrator()
            is_dev = self.is_developer()
            is_mod = self.is_moderator()
            is_user = self.is_user()
            self.HostInterface.sessions[self.session]['admin'] = is_admin
            self.HostInterface.sessions[self.session]['developer'] = is_dev
            self.HostInterface.sessions[self.session]['moderator'] = is_mod
            self.HostInterface.sessions[self.session]['user'] = is_user
            if ip_address and uid and self.do_update_session_table:
                self._update_session_table(uid, ip_address)
            return True,user,uid,"ok"
        return rc,user,None,"login failed"

    # if we get here it means we are logged in
    def docmd_userdata(self):
        data = self.get_user_data()
        return True, data, 'ok'

    def docmd_logout(self, myargs):

        # filter n00bs
        if (len(myargs) < 1) or (len(myargs) > 1):
            return False,None,'wrong arguments'

        user = myargs[0]
        # filter n00bs
        if not user or not isinstance(user,basestring):
            return False,None,"wrong user"

        if not self.is_logged_in():
            return False,user,"already logged out"

        return True,user,"ok"

    def set_exc_permissions(self, *args, **kwargs):
        pass

    def hide_login_data(self, args):
        myargs = args[:]
        myargs[-1] = 'hidden'
        return myargs

    def terminate_instance(self):
        self.disconnect()

# commands that can be used by SocketHostInterface based instances, with phpbb3Authenticator
class phpbb3Commands(SocketCommandsSkel):

    import dumpTools
    import entropyTools

    def __init__(self, HostInterface):

        SocketCommandsSkel.__init__(self, HostInterface, inst_name = "phpbb3-commands")

        self.valid_commands = {
            'is_user':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_is_user,
                'args': ["authenticator"],
                'as_user': False,
                'desc': "returns whether the username linked with the session belongs to a simple user",
                'syntax': "<SESSION_ID> is_user",
                'from': unicode(self), # from what class
            },
            'is_developer':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_is_developer,
                'args': ["authenticator"],
                'as_user': False,
                'desc': "returns whether the username linked with the session belongs to a developer",
                'syntax': "<SESSION_ID> is_developer",
                'from': unicode(self), # from what class
            },
            'is_moderator':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_is_moderator,
                'args': ["authenticator"],
                'as_user': False,
                'desc': "returns whether the username linked with the session belongs to a moderator",
                'syntax': "<SESSION_ID> is_moderator",
                'from': unicode(self), # from what class
            },
            'is_administrator':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_is_administrator,
                'args': ["authenticator"],
                'as_user': False,
                'desc': "returns whether the username linked with the session belongs to an administrator",
                'syntax': "<SESSION_ID> is_administrator",
                'from': unicode(self), # from what class
            },
        }

    def docmd_is_user(self, authenticator):
        return authenticator.is_user(),'ok'

    def docmd_is_developer(self, authenticator):
        return authenticator.is_developer(),'ok'

    def docmd_is_administrator(self, authenticator):
        return authenticator.is_administrator(),'ok'

    def docmd_is_moderator(self, authenticator):
        return authenticator.is_moderator(),'ok'

class RepositorySocketServerInterface(SocketHostInterface):

    class RepositoryCommands(SocketCommandsSkel):

        import dumpTools
        import entropyTools

        def __init__(self, HostInterface):

            SocketCommandsSkel.__init__(self, HostInterface, inst_name = "repository_server")

            self.valid_commands = {
                'repository_server:dbdiff':    {
                    'auth': False,
                    'built_in': False,
                    'cb': self.docmd_dbdiff,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "returns idpackage differences against the latest available repository",
                    'syntax': "<SESSION_ID> repository_server:dbdiff <repository> <arch> <product> <branch> [idpackages]",
                    'from': unicode(self), # from what class
                },
                'repository_server:pkginfo':    {
                    'auth': False,
                    'built_in': False,
                    'cb': self.docmd_pkginfo,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "returns idpackage differences against the latest available repository",
                    'syntax': "<SESSION_ID> repository_server:pkginfo <content fmt True/False> <repository> <arch> <product> <branch> <idpackage>",
                    'from': unicode(self), # from what class
                },
                'repository_server:treeupdates':    {
                    'auth': False,
                    'built_in': False,
                    'cb': self.docmd_treeupdates,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "returns repository treeupdates metadata",
                    'syntax': "<SESSION_ID> repository_server:treeupdates <repository> <arch> <product> <branch>",
                    'from': unicode(self), # from what class
                },
                'repository_server:get_package_sets': {
                    'auth': False,
                    'built_in': False,
                    'cb': self.docmd_package_sets,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "returns repository package sets metadata",
                    'syntax': "<SESSION_ID> repository_server:get_package_sets <repository> <arch> <product> <branch>",
                    'from': unicode(self), # from what class
                }
            }

        def trash_old_databases(self):
            for db in self.HostInterface.syscache['db_trashed']:
                db.closeDB()
            self.HostInterface.syscache['db_trashed'].clear()

        def docmd_dbdiff(self, myargs):

            self.trash_old_databases()

            if len(myargs) < 5:
                return None
            repository = myargs[0]
            arch = myargs[1]
            product = myargs[2]
            try:
                branch = str(myargs[3])
            except (UnicodeEncodeError,UnicodeDecodeError,):
                return None
            foreign_idpackages = myargs[4:]
            x = (repository,arch,product,branch,)

            valid = self.HostInterface.is_repository_available(x)
            if not valid:
                return valid

            dbpath = self.get_database_path(repository, arch, product, branch)
            dbconn = self.HostInterface.open_db(dbpath, docache = False)
            mychecksum = dbconn.database_checksum(do_order = True, strict = False, strings = True)
            myids = dbconn.listAllIdpackages()
            dbconn.closeDB()
            foreign_idpackages = set(foreign_idpackages)

            removed_ids = foreign_idpackages - myids
            added_ids = myids - foreign_idpackages

            return {'removed': removed_ids, 'added': added_ids, 'checksum': mychecksum}

        def docmd_package_sets(self, myargs):

            self.trash_old_databases()

            if len(myargs) < 4:
                return None
            repository = myargs[0]
            arch = myargs[1]
            product = myargs[2]
            try:
                branch = str(myargs[3])
            except (UnicodeEncodeError,UnicodeDecodeError,):
                return None

            x = (repository,arch,product,branch,)
            valid = self.HostInterface.is_repository_available(x)
            if not valid:
                return valid

            cached = self.HostInterface.get_dcache((repository, arch, product, branch, 'docmd_package_sets'), repository)
            if cached != None:
                return cached

            dbpath = self.get_database_path(repository, arch, product, branch)
            dbconn = self.HostInterface.open_db(dbpath, docache = False)

            # get data
            data = dbconn.retrievePackageSets()

            self.HostInterface.set_dcache((repository, arch, product, branch, 'docmd_package_sets'), data, repository)
            dbconn.closeDB()

            return data


        def docmd_treeupdates(self, myargs):

            self.trash_old_databases()

            if len(myargs) < 4:
                return None
            repository = myargs[0]
            arch = myargs[1]
            product = myargs[2]
            try:
                branch = str(myargs[3])
            except (UnicodeEncodeError,UnicodeDecodeError,):
                return None

            x = (repository,arch,product,branch,)
            valid = self.HostInterface.is_repository_available(x)
            if not valid:
                return valid

            cached = self.HostInterface.get_dcache((repository, arch, product, branch, 'docmd_treeupdates'), repository)
            if cached != None:
                return cached

            dbpath = self.get_database_path(repository, arch, product, branch)
            dbconn = self.HostInterface.open_db(dbpath, docache = False)

            # get data
            data = {}
            data['actions'] = dbconn.listAllTreeUpdatesActions()
            data['digest'] = dbconn.retrieveRepositoryUpdatesDigest(repository)

            self.HostInterface.set_dcache((repository, arch, product, branch, 'docmd_treeupdates'), data, repository)
            dbconn.closeDB()

            return data


        def docmd_pkginfo(self, myargs):

            self.trash_old_databases()

            if len(myargs) < 6:
                return None
            format_content_for_insert = myargs[0]
            if type(format_content_for_insert) is not bool:
                format_content_for_insert = False
            repository = myargs[1]
            arch = myargs[2]
            product = myargs[3]
            try:
                branch = str(myargs[4])
            except (UnicodeEncodeError,UnicodeDecodeError,):
                return None
            zidpackages = myargs[5:]
            idpackages = []
            for idpackage in zidpackages:
                if type(idpackage) is int:
                    idpackages.append(idpackage)
            if not idpackages:
                return None
            idpackages = tuple(sorted(idpackages))
            x = (repository,arch,product,branch,)

            valid = self.HostInterface.is_repository_available(x)
            if not valid:
                return valid

            cached = self.HostInterface.get_dcache(
                (repository, arch, product, branch, idpackages, 'docmd_pkginfo'),
                repository
            )
            if cached != None:
                return cached

            dbpath = self.get_database_path(repository, arch, product, branch)
            dbconn = self.HostInterface.open_db(dbpath, docache = False)

            result = {}
            for idpackage in idpackages:
                try:
                    mydata = dbconn.getPackageData(
                        idpackage,
                        content_insert_formatted = format_content_for_insert,
                        trigger_unicode = True
                    )
                except:
                    self.entropyTools.printTraceback()
                    self.entropyTools.printTraceback(f = self.HostInterface.socketLog)
                    dbconn.closeDB()
                    return None
                result[idpackage] = mydata.copy()

            self.HostInterface.set_dcache(
                (repository, arch, product, branch, idpackages, 'docmd_pkginfo'),
                result,
                repository
            )
            dbconn.closeDB()
            return result

        def get_database_path(self, repository, arch, product, branch):
            repoitems = (repository,arch,product,branch,)
            mydbroot = self.HostInterface.repositories[repoitems]['dbpath']
            dbpath = os.path.join(mydbroot,etpConst['etpdatabasefile'])
            return dbpath

    class ServiceInterface(TextInterface):
        def __init__(self, *args, **kwargs):
            pass

    import entropyTools, dumpTools
    def __init__(self, repositories, do_ssl = False, stdout_logging = True, **kwargs):
        self.Entropy = EquoInterface(noclientdb = 2)
        self.do_ssl = do_ssl
        self.LockScanner = None
        self.syscache = {
            'db': {},
            'db_trashed': set(),
            'dbs_not_available': set(),
        }
        etpConst['socket_service']['max_connections'] = 5000
        etpConst['socketloglevel'] = 1
        if not kwargs.has_key('external_cmd_classes'):
            kwargs['external_cmd_classes'] = []
        kwargs['external_cmd_classes'].insert(0,self.RepositoryCommands)
        SocketHostInterface.__init__(
            self,
            self.ServiceInterface,
            noclientdb = 2,
            sock_output = self.Entropy,
            ssl = do_ssl,
            **kwargs
        )
        self.stdout_logging = stdout_logging
        self.repositories = repositories
        self.expand_repositories()
        # start timed lock file scanning
        self.start_repository_lock_scanner()

    def killall(self):
        SocketHostInterface.killall(self)
        if self.LockScanner != None:
            self.LockScanner.kill()

    def start_repository_lock_scanner(self):
        self.LockScanner = self.entropyTools.TimeScheduled( self.lock_scan, 0.5 )
        self.LockScanner.setName("Lock_Scanner::"+str(abs(hash(os.urandom(20)))))
        self.LockScanner.start()

    def set_repository_db_availability(self, repo_tuple):
        self.repositories[repo_tuple]['enabled'] = False
        mydbpath = os.path.join(self.repositories[repo_tuple]['dbpath'],etpConst['etpdatabasefile'])
        if os.path.isfile(mydbpath) and os.access(mydbpath, os.W_OK):
            self.syscache['dbs_not_available'].discard(repo_tuple)
            self.repositories[repo_tuple]['enabled'] = True

    def is_repository_available(self, repo_tuple):

        if repo_tuple not in self.repositories:
            return None
        # is repository being updated
        if self.repositories[repo_tuple]['locked']:
            return False
        # repository database does not exist
        if not self.repositories[repo_tuple]['enabled']:
            return False

        return True

    def lock_scan(self):
        do_clear = set()
        for repository,arch,product,branch in self.repositories:
            x = (repository,arch,product,branch,)
            self.set_repository_db_availability(x)
            if not self.repositories[x]['enabled']:
                if x in self.syscache['dbs_not_available']:
                    continue
                self.syscache['dbs_not_available'].add(x)
                mytxt = blue("%s.") % (_("database does not exist. Locking services for it"),)
                self.updateProgress(
                    "[%s] %s" % (
                            brown(str(x)),
                            mytxt,
                    ),
                    importance = 1,
                    type = "info"
                )
                do_clear.add(repository)
                continue
            if os.path.isfile(self.repositories[x]['download_lock']) and \
                not self.repositories[x]['locked']:
                    self.repositories[x]['locked'] = True
                    mydbpath = os.path.join(self.repositories[x]['dbpath'],etpConst['etpdatabasefile'])
                    self.close_db(mydbpath)
                    self.eapi3_lock_repo(*x)
                    do_clear.add(repository)
                    mytxt = blue("%s.") % (_("database got locked. Locking services for it"),)
                    self.updateProgress(
                        "[%s] %s" % (
                                brown(str(x)),
                                mytxt,
                        ),
                        importance = 1,
                        type = "info"
                    )
            elif not os.path.isfile(self.repositories[x]['download_lock']) and \
                self.repositories[x]['locked']:
                mydbpath = os.path.join(self.repositories[x]['dbpath'],etpConst['etpdatabasefile'])
                self.close_db(mydbpath)
                mytxt = blue("%s. %s:") % (
                    _("unlocking and indexing database"),
                    _("hash"),
                )
                self.updateProgress(
                    "[%s] %s" % (
                            brown(str(x)),
                            mytxt,
                    ),
                    importance = 1,
                    type = "info"
                )
                # woohoo, got unlocked eventually
                mydb = self.open_db(mydbpath, docache = False)
                mydb.createAllIndexes()
                self.updateProgress(
                    str(mydb.database_checksum(do_order = True, strict = False, strings = True)),
                    importance = 1,
                    type = "info"
                )
                mydb.closeDB()
                self.Entropy.clear_dump_cache(etpCache['repository_server']+"/"+repository+"/")
                self.repositories[x]['locked'] = False
                self.eapi3_unlock_repo(*x)

        for repo in do_clear:
            self.Entropy.clear_dump_cache(etpCache['repository_server']+"/"+repo+"/")

    def eapi3_lock_repo(self, repository, arch, product, branch):
        lock_file = os.path.join(self.repositories[(repository, arch, product, branch,)]['dbpath'],etpConst['etpdatabaseeapi3lockfile'])
        if not os.path.lexists(lock_file):
            f = open(lock_file,"w")
            f.write("this repository is EAPI3 locked")
            f.flush()
            f.close()

    def eapi3_unlock_repo(self, repository, arch, product, branch):
        lock_file = os.path.join(self.repositories[(repository, arch, product, branch,)]['dbpath'],etpConst['etpdatabaseeapi3lockfile'])
        if os.path.isfile(lock_file):
            os.remove(lock_file)

    def get_dcache(self, item, repo = '_norepo_'):
        return self.dumpTools.loadobj(etpCache['repository_server']+"/"+repo+"/"+str(hash(item)))

    def set_dcache(self, item, data, repo = '_norepo_'):
        self.dumpTools.dumpobj(etpCache['repository_server']+"/"+repo+"/"+str(hash(item)),data)

    def close_db(self, dbpath):
        try:
            dbc = self.syscache['db'].pop(dbpath)
            dbc.closeDB()
        except KeyError:
            pass
        except dbapi2.ProgrammingError:
            # they've been opened by the Commands Processor
            self.syscache['db_trashed'].add(dbc)

    def open_db(self, dbpath, docache = True):
        if docache:
            cached = self.syscache['db'].get(dbpath)
            if cached != None:
                return cached
        dbc = self.Entropy.openGenericDatabase(
            dbpath,
            xcache = False,
            readOnly = True,
            skipChecks = True
        )
        if docache:
            self.syscache['db'][dbpath] = dbc
        return dbc

    def expand_repositories(self):

        for repository,arch,product, branch in self.repositories:
            x = (repository,arch,product,branch,)
            self.repositories[x]['locked'] = True # loading locked
            self.set_repository_db_availability(x)
            mydbpath = self.repositories[x]['dbpath']
            myrevfile = os.path.join(mydbpath,etpConst['etpdatabaserevisionfile'])
            myrev = '0'
            if os.path.isfile(myrevfile):
                while 1:
                    try:
                        f = open(myrevfile)
                        myrev = f.readline().strip()
                        f.close()
                    except IOError: # should never happen but who knows
                        continue
                    break
            self.repositories[x]['dbrevision'] = myrev
            self.repositories[x]['download_lock'] = os.path.join(
                mydbpath,
                etpConst['etpdatabasedownloadlockfile']
            )

class EntropySocketClientCommands:

    def __init__(self, Entropy, Service):

        if not isinstance(Entropy, (EquoInterface, ServerInterface)) and \
            not issubclass(Entropy, (EquoInterface, ServerInterface)):
                mytxt = _("A valid EquoInterface/ServerInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (Entropy,mytxt,))

        if not isinstance(Service, (SystemSocketClientInterface,)) and \
            not issubclass(Service, (SystemSocketClientInterface,)):
                mytxt = _("A valid SystemSocketClientInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (Service,mytxt,))

        import entropyTools, socket, zlib, struct, dumpTools
        self.entropyTools, self.socket, self.zlib, self.struct, self.dumpTools = entropyTools, socket, zlib, struct, dumpTools
        self.Entropy = Entropy
        self.Service = Service
        self.output_header = ''

    def handle_standard_answer(self, data, repository = None, arch = None, product = None):
        do_skip = False
        # elaborate answer
        if data == None:
            mytxt = _("feature not supported remotely")
            self.Entropy.updateProgress(
                "[%s:%s|%s:%s|%s:%s] %s" % (
                        darkblue(_("repo")),
                        bold(str(repository)),
                        darkred(_("arch")),
                        bold(str(arch)),
                        darkgreen(_("product")),
                        bold(str(product)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "error",
                header = self.output_header
            )
            do_skip = True
        elif not data:
            mytxt = _("service temporarily not available")
            self.Entropy.updateProgress(
                "[%s:%s|%s:%s|%s:%s] %s" % (
                        darkblue(_("repo")),
                        bold(str(repository)),
                        darkred(_("arch")),
                        bold(str(arch)),
                        darkgreen(_("product")),
                        bold(str(product)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "error",
                header = self.output_header
            )
            do_skip = True
        elif data == self.Service.answers['no']:
            # command failed
            mytxt = _("command failed")
            self.Entropy.updateProgress(
                "[%s:%s|%s:%s|%s:%s] %s" % (
                        darkblue(_("repo")),
                        bold(str(repository)),
                        darkred(_("arch")),
                        bold(str(arch)),
                        darkgreen(_("product")),
                        bold(str(product)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "error",
                header = self.output_header
            )
        elif data != self.Service.answers['ok']:
            mytxt = _("received wrong answer")
            self.Entropy.updateProgress(
                "[%s:%s|%s:%s|%s:%s] %s: %s" % (
                        darkblue(_("repo")),
                        bold(str(repository)),
                        darkred(_("arch")),
                        bold(str(arch)),
                        darkgreen(_("product")),
                        bold(str(product)),
                        blue(mytxt),
                        repr(data),
                ),
                importance = 1,
                type = "error",
                header = self.output_header
            )
            do_skip = True
        return do_skip

    def get_result(self, session):
        # get the information
        cmd = "%s rc" % (session,)
        self.Service.transmit(cmd)
        try:
            data = self.Service.receive()
            return data
        except:
            self.entropyTools.printTraceback()
            return None

    def convert_stream_to_object(self, data, gzipped, repository = None, arch = None, product = None):

        # unstream object
        error = False
        try:
            data = self.Service.stream_to_object(data, gzipped)
        except (EOFError,IOError,self.zlib.error,self.dumpTools.pickle.UnpicklingError,):
            mytxt = _("cannot convert stream into object")
            self.Entropy.updateProgress(
                "[%s:%s|%s:%s|%s:%s] %s" % (
                        darkblue(_("repo")),
                        bold(unicode(repository)),
                        darkred(_("arch")),
                        bold(unicode(arch)),
                        darkgreen(_("product")),
                        bold(unicode(product)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "error",
                header = self.output_header
            )
            data = None
            error = True
        return data, error

    def retrieve_command_answer(self, cmd, session_id, repository = None, arch = None, product = None, compression = False):

        tries = 3
        lasterr = None
        while 1:

            if tries <= 0:
                return lasterr
            tries -= 1

            try:
                # send command
                self.Service.transmit(cmd)
            except (exceptionTools.SSLError,):
                return None
            # receive answer
            data = self.Service.receive()

            skip = self.handle_standard_answer(data, repository, arch, product)
            if skip:
                continue

            data = self.get_result(session_id)
            if data == None:
                lasterr = None
                continue
            elif not data:
                lasterr = False
                continue

            objdata, error = self.convert_stream_to_object(data, compression, repository, arch, product)
            if not error:
                return objdata

    def do_generic_handler(self, cmd, session_id, tries = 10, compression = False):

        self.Service.check_socket_connection()

        while 1:
            try:
                result = self.retrieve_command_answer(cmd, session_id, compression = compression)
                if result == None:
                    return False,'command not supported' # untranslated on purpose
                return result
            except (self.socket.error,self.struct.error,):
                self.Service.reconnect_socket()
                tries -= 1
                if tries < 1:
                    raise

    def set_gzip_compression(self, session, do):
        self.Service.check_socket_connection()
        cmd = "%s %s %s %s zlib" % (session, 'session_config', 'compression', do,)
        self.Service.transmit(cmd)
        data = self.Service.receive()
        if data == self.Service.answers['ok']:
            return True
        return False

    def service_login(self, username, password, session_id):

        cmd = "%s %s %s %s" % (
            session_id,
            'login',
            username,
            password,
        )
        return self.do_generic_handler(cmd, session_id)

    def service_logout(self, username, session_id):

        cmd = "%s %s %s" % (
            session_id,
            'logout',
            username,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_logged_user_data(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'user_data',
        )
        return self.do_generic_handler(cmd, session_id)

    def is_user(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'is_user',
        )
        return self.do_generic_handler(cmd, session_id)

    def is_developer(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'is_developer',
        )
        return self.do_generic_handler(cmd, session_id)

    def is_moderator(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'is_moderator',
        )
        return self.do_generic_handler(cmd, session_id)

    def is_administrator(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'is_administrator',
        )
        return self.do_generic_handler(cmd, session_id)

    def available_commands(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'available_commands',
        )
        return self.do_generic_handler(cmd, session_id)


class RepositorySocketClientCommands(EntropySocketClientCommands):

    def __init__(self, EntropyInterface, ServiceInterface):
        EntropySocketClientCommands.__init__(self, EntropyInterface, ServiceInterface)

    def differential_packages_comparison(self, session_id, idpackages, repository, arch, product):

        myidlist = ' '.join([str(x) for x in idpackages])
        cmd = "%s %s %s %s %s %s %s" % (
            session_id,
            'repository_server:dbdiff',
            repository,
            arch,
            product,
            etpConst['branch'],
            myidlist,
        )

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        data = self.do_generic_handler(cmd, session_id, tries = 5, compression = compression)

        # disable compression
        self.set_gzip_compression(session_id, False)

        return data

    def get_repository_treeupdates(self, session_id, repository, arch, product):

        cmd = "%s %s %s %s %s %s" % (
            session_id,
            'repository_server:treeupdates',
            repository,
            arch,
            product,
            etpConst['branch'],
        )
        return self.do_generic_handler(cmd, session_id, tries = 5)

    def get_package_sets(self, session_id, repository, arch, product):

        cmd = "%s %s %s %s %s %s" % (
            session_id,
            'repository_server:get_package_sets',
            repository,
            arch,
            product,
            etpConst['branch'],
        )
        return self.do_generic_handler(cmd, session_id, tries = 5)

    def get_package_information(self, session_id, idpackages, repository, arch, product):

        cmd = "%s %s %s %s %s %s %s %s" % (
            session_id,
            'repository_server:pkginfo',
            True,
            repository,
            arch,
            product,
            etpConst['branch'],
            ' '.join([str(x) for x in idpackages]),
        )

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        data = self.do_generic_handler(cmd, session_id, compression = compression)

        # disable compression
        self.set_gzip_compression(session_id, False)

        return data

    def ugc_do_download(self, session_id, pkgkey):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:do_download',
            pkgkey,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_do_downloads(self, session_id, pkgkeys):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:do_downloads',
            ' '.join(pkgkeys),
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_downloads(self, session_id, pkgkey):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_downloads',
            pkgkey,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_alldownloads(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'ugc:get_alldownloads',
        )

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        rc = self.do_generic_handler(cmd, session_id, compression = compression)

        # disable compression
        self.set_gzip_compression(session_id, False)

        return rc

    def ugc_do_vote(self, session_id, pkgkey, vote):

        cmd = "%s %s %s %s" % (
            session_id,
            'ugc:do_vote',
            pkgkey,
            vote,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_vote(self, session_id, pkgkey):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_vote',
            pkgkey,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_allvotes(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'ugc:get_allvotes',
        )

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        rc = self.do_generic_handler(cmd, session_id, compression = compression)

        # disable compression
        self.set_gzip_compression(session_id, False)

        return rc

    def ugc_add_comment(self, session_id, pkgkey, comment, title, keywords):

        mydict = {
            'comment': comment,
            'title': title,
            'keywords': keywords,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s %s" % (
            session_id,
            'ugc:add_comment',
            pkgkey,
            xml_string,
        )

        return self.do_generic_handler(cmd, session_id)

    def ugc_edit_comment(self, session_id, iddoc, new_comment, new_title, new_keywords):

        mydict = {
            'comment': new_comment,
            'title': new_title,
            'keywords': new_keywords,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s %s" % (
            session_id,
            'ugc:edit_comment',
            iddoc,
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_remove_comment(self, session_id, iddoc):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:remove_comment',
            iddoc,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_remove_image(self, session_id, iddoc):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:remove_image',
            iddoc,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_remove_file(self, session_id, iddoc):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:remove_file',
            iddoc,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_remove_youtube_video(self, session_id, iddoc):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:remove_youtube_video',
            iddoc,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_docs(self, session_id, pkgkey):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_alldocs',
            pkgkey,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_textdocs(self, session_id, pkgkey):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_textdocs',
            pkgkey,
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_textdocs_by_identifiers(self, session_id, identifiers):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_textdocs_by_identifiers',
            ' '.join([str(x) for x in identifiers]),
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_get_documents_by_identifiers(self, session_id, identifiers):

        cmd = "%s %s %s" % (
            session_id,
            'ugc:get_documents_by_identifiers',
            ' '.join([str(x) for x in identifiers]),
        )
        return self.do_generic_handler(cmd, session_id)

    def ugc_send_file_stream(self, session_id, file_path):

        if not (os.path.isfile(file_path) and os.access(file_path,os.R_OK)):
            return False,False,'cannot read file_path'

        import zlib
        # enable stream
        cmd = "%s %s %s on" % (
            session_id,
            'session_config',
            'stream',
        )
        status, msg = self.do_generic_handler(cmd, session_id)
        if not status:
            return False,status,msg

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        # start streamer
        stream_status = True
        stream_msg = 'ok'
        f = open(file_path,"rb")
        chunk = f.read(8192)
        base_path = os.path.basename(file_path)
        transferred = len(chunk)
        max_size = self.entropyTools.get_file_size(file_path)
        while chunk:

            if (not self.Service.quiet) or self.Service.show_progress:
                self.Entropy.updateProgress(
                    "%s, %s: %s" % (
                        blue(_("User Generated Content")),
                        darkgreen(_("sending file")),
                        darkred(base_path),
                    ),
                    importance = 1,
                    type = "info",
                    header = brown(" @@ "),
                    back = True,
                    count = (transferred,max_size,),
                    percent = True
                )

            chunk = zlib.compress(chunk, 7) # compression level 1-9
            cmd = "%s %s %s" % (
                session_id,
                'stream',
                chunk,
            )
            status, msg = self.do_generic_handler(cmd, session_id, compression = compression)
            if not status:
                stream_status = status
                stream_msg = msg
                break
            chunk = f.read(8192)
            transferred += len(chunk)

        f.close()

        # disable compression
        self.set_gzip_compression(session_id, False)

        # disable config
        cmd = "%s %s %s off" % (
            session_id,
            'session_config',
            'stream',
        )
        status, msg = self.do_generic_handler(cmd, session_id)
        if not status:
            return False,status,msg

        return True,stream_status,stream_msg

    def ugc_send_file(self, session_id, pkgkey, file_path, doc_type, title, description, keywords):

        status, rem_status, err_msg = self.ugc_send_file_stream(session_id, file_path)
        if not (status and rem_status):
            return False,err_msg

        mydict = {
            'doc_type': str(doc_type),
            'title': title,
            'description': description,
            'keywords': keywords,
            'file_name': os.path.join(pkgkey,os.path.basename(file_path)),
            'real_filename': os.path.basename(file_path),
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s %s" % (
            session_id,
            'ugc:register_stream',
            pkgkey,
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)


class SystemSocketClientInterface:

    import socket
    import dumpTools
    import entropyTools
    import zlib
    import select
    def __init__(self, EntropyInterface, ClientCommandsClass, quiet = False, show_progress = True, output_header = '', ssl = False, socket_timeout = 25): #, server_ca_cert = None, server_cert = None):

        if not isinstance(EntropyInterface, (EquoInterface, ServerInterface)) and \
            not issubclass(EntropyInterface, (EquoInterface, ServerInterface)):
                mytxt = _("A valid EquoInterface/ServerInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,))

        if not issubclass(ClientCommandsClass, (EntropySocketClientCommands,)):
            mytxt = _("A valid EntropySocketClientCommands based class is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (ClientCommandsClass,mytxt,))

        self.setup_ssl(ssl) #, server_ca_cert, server_cert)

        self.answers = etpConst['socket_service']['answers']
        self.Entropy = EntropyInterface
        self.sock_conn = None
        self.real_sock_conn = None
        self.hostname = None
        self.hostport = None
        self.buffered_data = ''
        self.buffer_length = None
        self.quiet = quiet
        self.show_progress = show_progress
        self.output_header = output_header
        self.CmdInterface = ClientCommandsClass(self.Entropy, self)
        self.CmdInterface.output_header = self.output_header
        self.socket_timeout = socket_timeout
        self.socket.setdefaulttimeout(self.socket_timeout)


    def setup_ssl(self, ssl): # , server_ca_cert, server_cert):
        # SSL Support
        self.SSL = {}
        self.SSL_exceptions = {
            'WantReadError': None,
            'WantWriteError': None,
            'WantX509LookupError': None,
            'ZeroReturnError': None,
            'Error': None,
            'SysCallError': None
        }
        self.ssl = ssl
        self.pyopenssl = True
        self.context = None

        '''
            self.server_cert = server_cert
            self.server_ca_cert = server_ca_cert
            self.ssl_pkey = None
            self.ssl_cert = None
            self.ssl_CN = 'Entropy Repository Service Client'
            self.ssl_digest = 'md5'
            self.ssl_serial = 1
            self.ssl_not_before = 0
            self.ssl_not_after = 60*60*24*1 # 1 day
        '''

        if self.ssl:

            try:
                from OpenSSL import SSL, crypto
            except ImportError:
                self.pyopenssl = False

            '''
            if not (self.server_cert and self.server_ca_cert):
                raise exceptionTools.SSLError('SSLError: %s: %s' % (_("Specified SSL server certificate not available"),)
            if not (os.path.isfile(self.server_cert) and \
                    os.access(self.server_cert,os.R_OK) and \
                    os.path.isfile(self.server_ca_cert) and \
                    os.access(self.server_ca_cert,os.R_OK)) and self.pyopenssl:
                        raise exceptionTools.SSLError('SSLError: %s: %s' % (_("Specified SSL server certificate not available"),self.server_cert,))
            '''

            if self.pyopenssl:

                self.SSL_exceptions['WantReadError'] = SSL.WantReadError
                self.SSL_exceptions['WantWriteError'] = SSL.WantWriteError
                self.SSL_exceptions['WantX509LookupError'] = SSL.WantX509LookupError
                self.SSL_exceptions['ZeroReturnError'] = SSL.ZeroReturnError
                self.SSL_exceptions['Error'] = SSL.Error
                self.SSL_exceptions['SysCallError'] = SSL.SysCallError
                self.SSL['m'] = SSL
                self.SSL['crypto'] = crypto

                # setup an SSL context.
                self.context = self.SSL['m'].Context(self.SSL['m'].SSLv23_METHOD)
                #self.context.set_verify(self.SSL['m'].VERIFY_PEER, self.verify_ssl_cb)

                # load up certificate stuff.
                '''
                    self.ssl_pkey = self.create_ssl_key_pair(self.SSL['crypto'].TYPE_RSA, 1024)
                    self.context.use_privatekey(self.ssl_pkey)
                    self.context.use_certificate_file(self.server_cert)
                    self.context.load_verify_locations(self.server_ca_cert)
                    self.context.load_client_ca(self.server_cert)
                    self.ssl_pkey = self.create_ssl_key_pair(self.SSL['crypto'].TYPE_RSA, 1024)
                    self.ssl_cert = self.create_ssl_certificate(self.ssl_pkey)
                    self.context.use_privatekey(self.ssl_pkey)
                    self.context.use_certificate(self.ssl_cert)
                    self.context.load_client_ca(self.server_cert)
                '''

        else:
            self.ssl = False
            self.pyopenssl = False


    def check_pyopenssl(self):
        if not self.pyopenssl:
            raise exceptionTools.SSLError('SSLError: %s' % (_("OpenSSL Python module not available, you need dev-python/pyopenssl"),))

    '''
    # this function should do the authentication checking to see that
    # the client is who they say they are.
    def verify_ssl_cb(self, conn, cert, errnum, depth, ok):
        self.check_pyopenssl()
        #print 'Got certificate: %s' % cert.get_subject()
        #print repr(ok),repr(cert),repr(errnum),repr(depth)
        return ok


    def create_ssl_key_pair(self, keytype, bits):
        if not self.pyopenssl:
            raise exceptionTools.SSLError('SSLError: %s' % (_("OpenSSL Python module not available, you need dev-python/pyopenssl"),))
        pkey = self.SSL['crypto'].PKey()
        pkey.generate_key(keytype, bits)
        return pkey

    def create_ssl_certificate(self, pkey):
        self.check_pyopenssl()
        myreq = self.create_ssl_certificate_request(pkey, CN = self.ssl_CN)
        cert = self.SSL['crypto'].X509()
        cert.set_serial_number(self.ssl_serial)
        cert.gmtime_adj_notBefore(self.ssl_not_before)
        cert.gmtime_adj_notAfter(self.ssl_not_after)
        cert.set_issuer(myreq.get_subject())
        cert.set_subject(myreq.get_subject())
        cert.set_pubkey(myreq.get_pubkey())
        cert.sign(pkey, self.ssl_digest)
        return cert

    def create_ssl_certificate_request(self, pkey, **name):
        self.check_pyopenssl()
        req = self.SSL['crypto'].X509Req()
        subj = req.get_subject()
        for (key,value) in name.items():
            setattr(subj, key, value)
        req.set_pubkey(pkey)
        req.sign(pkey, self.ssl_digest)

        return req
    '''

    def stream_to_object(self, data, gzipped):

        if gzipped:
            data = self.zlib.decompress(data)
        obj = self.dumpTools.unserialize_string(data)

        return obj

    def append_eos(self, data):
        return str(len(data))+self.answers['eos']+data

    def transmit(self, data):
        self.check_socket_connection()
        if hasattr(self.sock_conn,'settimeout'):
            self.sock_conn.settimeout(self.socket_timeout)
        data = self.append_eos(data)
        try:

            if self.ssl and not self.pyopenssl:
                try:
                    self.sock_conn.write(data)
                except UnicodeEncodeError:
                    self.sock_conn.write(data.encode('utf-8'))
            else:
                encode_done = False
                mydata = data[:]
                while 1:
                    try:
                        sent = self.sock_conn.send(mydata)
                        if sent == len(mydata):
                            break
                        mydata = mydata[sent:]
                    except (self.SSL_exceptions['WantWriteError'],self.SSL_exceptions['WantReadError'],):
                        time.sleep(0.2)
                        continue
                    except UnicodeEncodeError, e:
                        if encode_done:
                            raise
                        mydata = mydata.encode('utf-8')
                        encode_done = True
                        continue

        except self.SSL_exceptions['Error'], e:
            self.disconnect()
            raise exceptionTools.SSLError('SSLError: %s' % (e,))
        except self.socket.sslerror, e:
            self.disconnect()
            raise exceptionTools.SSLError('SSL Socket error: %s' % (e,))
        except:
            self.disconnect()
            raise

    def close_session(self, session_id):
        self.check_socket_connection()
        try:
            self.transmit("%s end" % (session_id,))
            # since we don't know if it's expired, we need to wrap it
            data = self.receive()
        except self.socket.error, e:
            if etpUi['debug']:
                self.entropyTools.printTraceback()
                import pdb
                pdb.set_trace()
            if e[0] == 32: # broken pipe
                return None
            raise
        except exceptionTools.SSLError:
            raise
        return data

    def open_session(self):
        self.check_socket_connection()
        self.socket.setdefaulttimeout(self.socket_timeout)
        self.transmit('begin')
        data = self.receive()
        return data

    def receive(self):

        self.check_socket_connection()
        if hasattr(self.sock_conn,'settimeout'):
            self.sock_conn.settimeout(self.socket_timeout)
        self.ssl_prepending = True

        def do_receive():
            data = ''
            if self.ssl and not self.pyopenssl:
                data = self.sock_conn.read(1024)
            elif self.ssl:
                if self.ssl_prepending:
                    data = self.sock_conn.recv(1024)
                    self.ssl_prepending = False
                while self.sock_conn.pending():
                    data += self.sock_conn.recv(1024)
            else:
                data = self.sock_conn.recv(1024)
            return data

        myeos = self.answers['eos']
        ssl_error_loop_count = 0
        while 1:

            try:

                data = do_receive()
                if self.buffer_length == None:
                    self.buffered_data = ''
                    if (data == '') or (data == self.answers['cl']):
                        # nein! no support, KAPUTT!
                        # RAUSS!
                        if not self.quiet:
                            mytxt = _("command not supported. receive aborted")
                            self.Entropy.updateProgress(
                                "[%s:%s] %s" % (
                                        brown(self.hostname),
                                        bold(str(self.hostport)),
                                        blue(mytxt),
                                ),
                                importance = 1,
                                type = "warning",
                                header = self.output_header
                            )
                        return None
                    elif len(data) < len(myeos):
                        if not self.quiet:
                            mytxt = _("malformed EOS. receive aborted")
                            self.Entropy.updateProgress(
                                "[%s:%s] %s" % (
                                        brown(self.hostname),
                                        bold(str(self.hostport)),
                                        blue(mytxt),
                                ),
                                importance = 1,
                                type = "warning",
                                header = self.output_header
                            )
                        if etpUi['debug']:
                            self.entropyTools.printTraceback()
                            import pdb
                            pdb.set_trace()
                        return None
                    mystrlen = data.split(myeos)[0]
                    self.buffer_length = int(mystrlen)
                    data = data[len(mystrlen)+1:]
                    self.buffer_length -= len(data)
                    self.buffered_data = data
                else:
                    self.buffer_length -= len(data)
                    self.buffered_data += data

                while self.buffer_length > 0:
                    x = do_receive()
                    if self.ssl and self.pyopenssl and not x:
                        self.ssl_prepending = True
                    self.buffer_length -= len(x)
                    self.buffered_data += x
                self.buffer_length = None
                break

            except ValueError, e:
                if not self.quiet:
                    mytxt = _("malformed data. receive aborted")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s: %s" % (
                                brown(self.hostname),
                                bold(str(self.hostport)),
                                blue(mytxt),
                                e,
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
                return None
            except self.socket.timeout, e:
                if not self.quiet:
                    mytxt = _("connection timed out while receiving data")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s: %s" % (
                                brown(self.hostname),
                                bold(str(self.hostport)),
                                blue(mytxt),
                                e,
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
                return None
            except self.socket.error, e:
                if not self.quiet:
                    mytxt = _("connection error while receiving data")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s: %s" % (
                                brown(self.hostname),
                                bold(str(self.hostport)),
                                blue(mytxt),
                                e,
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
                return None
            except (self.SSL_exceptions['WantReadError'],self.SSL_exceptions['WantX509LookupError'],), e:
                ssl_error_loop_count += 1
                if ssl_error_loop_count > 3000000:
                    if not self.quiet:
                        mytxt = _("too many WantReadError error while receiving data")
                        self.Entropy.updateProgress(
                            "[%s:%s] %s: %s" % (
                                    brown(self.hostname),
                                    bold(str(self.hostport)),
                                    blue(mytxt),
                                    e,
                            ),
                            importance = 1,
                            type = "warning",
                            header = self.output_header
                        )
                    return None
                continue
            except self.SSL_exceptions['ZeroReturnError']:
                break
            except self.SSL_exceptions['SysCallError'], e:
                if not self.quiet:
                    mytxt = _("syscall error while receiving data")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s: %s" % (
                                brown(self.hostname),
                                bold(str(self.hostport)),
                                blue(mytxt),
                                e,
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
                return None

        return self.buffered_data

    def reconnect_socket(self):
        if not self.quiet:
            mytxt = _("Reconnecting to socket")
            self.Entropy.updateProgress(
                "[%s:%s] %s" % (
                        brown(unicode(self.hostname)),
                        bold(unicode(self.hostport)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "info",
                header = self.output_header
            )
        self.connect(self.hostname,self.hostport)

    def check_socket_connection(self):
        if not self.sock_conn:
            raise exceptionTools.ConnectionError("ConnectionError: %s" % (_("Not connected to host"),))

    def connect(self, host, port):

        if self.ssl:
            self.real_sock_conn = self.socket.socket(self.socket.AF_INET, self.socket.SOCK_STREAM)
            if hasattr(self.real_sock_conn,'settimeout'):
                self.real_sock_conn.settimeout(self.socket_timeout)
            if self.pyopenssl:
                self.sock_conn = self.SSL['m'].Connection(self.context, self.real_sock_conn)
            else:
                self.sock_conn = self.real_sock_conn
        else:
            self.sock_conn = self.socket.socket(self.socket.AF_INET, self.socket.SOCK_STREAM)
            if hasattr(self.sock_conn,'settimeout'):
                self.sock_conn.settimeout(self.socket_timeout)
            self.real_sock_conn = self.sock_conn

        self.hostname = host
        self.hostport = port

        try:
            self.sock_conn.connect((self.hostname, self.hostport))
            if self.ssl and not self.pyopenssl:
                self.sock_conn = self.socket.ssl(self.real_sock_conn)
                # inform about certificate verification
                if not self.quiet:
                    mytxt = _("Warning: you are using an emergency SSL interface, SSL certificate can't be verified. Please install dev-python/pyopenssl")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s" % (
                                brown(str(self.hostname)),
                                bold(str(self.hostport)),
                                blue(mytxt),
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
                    mytxt = _("Service issuer")
                    self.Entropy.updateProgress(
                        "[%s:%s] %s: %s" % (
                                brown(str(self.hostname)),
                                bold(str(self.hostport)),
                                blue(mytxt),
                                self.sock_conn.issuer()
                        ),
                        importance = 1,
                        type = "warning",
                        header = self.output_header
                    )
        except self.socket.error, e:
            if e[0] == 111:
                mytxt = "%s: %s, %s: %s" % (_("Cannot connect to"),host,_("on port"),port,)
                raise exceptionTools.ConnectionError("ConnectionError: %s" % (mytxt,))
            else:
                raise

        if not self.quiet:
            mytxt = _("Successfully connected to host")
            self.Entropy.updateProgress(
                "[%s:%s] %s" % (
                        brown(self.hostname),
                        bold(str(self.hostport)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "info",
                header = self.output_header
            )

    def disconnect(self):
        if not self.real_sock_conn:
            return True
        if self.ssl and self.pyopenssl:
            self.sock_conn.shutdown()
            self.sock_conn.close()
        elif self.ssl and not self.pyopenssl:
            try:
                self.real_sock_conn.shutdown(self.socket.SHUT_RDWR)
            except self.socket.error:
                pass
        del self.sock_conn
        self.sock_conn = None
        try:
            self.real_sock_conn.close()
        except self.socket.error:
            pass
        if not self.quiet:
            mytxt = _("Successfully disconnected from host")
            self.Entropy.updateProgress(
                "[%s:%s] %s" % (
                        brown(self.hostname),
                        bold(str(self.hostport)),
                        blue(mytxt),
                ),
                importance = 1,
                type = "info",
                header = self.output_header
            )
        self.real_sock_conn = None
        # otherwise reconnect_socket won't work
        #self.hostname = None
        #self.hostport = None

class SystemManagerExecutorInterface:

    def __init__(self, SystemInterface, Entropy):
        import entropyTools
        self.entropyTools = entropyTools
        self.Entropy = Entropy
        self.SystemInterface = SystemInterface
        self.available_commands = {}
        self.task_result = None

    def register(self, available_commands):
        self.available_commands.update(available_commands)

    def execute_task(self, command_data):

        import signal
        queue_id = command_data['queue_id']
        args = command_data['args']
        kwargs = command_data['kwargs']
        data = self.available_commands.get(command_data['call'])

        if data == None:
            return False, 'no command'
        elif len(args)+1 < data['args']:
            return False, 'not enough args'



        args.insert(0,queue_id)
        self.task_result = None
        t = self.entropyTools.parallelTask(data['func'], *args, **kwargs)
        t.start()
        killed = False
        while 1:
            if not t.isAlive():
                break
            time.sleep(1)
            live_item, key = self.SystemInterface.get_item_by_queue_id(queue_id)
            if isinstance(live_item,dict) and (key == "processing") and (not killed):
                if live_item['kill'] and (live_item['processing_pid'] != None):
                    os.kill(live_item['processing_pid'],signal.SIGKILL)
                    killed = True
        if killed:
            return False, 'killed by user'
        return True, t.result


class SystemManagerExecutorServerRepositoryInterface:

    import entropyTools
    def __init__(self, SystemManagerExecutorInstance, *args, **kwargs):

        try:
            import cPickle as pickle
        except ImportError:
            import pickle
        self.pickle = pickle

        import entropyTools
        self.entropyTools = entropyTools
        self.SystemManagerExecutor = SystemManagerExecutorInstance
        self.args = args
        self.kwargs = kwargs
        self.available_commands = {
            'sync_spm': {
                'func': self.sync_portage,
                'args': 1,
            },
            'compile_atoms': {
                'func': self.compile_atoms,
                'args': 2,
            },
            'spm_remove_atoms': {
                'func': self.spm_remove_atoms,
                'args': 2,
            },
            'get_spm_categories_updates': {
                'func': self.get_spm_categories_updates,
                'args': 2,
            },
            'get_spm_categories_installed': {
                'func': self.get_spm_categories_installed,
                'args': 2,
            },
            'enable_uses_for_atoms': {
                'func': self.enable_uses_for_atoms,
                'args': 3,
            },
            'disable_uses_for_atoms': {
                'func': self.disable_uses_for_atoms,
                'args': 3,
            },
            'get_spm_atoms_info': {
                'func': self.get_spm_atoms_info,
                'args': 2,
            },
            'run_spm_info': {
                'func': self.run_spm_info,
                'args': 1,
            },
            'run_custom_shell_command': {
                'func': self.run_custom_shell_command,
                'args': 1,
            },
            'get_spm_glsa_data': {
                'func': self.get_spm_glsa_data,
                'args': 1,
            },
            'move_entropy_packages_to_repository': {
                'func': self.move_entropy_packages_to_repository,
                'args': 5,
            },
            'scan_entropy_packages_database_changes': {
                'func': self.scan_entropy_packages_database_changes,
                'args': 1,
            },
            'run_entropy_database_updates': {
                'func': self.run_entropy_database_updates,
                'args': 4,
            },
            'run_entropy_dependency_test': {
                'func': self.run_entropy_dependency_test,
                'args': 1,
            },
            'run_entropy_library_test': {
                'func': self.run_entropy_library_test,
                'args': 1,
            },
            'run_entropy_treeupdates': {
                'func': self.run_entropy_treeupdates,
                'args': 2,
            },
            'scan_entropy_mirror_updates': {
                'func': self.scan_entropy_mirror_updates,
                'args': 2,
            },
            'run_entropy_mirror_updates': {
                'func': self.run_entropy_mirror_updates,
                'args': 2,
            },
            'run_entropy_checksum_test': {
                'func': self.run_entropy_checksum_test,
                'args': 3,
            },
            'get_notice_board': {
                'func': self.get_notice_board,
                'args': 2,
            },
            'remove_notice_board_entries': {
                'func': self.remove_notice_board_entries,
                'args': 3,
            },
            'add_notice_board_entry': {
                'func': self.add_notice_board_entry,
                'args': 5,
            },
        }

    def _set_processing_pid(self, queue_id, process_pid):
        self.SystemManagerExecutor.SystemInterface.queue_lock_acquire()
        try:
            live_item, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
            if isinstance(live_item,dict):
                live_item['processing_pid'] = process_pid
                self.SystemManagerExecutor.SystemInterface.store_queue()
        finally:
            self.SystemManagerExecutor.SystemInterface.queue_lock_release()

    def sync_portage(self, queue_id):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        cmd = ["emerge", "--sync"]
        try:
            p = subprocess.Popen(cmd, stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id))
            self._set_processing_pid(queue_id, p.pid)
            rc = p.wait()
        finally:
            stdout_err.write('\n\n Stdin: %s' % (self._get_stdin(queue_id),))
            stdout_err.write("\n### Done ###\n")
            stdout_err.flush()
            stdout_err.close()
        return True,rc

    def compile_atoms(  self,
                        queue_id, atoms,
                        pretend = False, oneshot = False,
                        verbose = True, nocolor = True,
                        fetchonly = False, buildonly = False,
                        nodeps = False, custom_use = '', ldflags = '', cflags = ''
        ):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        cmd = [etpConst['spm']['env_update_cmd'],"&&"]
        cmd += etpConst['spm']['source_profile']+["&&"]
        if custom_use:
            cmd += ['export USE="']+custom_use.strip().split()+['"','&&']
        if ldflags:
            cmd += ['export LDFLAGS="']+custom_use.strip().split()+['"','&&']
        if cflags:
            cmd += ['export CFLAGS="']+custom_use.strip().split()+['"','&&']
        cmd += [etpConst['spm']['exec']]+atoms
        if pretend:
            cmd.append(etpConst['spm']['pretend_cmd'])
        if verbose:
            cmd.append(etpConst['spm']['verbose_cmd'])
        if oneshot:
            cmd.append(etpConst['spm']['oneshot_cmd'])
        if nocolor:
            cmd.append(etpConst['spm']['nocolor_cmd'])
        if fetchonly:
            cmd.append(etpConst['spm']['fetchonly_cmd'])
        if buildonly:
            cmd.append(etpConst['spm']['buildonly_cmd'])
        if nodeps:
            cmd.append(etpConst['spm']['nodeps_cmd'])

        stdout_err.write("Preparing to spawn parameter: '%s'. Good luck mate!\n" % (' '.join(cmd),))
        stdout_err.flush()

        try:
            p = subprocess.Popen(' '.join(cmd), stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id), shell = True)
            self._set_processing_pid(queue_id, p.pid)
            rc = p.wait()
        finally:
            stdout_err.write('\n\n Stdin: %s' % (self._get_stdin(queue_id),))
            stdout_err.write("\n### Done ###\n")
            stdout_err.flush()
            stdout_err.close()
        return True,rc

    def spm_remove_atoms(self, queue_id, atoms, pretend = True, verbose = True, nocolor = True):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        cmd = [etpConst['spm']['env_update_cmd'],"&&"]
        cmd += etpConst['spm']['source_profile']+["&&"]
        cmd += [etpConst['spm']['exec'],etpConst['spm']['remove_cmd']]+atoms
        if pretend:
            cmd.append(etpConst['spm']['pretend_cmd'])
        if verbose:
            cmd.append(etpConst['spm']['verbose_cmd'])
        if nocolor:
            cmd.append(etpConst['spm']['nocolor_cmd'])

        stdout_err.write("Preparing to spawn parameter: '%s'. Good luck mate!\n" % (' '.join(cmd),))
        stdout_err.flush()

        try:
            p = subprocess.Popen(' '.join(cmd), stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id), shell = True)
            self._set_processing_pid(queue_id, p.pid)
            rc = p.wait()
        finally:
            stdout_err.write('\n\n Stdin: %s' % (self._get_stdin(queue_id),))
            stdout_err.write("\n### Done ###\n")
            stdout_err.flush()
            stdout_err.close()
        return True,rc

    def enable_uses_for_atoms(self, queue_id, atoms, useflags):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        use_data = {}
        for atom in atoms:
            try:
                status = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.enable_package_useflags(atom, useflags)
            except:
                continue
            if status:
                use_data[atom] = {}
                matched_atom = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_best_atom(atom)
                use_data[atom] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_useflags(matched_atom)

        return True, use_data

    def disable_uses_for_atoms(self, queue_id, atoms, useflags):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        use_data = {}
        for atom in atoms:
            try:
                status = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.disable_package_useflags(atom, useflags)
            except:
                continue
            if status:
                use_data[atom] = {}
                matched_atom = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_best_atom(atom)
                use_data[atom] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_useflags(matched_atom)

        return True, use_data

    def get_spm_atoms_info(self, queue_id, atoms):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        atoms_data = {}
        for atom in atoms:

            try:
                key = self.entropyTools.dep_getkey(atom)
                category = key.split("/")[0]
            except:
                continue
            matched_atom = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_best_atom(atom)
            if not matched_atom: continue

            if not atoms_data.has_key(category):
                atoms_data[category] = {}

            atoms_data[category][matched_atom] = self._get_spm_pkginfo(matched_atom)

        return True, atoms_data

    def get_spm_categories_updates(self, queue_id, categories):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        packages = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_available_packages(categories)
        package_data = {}
        for package in packages:
            try:
                key = self.entropyTools.dep_getkey(package)
                category = key.split("/")[0]
            except:
                continue
            if not package_data.has_key(category):
                package_data[category] = {}
            package_data[category][package] = self._get_spm_pkginfo(package)

        return True, package_data

    def get_spm_categories_installed(self, queue_id, categories):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        packages, pkg_len = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_installed_packages(categories = categories)
        package_data = {}
        for package in packages:
            try:
                key = self.entropyTools.dep_getkey(package)
                category = key.split("/")[0]
            except:
                continue
            if not package_data.has_key(category):
                package_data[category] = {}
            package_data[category][package] = self._get_spm_pkginfo(package, from_installed = True)

        return True, package_data

    def run_spm_info(self, queue_id):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        cmd = [etpConst['spm']['exec'],etpConst['spm']['info_cmd']]

        stdout_err.write("Preparing to spawn parameter: '%s'. Good luck mate!\n" % (' '.join(cmd),))
        stdout_err.flush()

        try:
            p = subprocess.Popen(cmd, stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id))
            self._set_processing_pid(queue_id, p.pid)
            rc = p.wait()
        finally:
            stdout_err.write('\n\n Stdin: %s' % (self._get_stdin(queue_id),))
            stdout_err.write("\n### Done ###\n")
            stdout_err.flush()
            stdout_err.close()
        return True,rc

    def run_custom_shell_command(self, queue_id, command):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        cmd = [etpConst['spm']['env_update_cmd'],"&&"]
        cmd += etpConst['spm']['source_profile']+[";"]
        cmd += command.split()

        cmd = ' '.join(cmd)
        stdout_err.write("Preparing to spawn parameter: '%s'. Good luck mate!\n" % (cmd,))
        stdout_err.flush()

        try:
            p = subprocess.Popen(cmd, stdout = stdout_err, stderr = stdout_err, stdin = self._get_stdin(queue_id), shell = True)
            self._set_processing_pid(queue_id, p.pid)
            rc = p.wait()
        finally:
            stdout_err.write('Stdin: %s\n' % (self._get_stdin(queue_id),))
            stdout_err.write("\n### Done ###\n")
            stdout_err.flush()
            stdout_err.close()
        return True,rc

    def move_entropy_packages_to_repository(self, queue_id, from_repo, to_repo, idpackages, do_copy):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        # run
        matches = []
        for idpackage in idpackages:
            matches.append((idpackage,from_repo,))

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                switched = self.SystemManagerExecutor.SystemInterface.Entropy.move_packages(
                    matches, to_repo,
                    from_repo = from_repo,
                    ask = False,
                    do_copy = do_copy
                )
                return switched
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        switched = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()

        rc = 1
        if len(switched) == len(idpackages):
            rc = 0
        return True,rc

    def scan_entropy_packages_database_changes(self, queue_id):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")
        Entropy = self.SystemManagerExecutor.SystemInterface.Entropy

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:

                for repoid in Entropy.get_available_repositories():
                    self.run_entropy_treeupdates(queue_id, repoid)

                stdout_err.write("\n"+_("Calculating updates...").encode('utf-8')+"\n")
                stdout_err.flush()

                to_add, to_remove, to_inject = Entropy.scan_package_changes()
                mydict = { 'add': to_add, 'remove': to_remove, 'inject': to_inject }

                # setup add data
                mydict['add_data'] = {}
                for portage_atom, portage_counter in to_add:
                    mydict['add_data'][(portage_atom, portage_counter,)] = self._get_spm_pkginfo(portage_atom,from_installed = True)

                mydict['remove_data'] = {}
                for idpackage, repoid in to_remove:
                    dbconn = Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
                    mydict['remove_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
                    dbconn.closeDB()

                mydict['inject_data'] = {}
                for idpackage, repoid in to_inject:
                    dbconn = Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
                    mydict['inject_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
                    dbconn.closeDB()

                return True,mydict

            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        data = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return data

    def run_entropy_database_updates(self, queue_id, to_add, to_remove, to_inject):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")
        Entropy = self.SystemManagerExecutor.SystemInterface.Entropy

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:

                atoms_removed = []
                matches_injected = set()

                if to_inject: Entropy.updateProgress(_("Running package injection"))

                # run inject
                for idpackage, repoid in to_inject:
                    matches_injected.add((idpackage,repoid,))
                    Entropy.transform_package_into_injected(idpackage, repo = repoid)

                if to_remove: Entropy.updateProgress(_("Running package removal"))

                # run remove
                remdata = {}
                for idpackage,repoid in to_remove:
                    dbconn = Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
                    atoms_removed.append(dbconn.retrieveAtom(idpackage))
                    dbconn.closeDB()
                    if not remdata.has_key(repoid):
                        remdata[repoid] = set()
                    remdata[repoid].add(idpackage)
                for repoid in remdata:
                    Entropy.remove_packages(remdata[repoid], repo = repoid)

                mydict = {
                    'added_data': {},
                    'remove_data': atoms_removed,
                    'inject_data': {}
                }

                if to_add:
                    problems = Entropy.check_config_file_updates()
                    if problems:
                        return False,mydict
                    Entropy.updateProgress(_("Running package quickpkg"))

                # run quickpkg
                for repoid in to_add:
                    store_dir = Entropy.get_local_store_directory(repo = repoid)
                    for atom in to_add[repoid]:
                        Entropy.quickpkg(atom,store_dir)

                # inject new into db
                avail_repos = Entropy.get_available_repositories()
                if etpConst['clientserverrepoid'] in avail_repos:
                    avail_repos.pop(etpConst['clientserverrepoid'])
                matches_added = set()
                for repoid in avail_repos:
                    store_dir = Entropy.get_local_store_directory(repo = repoid)
                    package_files = os.listdir(store_dir)
                    if not package_files: continue
                    package_files = [(os.path.join(store_dir,x),False) for x in package_files]

                    Entropy.updateProgress( "[%s|%s] %s" % (
                            repoid,
                            etpConst['branch'],
                            _("Adding packages"),
                        )
                    )
                    for package_file, inject in package_files:
                        Entropy.updateProgress("    %s" % (package_file,))

                    idpackages = Entropy.add_packages_to_repository(package_files, ask = False, repo = repoid)
                    matches_added |= set([(x,repoid,) for x in idpackages])


                Entropy.dependencies_test()

                for idpackage, repoid in matches_added:
                    dbconn = Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
                    mydict['added_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
                    dbconn.closeDB()
                for idpackage, repoid in matches_injected:
                    dbconn = Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
                    mydict['inject_data'][(idpackage, repoid,)] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
                    dbconn.closeDB()
                return True, mydict

            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        data = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return data

    def run_entropy_dependency_test(self, queue_id):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                deps_not_matched = self.SystemManagerExecutor.SystemInterface.Entropy.dependencies_test()
                return deps_not_matched
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        data = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return data

    def run_entropy_library_test(self, queue_id):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                return self.SystemManagerExecutor.SystemInterface.Entropy.libraries_test()
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        status, result = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()

        mystatus = False
        if status == 0: mystatus = True
        if not result: result = set()
        return mystatus,result

    def run_entropy_checksum_test(self, queue_id, repoid, mode):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                if mode == "local":
                    data = self.SystemManagerExecutor.SystemInterface.Entropy.verify_local_packages([], ask = False, repo = repoid)
                else:
                    data = self.SystemManagerExecutor.SystemInterface.Entropy.verify_remote_packages([], ask = False, repo = repoid)
                return True, data
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        mydata = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return mydata

    def run_entropy_treeupdates(self, queue_id, repoid):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                sys.stdout.write(_("Opening database to let it run treeupdates. If you won't see anything below, it's just fine.").encode('utf-8')+"\n")
                dbconn = self.SystemManagerExecutor.SystemInterface.Entropy.openServerDatabase(
                    repo = repoid, do_cache = False,
                    read_only = True
                )
                dbconn.closeDB()
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return True, 0

    def scan_entropy_mirror_updates(self, queue_id, repositories):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")
        import socket
        Entropy = self.SystemManagerExecutor.SystemInterface.Entropy

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:

                sys.stdout.write(_("Scanning").encode('utf-8')+"\n")
                repo_data = {}
                for repoid in repositories:

                    repo_data[repoid] = {}

                    for uri in Entropy.get_remote_mirrors(repoid):

                        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

                        repo_data[repoid][crippled_uri] = {}
                        repo_data[repoid][crippled_uri]['packages'] = {}

                        try:
                            upload_queue, download_queue, removal_queue, \
                                fine_queue, remote_packages_data = Entropy.MirrorsService.calculate_packages_to_sync(uri, etpConst['branch'], repoid)
                        except socket.error:
                            self.entropyTools.printTraceback(f = stdout_err)
                            stdout_err.write("\n"+_("Socket error, continuing...").encode('utf-8')+"\n")
                            continue

                        if (upload_queue or download_queue or removal_queue):
                            upload, download, removal, copy, metainfo = Entropy.MirrorsService.expand_queues(
                                upload_queue,
                                download_queue,
                                removal_queue,
                                remote_packages_data,
                                etpConst['branch'],
                                repoid
                            )
                            if len(upload)+len(download)+len(removal)+len(copy):
                                repo_data[repoid][crippled_uri]['packages'] = {
                                    'upload': upload,
                                    'download': download,
                                    'removal': removal,
                                    'copy': copy,
                                }

                        # now the db
                        current_revision = Entropy.get_local_database_revision(repoid)
                        remote_revision = Entropy.get_remote_database_revision(repoid)
                        download_latest, upload_queue = Entropy.MirrorsService.calculate_database_sync_queues(repoid)

                        repo_data[repoid][crippled_uri]['database'] = {
                            'current_revision': current_revision,
                            'remote_revision': remote_revision,
                            'download_latest': download_latest,
                            'upload_queue': [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1],) for x in upload_queue]
                        }

                return True, repo_data

            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        data = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return data

    def run_entropy_mirror_updates(self, queue_id, repository_data):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")
        Entropy = self.SystemManagerExecutor.SystemInterface.Entropy

        def sync_remote_databases(repoid, pretend):

            rdb_status = Entropy.MirrorsService.get_remote_databases_status()
            Entropy.updateProgress(
                "%s:" % (_("Remote Entropy Database Repository Status"),),
                header = " * "
            )
            for myuri, myrev in rdb_status:
                Entropy.updateProgress("\t %s:\t %s" % (_("Host"),self.entropyTools.extractFTPHostFromUri(myuri),))
                Entropy.updateProgress("\t  * %s: %s" % (_("Database revision"),myrev,))
            local_revision = Entropy.get_local_database_revision(repoid)
            Entropy.updateProgress("\t  * %s: %s" % (_("Database local revision currently at"),local_revision,))
            if pretend:
                return 0,set(),set()

            errors, fine_uris, broken_uris = Entropy.MirrorsService.sync_databases(no_upload = False)
            remote_status = Entropy.MirrorsService.get_remote_databases_status(repoid)
            Entropy.updateProgress(" * %s: " % (_("Remote Entropy Database Repository Status"),))
            for myuri, myrev in remote_status:
                Entropy.updateProgress("\t %s:\t%s" % (_("Host"),Entropy.entropyTools.extractFTPHostFromUri(myuri),))
                Entropy.updateProgress("\t  * %s: %s" % (_("Database revision"),myrev,) )

            return errors, fine_uris, broken_uris


        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:

                repo_data = {}
                for repoid in repository_data:

                    # avoid __default__
                    if repoid == etpConst['clientserverrepoid']: continue

                    successfull_mirrors = set()
                    mirrors_errors = False
                    mirrors_tainted = False
                    broken_mirrors = set()
                    check_data = []

                    repo_data[repoid] = {
                        'mirrors_tainted': mirrors_tainted,
                        'mirrors_errors': mirrors_errors,
                        'successfull_mirrors': successfull_mirrors.copy(),
                        'broken_mirrors': broken_mirrors.copy(),
                        'check_data': check_data,
                        'db_errors': 0,
                        'db_fine': set(),
                        'db_broken': set(),
                    }

                    if repository_data[repoid]['pkg']:

                        mirrors_tainted, mirrors_errors, \
                        successfull_mirrors, broken_mirrors, \
                        check_data = Entropy.MirrorsService.sync_packages(
                            ask = False, pretend = repository_data[repoid]['pretend'],
                            packages_check = repository_data[repoid]['pkg_check'], repo = repoid)

                        repo_data[repoid]['mirrors_tainted'] = mirrors_tainted
                        repo_data[repoid]['mirrors_errors'] = mirrors_errors
                        repo_data[repoid]['successfull_mirrors'] = successfull_mirrors
                        repo_data[repoid]['broken_mirrors'] = broken_mirrors
                        repo_data[repoid]['check_data'] = check_data

                        if (not successfull_mirrors) and (not repository_data[repoid]['pretend']): continue

                    if (not mirrors_errors) and repository_data[repoid]['db']:

                        if mirrors_tainted and etpConst['rss-feed']:
                            commit_msg = repository_data[repoid]['commit_msg']
                            if not commit_msg: commit_msg = "Autodriven update"
                            Entropy.rssMessages['commitmessage'] = commit_msg

                        errors, fine, broken = sync_remote_databases(repoid, repository_data[repoid]['pretend'])
                        repo_data[repoid]['db_errors'] = errors
                        repo_data[repoid]['db_fine'] = fine.copy()
                        repo_data[repoid]['db_broken'] = broken.copy()
                        if errors: continue
                        Entropy.MirrorsService.lock_mirrors(lock = False, repo = repoid)
                        Entropy.MirrorsService.tidy_mirrors(
                            repo = repoid, ask = False,
                            pretend = repository_data[repoid]['pretend']
                        )

                return True, repo_data

            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        data = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return data

    def get_spm_glsa_data(self, queue_id, list_type):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        data = {}
        glsa_ids = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.list_glsa_packages(list_type)
        if not glsa_ids: return False,data
        for myid in glsa_ids:
            data[myid] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_glsa_id_information(myid)
        return True,data

    def get_notice_board(self, queue_id, repoid):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.read_notice_board(repo = repoid)
                if data == None:
                    return False,None
                return True,data
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        mydata = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return mydata

    def remove_notice_board_entries(self, queue_id, repoid, entry_ids):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                for entry_id in entry_ids:
                    data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.remove_from_notice_board(entry_id, repo = repoid)
                self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.upload_notice_board(repo = repoid)
                return True,data
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        mydata = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return mydata

    def add_notice_board_entry(self, queue_id, repoid, title, notice_text, link):

        queue_data, key = self.SystemManagerExecutor.SystemInterface.get_item_by_queue_id(queue_id)
        if queue_data == None:
            return False,'no item in queue'

        stdout_err = open(queue_data['stdout'],"a+")

        def myfunc():
            sys.stdout = stdout_err
            sys.stderr = stdout_err
            mystdin = self._get_stdin(queue_id)
            if mystdin: sys.stdin = os.fdopen(mystdin, 'rb')
            try:
                data = self.SystemManagerExecutor.SystemInterface.Entropy.MirrorsService.update_notice_board(title, notice_text, link = link, repo = repoid)
                return True,data
            except Exception, e:
                self.entropyTools.printTraceback()
                return False,unicode(e)
            finally:
                sys.stdout.write("\n### Done ###\n")
                sys.stdout.flush()
                sys.stdout = sys.__stdout__
                sys.stderr = sys.__stderr__
                sys.stdin = sys.__stdin__

        def write_pid(pid):
            self._set_processing_pid(queue_id, pid)

        mydata = self.entropyTools.spawnFunction(myfunc, write_pid_func = write_pid)
        stdout_err.close()
        return mydata

    def _get_stdin(self, queue_id):
        mystdin = None
        std_data = self.SystemManagerExecutor.SystemInterface.ManagerQueueStdInOut.get(queue_id)
        if std_data != None: mystdin = std_data[0]
        return mystdin

    def _file_updateProgress(self, f, *myargs, **mykwargs):

        f.flush()
        back = mykwargs.get("back")
        count = mykwargs.get("count")
        header = mykwargs.get("header")
        percent = mykwargs.get("percent")
        text = myargs[0].encode('utf-8')
        if not header: header = ''

        count_str = ""
        if count:
            if len(count) > 1:
                if percent:
                    count_str = " ("+str(round((float(count[0])/count[1])*100,1))+"%) "
                else:
                    count_str = " (%s/%s) " % (red(str(count[0])),blue(str(count[1])),)

        def is_last_newline(f):
            try:
                f.seek(-1,2)
                last = f.read(1)
                if last == "\n":
                    return True
            except IOError:
                pass
            return False

        if back:
            self.entropyTools.seek_till_newline(f)
            txt = header+count_str+text
        else:
            if not is_last_newline(f): f.write("\n")
            txt = header+count_str+text+"\n"
        f.write(txt)

        f.flush()

    # !!! duplicate
    def _get_entropy_pkginfo(self, dbconn, idpackage, repoid):
        data = {}
        try:
            data['atom'], data['name'], data['version'], data['versiontag'], \
            data['description'], data['category'], data['chost'], \
            data['cflags'], data['cxxflags'],data['homepage'], \
            data['license'], data['branch'], data['download'], \
            data['digest'], data['slot'], data['etpapi'], \
            data['datecreation'], data['size'], data['revision']  = dbconn.getBaseData(idpackage)
        except TypeError:
            return data
        data['injected'] = dbconn.isInjected(idpackage)
        data['repoid'] = repoid
        data['idpackage'] = idpackage
        return data

    def _get_spm_pkginfo(self, matched_atom, from_installed = False):
        data = {}
        data['atom'] = matched_atom
        data['key'] = self.entropyTools.dep_getkey(matched_atom)
        try:
            if from_installed:
                data['slot'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_installed_package_slot(matched_atom)
                portage_matched_atom = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_best_atom("%s:%s" % (data['key'],data['slot'],))
                # get installed package description
                data['available_atom'] = portage_matched_atom
                if portage_matched_atom:
                    data['use'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_useflags(portage_matched_atom)
                else:
                    # get use flags of the installed package
                    data['use'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_installed_package_useflags(matched_atom)
                data['description'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_installed_package_description(matched_atom)
            else:
                data['slot'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_slot(matched_atom)
                data['use'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_useflags(matched_atom)
                data['installed_atom'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_installed_atom("%s:%s" % (data['key'],data['slot'],))
                data['description'] = self.SystemManagerExecutor.SystemInterface.Entropy.SpmService.get_package_description(matched_atom)
        except KeyError:
            pass

        return data

class SystemManagerRepositoryCommands(SocketCommandsSkel):

    import entropyTools, dumpTools
    def __init__(self, HostInterface):

        SocketCommandsSkel.__init__(self, HostInterface, inst_name = "srvrepo")
        self.raw_commands = [
            'srvrepo:enable_uses_for_atoms',
            'srvrepo:disable_uses_for_atoms',
            'srvrepo:compile_atoms',
            'srvrepo:spm_remove_atoms',
            'srvrepo:run_custom_shell_command',
            'srvrepo:remove_entropy_packages',
            'srvrepo:search_entropy_packages',
            'srvrepo:run_entropy_database_updates',
            'srvrepo:run_entropy_mirror_updates',
            'srvrepo:add_notice_board_entry'
        ]
        self.valid_commands = {
            'srvrepo:sync_spm':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_sync_spm,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "spawn portage sync (emerge --sync)",
                'syntax': "<SESSION_ID> srvrepo:sync_spm",
                'from': unicode(self)
            },
            'srvrepo:compile_atoms':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_compile_atoms,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "compile specified atoms using Spm (Portage?)",
                'syntax': "<SESSION_ID> srvrepo:compile_atoms <xml string containing atoms and compile options>",
                'from': unicode(self)
            },
            'srvrepo:spm_remove_atoms':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_spm_remove_atoms,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "remove specified atoms using Spm (Portage?)",
                'syntax': "<SESSION_ID> srvrepo:spm_remove_atoms <xml string containing atoms and remove options>",
                'from': unicode(self)
            },
            'srvrepo:get_spm_categories_updates':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_spm_categories_updates,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "get SPM updates for the specified package categories",
                'syntax': "<SESSION_ID> srvrepo:get_spm_categories_updates <category 1> <category 2> <...>",
                'from': unicode(self)
            },
            'srvrepo:get_spm_categories_installed':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_spm_categories_installed,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "get SPM installed packages for the specified package categories",
                'syntax': "<SESSION_ID> srvrepo:get_spm_categories_installed <category 1> <category 2> <...>",
                'from': unicode(self)
            },
            'srvrepo:enable_uses_for_atoms':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_enable_uses_for_atoms,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "enable use flags for the specified atom",
                'syntax': "<SESSION_ID> srvrepo:enable_uses_for_atom <xml string containing atoms and use flags>",
                'from': unicode(self)
            },
            'srvrepo:disable_uses_for_atoms':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_disable_uses_for_atoms,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "enable use flags for the specified atom",
                'syntax': "<SESSION_ID> srvrepo:disable_uses_for_atom <xml string containing atoms and use flags>",
                'from': unicode(self)
            },
            'srvrepo:get_spm_atoms_info':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_spm_atoms_info,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "get info from SPM for the specified atoms",
                'syntax': "<SESSION_ID> srvrepo:get_spm_atoms_info <atom1> <atom2> <atom3>",
                'from': unicode(self)
            },
            'srvrepo:run_spm_info':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_spm_info,
                'args': ["cmd","authenticator"],
                'as_user': False,
                'desc': "run SPM info command",
                'syntax': "<SESSION_ID> srvrepo:run_spm_info",
                'from': unicode(self)
            },
            'srvrepo:run_custom_shell_command':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_custom_shell_command,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "run custom shell command",
                'syntax': "<SESSION_ID> srvrepo:run_custom_shell_command <shell command blah blah>",
                'from': unicode(self)
            },
            'srvrepo:get_spm_glsa_data':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_spm_glsa_data,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "get SPM security updates info",
                'syntax': "<SESSION_ID> srvrepo:get_spm_glsa_data <list_type string (affected,new,all)>",
                'from': unicode(self)
            },
            'srvrepo:get_available_repositories':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_available_repositories,
                'args': [],
                'as_user': False,
                'desc': "get information about available Entropy repositories",
                'syntax': "<SESSION_ID> srvrepo:get_available_repositories",
                'from': unicode(self)
            },
            'srvrepo:set_default_repository':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_set_default_repository,
                'args': ["myargs"],
                'as_user': False,
                'desc': "set a new default Entropy Server repository",
                'syntax': "<SESSION_ID> srvrepo:set_default_repository <repoid>",
                'from': unicode(self)
            },
            'srvrepo:get_available_entropy_packages':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_available_entropy_packages,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get available Entropy packages from the chosen repository id",
                'syntax': "<SESSION_ID> srvrepo:get_available_entropy_packages <repoid>",
                'from': unicode(self)
            },
            'srvrepo:get_entropy_idpackage_information':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_entropy_idpackage_information,
                'args': ["myargs"],
                'as_user': False,
                'desc': "get Entropy package metadata using its idpackage and repository id",
                'syntax': "<SESSION_ID> srvrepo:get_entropy_idpackage_information <idpackage> <repoid>",
                'from': unicode(self)
            },
            'srvrepo:remove_entropy_packages':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_entropy_packages,
                'args': ["myargs"],
                'as_user': False,
                'desc': "remove Entropy packages using their match -> (idpackage,repo)",
                'syntax': "<SESSION_ID> srvrepo:remove_entropy_packages idpackage:repoid,idpackage,repoid,...",
                'from': unicode(self)
            },
            'srvrepo:search_entropy_packages':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_search_entropy_packages,
                'args': ["myargs"],
                'as_user': False,
                'desc': "search Entropy packages using a defined search type, search string inside the specified repository",
                'syntax': "<SESSION_ID> srvrepo:search_entropy_packages <repoid> <search_type> <search string...>",
                'from': unicode(self)
            },
            'srvrepo:move_entropy_packages_to_repository':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_move_entropy_packages_to_repository,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "move or copy Entropy packages from a repository to another",
                'syntax': "<SESSION_ID> srvrepo:move_entropy_packages_to_repository <from_repo> <to_repo> <do_copy (True: copy, False: move)> <idpackages...>",
                'from': unicode(self)
            },
            'srvrepo:scan_entropy_packages_database_changes':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_scan_entropy_packages_database_changes,
                'args': ["cmd","authenticator"],
                'as_user': False,
                'desc': "scan Spm package changes to retrieve a list of action that should be run on the repositories",
                'syntax': "<SESSION_ID> srvrepo:scan_entropy_packages_database_changes",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_database_updates':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_database_updates,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "run Entropy database updates",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_database_updates <to_add: atom:counter:repoid,atom:counter:repoid,...> <to_remove: idpackage:repoid,idpackage:repoid,...> <to_inject: idpackage:repoid,idpackage:repoid,...>",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_dependency_test':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_dependency_test,
                'args': ["cmd","authenticator"],
                'as_user': False,
                'desc': "run Entropy dependency test",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_dependency_test",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_library_test':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_library_test,
                'args': ["cmd","authenticator"],
                'as_user': False,
                'desc': "run Entropy dependency test",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_library_test",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_treeupdates':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_treeupdates,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "run Entropy database treeupdates",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_treeupdates <repoid>",
                'from': unicode(self)
            },
            'srvrepo:scan_entropy_mirror_updates':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_scan_entropy_mirror_updates,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "scan mirror updates for the specified repository identifiers",
                'syntax': "<SESSION_ID> srvrepo:scan_entropy_mirror_updates <repoid 1> <repoid 2> <...>",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_mirror_updates':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_mirror_updates,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "run mirror updates for the provided repositories",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_mirror_updates <xml data, properly formatted>",
                'from': unicode(self)
            },
            'srvrepo:run_entropy_checksum_test':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_run_entropy_checksum_test,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "run Entropy packages checksum verification tool",
                'syntax': "<SESSION_ID> srvrepo:run_entropy_checksum_test <repoid> <mode>",
                'from': unicode(self)
            },
            'srvrepo:get_notice_board':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_get_notice_board,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "get repository notice board",
                'syntax': "<SESSION_ID> srvrepo:get_notice_board <repoid>",
                'from': unicode(self)
            },
            'srvrepo:remove_notice_board_entries':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_remove_notice_board_entries,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "remove notice board entries",
                'syntax': "<SESSION_ID> srvrepo:remove_notice_board_entries <repoid> <entry_id1> <entry_id2> <...>",
                'from': unicode(self)
            },
            'srvrepo:add_notice_board_entry':    {
                'auth': True,
                'built_in': False,
                'cb': self.docmd_add_notice_board_entry,
                'args': ["cmd","myargs","authenticator"],
                'as_user': False,
                'desc': "remove notice board entry",
                'syntax': "<SESSION_ID> srvrepo:add_notice_board_entry <xml formatted data>",
                'from': unicode(self)
            },
        }

    def docmd_sync_spm(self, cmd, myargs, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'sync_spm', [], {}, False, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_compile_atoms(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'

        xml_string = ' '.join(myargs)

        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)

        if not ( mydict.has_key('atoms') and mydict.has_key('pretend') and \
                 mydict.has_key('oneshot') and  mydict.has_key('verbose') and \
                 mydict.has_key('fetchonly') and  mydict.has_key('buildonly') and \
                 mydict.has_key('nodeps') and \
                 mydict.has_key('nocolor') and  mydict.has_key('custom_use') and \
                 mydict.has_key('ldflags') and  mydict.has_key('cflags') ):
            return None,'wrong dict arguments, xml must have 10 items with attr value' + \
                        ' -> atoms, pretend, oneshot, verbose, nocolor, fetchonly, ' + \
                        'buildonly, nodeps, custom_use, ldflags, cflags'

        atoms = mydict.get('atoms')
        if atoms: atoms = atoms.split()
        pretend = mydict.get('pretend')
        oneshot = mydict.get('oneshot')
        verbose = mydict.get('verbose')
        nocolor = mydict.get('nocolor')
        fetchonly = mydict.get('fetchonly')
        buildonly = mydict.get('buildonly')
        nodeps = mydict.get('nodeps')
        custom_use = mydict.get('custom_use')
        ldflags = mydict.get('ldflags')
        cflags = mydict.get('cflags')

        if pretend == "1": pretend = True
        else: pretend = False
        if oneshot == "1": oneshot = True
        else: oneshot = False
        if verbose == "1": verbose = True
        else: verbose = False
        if nocolor == "1": nocolor = True
        else: nocolor = False
        if fetchonly == "1": fetchonly = True
        else: fetchonly = False
        if buildonly == "1": buildonly = True
        else: buildonly = False
        if nodeps == "1": nodeps = True
        else: nodeps = False

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        add_dict = {
            'pretend': pretend,
            'oneshot': oneshot,
            'verbose': verbose,
            'nocolor': nocolor,
            'fetchonly': fetchonly,
            'buildonly': buildonly,
            'nodeps': nodeps,
            'custom_use': custom_use,
            'ldflags': ldflags,
            'cflags': cflags,
        }

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'compile_atoms', [atoms[:]], add_dict.copy(), False, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_spm_remove_atoms(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        xml_string = ' '.join(myargs)

        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)

        if not ( mydict.has_key('atoms') and mydict.has_key('pretend') and \
                 mydict.has_key('verbose') and mydict.has_key('nocolor') ):
            return None,'wrong dict arguments, xml must have 4 items with attr value -> atoms, pretend, verbose, nocolor'

        atoms = mydict.get('atoms')
        if atoms: atoms = atoms.split()
        pretend = mydict.get('pretend')
        verbose = mydict.get('verbose')
        nocolor = mydict.get('nocolor')

        if pretend == "1": pretend = True
        else: pretend = False
        if verbose == "1": verbose = True
        else: verbose = False
        if nocolor == "1": nocolor = True
        else: nocolor = False

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        add_dict = {
            'pretend': pretend,
            'verbose': verbose,
            'nocolor': nocolor,
        }

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'spm_remove_atoms', [atoms[:]], add_dict.copy(), False, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_get_spm_categories_updates(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_categories_updates', [myargs], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_get_spm_categories_installed(self, cmd, myargs, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_categories_installed', [myargs], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_enable_uses_for_atoms(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        xml_string = ' '.join(myargs)
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('atoms') and mydict.has_key('useflags')):
            return None,'wrong dict arguments, xml must have 2 items with attr value -> atoms, useflags'

        atoms = mydict.get('atoms')
        useflags = mydict.get('useflags')
        if atoms: atoms = atoms.split()
        if useflags: useflags = useflags.split()

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'enable_uses_for_atoms', [atoms,useflags], {}, True, True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_disable_uses_for_atoms(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        xml_string = ' '.join(myargs)
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('atoms') and mydict.has_key('useflags')):
            return None,'wrong dict arguments, xml must have 2 items with attr value -> atoms, useflags'

        atoms = mydict.get('atoms')
        useflags = mydict.get('useflags')
        if atoms: atoms = atoms.split()
        if useflags: useflags = useflags.split()

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'disable_uses_for_atoms', [atoms,useflags], {}, True, True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_get_spm_atoms_info(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_atoms_info', [myargs], {}, True, True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_spm_info(self, cmd, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_spm_info', [], {}, True, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_custom_shell_command(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')
        command = ' '.join(myargs)

        queue_id = self.HostInterface.add_to_queue(cmd, command, uid, gid, 'run_custom_shell_command', [command], {}, True, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_get_spm_glsa_data(self, cmd, myargs, authenticator):
        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_spm_glsa_data', [myargs[0]], {}, True, True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_get_available_repositories(self):
        data = {}
        data['available'] = self.HostInterface.Entropy.get_available_repositories()
        if etpConst['clientserverrepoid'] in data['available']:
            data['available'].pop(etpConst['clientserverrepoid'])
        data['community_mode'] = self.HostInterface.Entropy.community_repo
        data['current'] = self.HostInterface.Entropy.default_repository
        data['branches'] = etpConst['branches']
        data['branch'] = etpConst['branch']
        return True, data

    def docmd_set_default_repository(self, myargs):

        if not myargs:
            return False,'wrong arguments'
        repoid = myargs[0]
        if repoid not in self.HostInterface.Entropy.get_available_repositories():
            return False,'repository id not available'

        status = True
        msg = 'ok'
        try:
            self.HostInterface.Entropy.switch_default_repository(repoid, save = True, handle_uninitialized = False)
        except Exception, e:
            status = False
            msg = unicode(e)
        return status, msg

    def docmd_get_available_entropy_packages(self, myargs):

        if not myargs:
            return False,'wrong arguments'
        repoid = myargs[0]
        if repoid not in self.HostInterface.Entropy.get_available_repositories():
            return False,'repository id not available'

        dbconn = self.HostInterface.Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
        idpackages = dbconn.listAllIdpackages(order_by = 'atom')
        package_data = []
        package_data = {
            'ordered_idpackages': idpackages,
            'data': {},
        }
        for idpackage in idpackages:

            data = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
            if not data: continue
            package_data['data'][idpackage] = data.copy()
        dbconn.closeDB()
        return True,package_data

    def docmd_get_entropy_idpackage_information(self, myargs):

        if len(myargs) < 2:
            return False,'wrong arguments'
        idpackage = myargs[0]
        repoid = myargs[1]

        dbconn = self.HostInterface.Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)
        package_data = dbconn.getPackageData(idpackage, trigger_unicode = True)
        dbconn.closeDB()
        return True,package_data

    def docmd_remove_entropy_packages(self, myargs):
        if not myargs:
            return False,'wrong arguments'
        string = myargs[0].split(",")
        matched_atoms = []
        try:
            for item in string:
                mysplit = item.split(":")
                matched_atoms.append((int(mysplit[0]),mysplit[1],))
        except:
            return False,'cannot eval() string correctly'

        repo_data = {}
        for idpackage,repoid in matched_atoms:
            if not repo_data.has_key(repoid):
                repo_data[repoid] = []
            repo_data[repoid].append(idpackage)

        status = True
        msg = 'ok'
        try:
            for repoid in repo_data:
                self.HostInterface.Entropy.remove_packages(repo_data[repoid],repo = repoid)
        except Exception, e:
            status = False
            msg = unicode(e)

        return status, msg

    def docmd_move_entropy_packages_to_repository(self, cmd, myargs, authenticator):

        if len(myargs) < 4:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        from_repo = myargs[0]
        to_repo = myargs[1]
        do_copy = myargs[2]
        idpackages = myargs[3:]

        queue_id = self.HostInterface.add_to_queue(
            cmd, ' '.join([str(x) for x in myargs]),
            uid, gid, 'move_entropy_packages_to_repository',
            [from_repo,to_repo,idpackages,do_copy], {}, False, True,
            interactive = True
        )
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_scan_entropy_packages_database_changes(self, cmd, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'scan_entropy_packages_database_changes', [], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_dependency_test(self, cmd, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_entropy_dependency_test', [], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_library_test(self, cmd, authenticator):

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, '', uid, gid, 'run_entropy_library_test', [], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_checksum_test(self, cmd, myargs, authenticator):
        if len(myargs) < 2:
            return False,'wrong arguments'
        repoid = myargs[0]
        mode = myargs[1]

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'run_entropy_checksum_test', [repoid,mode], {}, True, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_treeupdates(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'run_entropy_treeupdates', [myargs[0]], {}, False, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_scan_entropy_mirror_updates(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'scan_entropy_mirror_updates', [myargs], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_mirror_updates(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'

        serialized_string = '\n'.join(myargs)
        try:
            mydict = self.dumpTools.unserialize_string(serialized_string)
        except Exception, e:
            return False,'cannot parse data: %s' % (e,)

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, '<raw data>', uid, gid, 'run_entropy_mirror_updates', [mydict], {}, False, False, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_run_entropy_database_updates(self, cmd, myargs, authenticator):

        to_add = {}
        to_remove = []
        to_inject = []
        to_add_string = ''
        to_remove_string = ''
        to_inject_string = ''
        if myargs:
            to_add_string = myargs[0].split(",")
        if len(myargs) > 1:
            to_remove_string = myargs[1].split(",")
        if len(myargs) > 2:
            to_inject_string = myargs[2].split(",")
        try:

            for item in to_add_string:
                atom, counter, repoid = item.split(":")
                if not to_add.has_key(repoid):
                    to_add[repoid] = []
                to_add[repoid].append(atom)

            for item in to_remove_string:
                idpackage, repoid = item.split(":")
                to_remove.append((idpackage, repoid,))

            for item in to_inject_string:
                idpackage, repoid = item.split(":")
                to_inject.append((idpackage, repoid,))

        except Exception, e:
            return False,'cannot run database updates properly: %s' % (e,)

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(
            cmd, ' '.join(myargs), uid, gid,
            'run_entropy_database_updates', [to_add,to_remove,to_inject],
            {}, False, True, interactive = True
        )
        if queue_id < 0: return False, queue_id
        return True, queue_id


    def docmd_search_entropy_packages(self, myargs):
        if len(myargs) < 3:
            return False,'wrong arguments'

        repoid = myargs[0]
        search_type = myargs[1]
        search_string = ' '.join(myargs[2:])
        avail_search_types = ['atom','needed','depends','tag','file','description']

        if search_type not in avail_search_types:
            return False, 'available search types: %s' % (avail_search_types,)

        search_results = {
            'ordered_idpackages': set(),
            'data': {},
        }

        dbconn = self.HostInterface.Entropy.openServerDatabase(repo = repoid, just_reading = True, warnings = False, do_cache = False)

        if search_type == "atom":

            mysearchlist = search_string.split()
            for mystring in mysearchlist:
                results = dbconn.searchPackages(mystring)
                for atom, idpackage, branch in results:
                    data = self._get_entropy_pkginfo(dbconn, idpackage, repoid)
                    if not data: continue
                    search_results['ordered_idpackages'].add(idpackage)
                    search_results['data'][idpackage] = data.copy()

        elif search_type == "needed":

            mysearchlist = search_string.split()
            for mystring in mysearchlist:
                idpackages = dbconn.searchNeeded(mystring, like = True)
                for idpackage in idpackages:
                    search_results['ordered_idpackages'].add(idpackage)
                    search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)

        elif search_type == "depends":

            mysearchlist = search_string.split()
            for mystring in mysearchlist:
                m_idpackage, m_result = dbconn.atomMatch(mystring)
                if m_idpackage == -1: continue
                idpackages = dbconn.retrieveDepends(m_idpackage)
                for idpackage in idpackages:
                    search_results['ordered_idpackages'].add(idpackage)
                    search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)

        elif search_type == "tag":

            mysearchlist = search_string.split()
            for mystring in mysearchlist:
                idpackages = dbconn.searchTaggedPackages(mystring)
                for idpackage in idpackages:
                    search_results['ordered_idpackages'].add(idpackage)
                    search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)

        elif search_type == "file":
            # belong

            like = False
            if search_string.find("*") != -1:
                search_string.replace("*","%")
                like = True
            idpackages = dbconn.searchBelongs(search_string, like)
            for idpackage in idpackages:
                search_results['ordered_idpackages'].add(idpackage)
                search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)

        elif search_type == "description":

            results = dbconn.searchPackagesByDescription(search_string)
            for atom, idpackage in results:
                search_results['ordered_idpackages'].add(idpackage)
                search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)

        elif search_type == "eclass":

            mysearchlist = search_string.split()
            for eclass in mysearchlist:
                idpackages = dbconn.searchEclassedPackages(eclass)
                for idpackage in idpackages:
                    search_results['ordered_idpackages'].add(idpackage)
                    search_results['data'][idpackage] = self._get_entropy_pkginfo(dbconn, idpackage, repoid)


        dbconn.closeDB()
        return True, search_results

    def docmd_get_notice_board(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'
        repoid = myargs[0]

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(cmd, ' '.join(myargs), uid, gid, 'get_notice_board', [repoid], {}, True, True, interactive = True)
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_remove_notice_board_entries(self, cmd, myargs, authenticator):

        if len(myargs) < 2:
            return False,'wrong arguments'
        repoid = myargs[0]
        entry_ids = myargs[1:]

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(
            cmd, ' '.join([unicode(x) for x in myargs]), uid, gid,
            'remove_notice_board_entries', [repoid,entry_ids], {}, True, False, interactive = True
        )
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def docmd_add_notice_board_entry(self, cmd, myargs, authenticator):

        if not myargs:
            return False,'wrong arguments'

        xml_string = ' '.join(myargs)
        try:
            mydict = self.entropyTools.dict_from_xml(xml_string)
        except Exception, e:
            return None,"error: %s" % (e,)
        if not (mydict.has_key('repoid') and mydict.has_key('title') and \
                mydict.has_key('notice_text') and mydict.has_key('link')):
            return None,'wrong dict arguments, xml must have 4 items with attr value -> repoid, title, notice_text, link'

        repoid = mydict.get('repoid')
        title = mydict.get('title')
        notice_text = mydict.get('notice_text')
        link = mydict.get('link')

        status, userdata, err_str = authenticator.docmd_userdata()
        uid = userdata.get('uid')
        gid = userdata.get('gid')

        queue_id = self.HostInterface.add_to_queue(
            cmd, ' '.join(myargs), uid, gid,
            'add_notice_board_entry', [repoid,title,notice_text,link], {}, True, False, interactive = True
        )
        if queue_id < 0: return False, queue_id
        return True, queue_id

    def _get_entropy_pkginfo(self, dbconn, idpackage, repoid):
        data = {}
        try:
            data['atom'], data['name'], data['version'], data['versiontag'], \
            data['description'], data['category'], data['chost'], \
            data['cflags'], data['cxxflags'],data['homepage'], \
            data['license'], data['branch'], data['download'], \
            data['digest'], data['slot'], data['etpapi'], \
            data['datecreation'], data['size'], data['revision']  = dbconn.getBaseData(idpackage)
        except TypeError:
            return data
        data['injected'] = dbconn.isInjected(idpackage)
        data['repoid'] = repoid
        data['idpackage'] = idpackage
        return data


class SystemManagerServerInterface(SocketHostInterface):

    class SystemCommands(SocketCommandsSkel):

        import entropyTools
        def __init__(self, HostInterface):

            SocketCommandsSkel.__init__(self, HostInterface, inst_name = "systemsrv")
            self.raw_commands = [
                'systemsrv:add_to_pinboard',
                'systemsrv:write_to_running_command_pipe'
            ]

            self.valid_commands = {
                'systemsrv:get_queue':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_get_queue,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "get current queue",
                    'syntax': "<SESSION_ID> systemsrv:get_queue",
                    'from': unicode(self),
                },
                'systemsrv:get_queue_item_by_id':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_get_queue_item_by_id,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "get current queue item through its queue id",
                    'syntax': "<SESSION_ID> systemsrv:get_queue_item_by_id <queue_id>",
                    'from': unicode(self),
                },
                'systemsrv:get_queue_id_stdout':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_get_queue_id_stdout,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "get current queue item stdout/stderr output",
                    'syntax': "<SESSION_ID> systemsrv:get_queue_id_stdout <queue_id> <how many bytes (from tail)>",
                    'from': unicode(self),
                },
                'systemsrv:get_queue_id_result':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_get_queue_id_result,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "get current queue item result output",
                    'syntax': "<SESSION_ID> systemsrv:get_queue_id_result <queue_id>",
                    'from': unicode(self),
                },
                'systemsrv:remove_queue_ids':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_remove_queue_ids,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "remove queue items using their queue ids",
                    'syntax': "<SESSION_ID> systemsrv:remove_queue_ids <queue_id 1> <queue_id 2> <...>",
                    'from': unicode(self),
                },
                'systemsrv:pause_queue':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_pause_queue,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "toggle queue pause (understood?)",
                    'syntax': "<SESSION_ID> systemsrv:pause_queue <True/False>",
                    'from': unicode(self),
                },
                'systemsrv:kill_processing_queue_id':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_kill_processing_queue_id,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "kill a running process using its queue id",
                    'syntax': "<SESSION_ID> systemsrv:kill_processing_queue_id <queue_id>",
                    'from': unicode(self),
                },
                'systemsrv:swap_items_in_queue':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_swap_items_in_queue,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "swap items in queue to change their order",
                    'syntax': "<SESSION_ID> systemsrv:swap_items_in_queue <queue_id1> <queue_id1>",
                    'from': unicode(self),
                },
                'systemsrv:get_pinboard_data':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_get_pinboard_data,
                    'args': [],
                    'as_user': False,
                    'desc': "get pinboard content",
                    'syntax': "<SESSION_ID> systemsrv:get_pinboard_data",
                    'from': unicode(self),
                },
                'systemsrv:add_to_pinboard':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_add_to_pinboard,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "add item to pinboard",
                    'syntax': "<SESSION_ID> systemsrv:add_to_pinboard <xml string containing pinboard note and extended text>",
                    'from': unicode(self),
                },
                'systemsrv:remove_from_pinboard':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_remove_from_pinboard,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "remove item from pinboard",
                    'syntax': "<SESSION_ID> systemsrv:remove_from_pinboard <pinboard identifier 1> <pinboard identifier 2> <...>",
                    'from': unicode(self),
                },
                'systemsrv:set_pinboard_items_done':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_set_pinboard_items_done,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "set pinboard items status using their pinboard identifiers",
                    'syntax': "<SESSION_ID> systemsrv:set_pinboard_items_done <pinboard identifier 1> <pinboard identifier 2> <...> <status (True/False)>",
                    'from': unicode(self),
                },
                'systemsrv:write_to_running_command_pipe':    {
                    'auth': True,
                    'built_in': False,
                    'cb': self.docmd_write_to_running_command_pipe,
                    'args': ["myargs"],
                    'as_user': False,
                    'desc': "write text to stdin of a running command",
                    'syntax': "<SESSION_ID> systemsrv:write_to_running_command_pipe <queue_id> <write stdout (True/False)> <txt ...>",
                    'from': unicode(self),
                },
            }

        def docmd_get_queue(self, myargs):

            self.HostInterface.queue_lock_acquire()
            try:
                myqueue = copy.deepcopy(self.HostInterface.ManagerQueue)

                extended = False
                if myargs:
                    extended = myargs[0]

                if not extended:
                    for key in self.HostInterface.done_queue_keys:
                        for queue_id in myqueue.get(key):
                            item = myqueue[key].get(queue_id)
                            if not item.has_key('extended_result'):
                                continue
                            item['extended_result'] = None

                return True, myqueue
            finally:
                self.HostInterface.queue_lock_release()

        def docmd_get_queue_item_by_id(self, myargs):

            if not myargs:
                return False,'wrong arguments'
            queue_id = myargs[0]
            try:
                queue_id = int(queue_id)
            except ValueError:
                return False,'wrong argument: queue_id'

            item, key = self.HostInterface.get_item_by_queue_id(queue_id)
            if item == None:
                return False,'wrong queue id'
            item = item.copy()

            return True,item

        def docmd_get_queue_id_stdout(self, myargs):

            if len(myargs) < 1:
                return False,'wrong arguments'
            queue_id = myargs[0]
            bytes_from_tail = myargs[1]

            try:
                queue_id = int(queue_id)
            except ValueError:
                return False,'wrong argument: queue_id'
            try:
                bytes_from_tail = int(bytes_from_tail)
            except ValueError:
                return False,'wrong argument: lines from tail'

            item, key = self.HostInterface.get_item_by_queue_id(queue_id)
            if item == None:
                return False,'wrong queue id'
            item = item.copy()

            file_path = item['stdout']
            if not (os.path.isfile(file_path) and os.access(file_path,os.R_OK)):
                text = ''
            else:
                f = open(file_path,"r")
                f.seek(0,2)
                tell_me = f.tell()
                if bytes_from_tail < 1:
                    bytes_from_tail = tell_me
                if bytes_from_tail > tell_me:
                    bytes_from_tail = tell_me
                f.seek(-1*bytes_from_tail,2)
                text = f.read()
                f.close()

            return True,text

        def docmd_get_queue_id_result(self, myargs):

            if not myargs:
                return False,'wrong arguments'
            queue_id = myargs[0]
            try:
                queue_id = int(queue_id)
            except ValueError:
                return False,'wrong argument: queue_id'

            item, key = self.HostInterface.get_item_by_queue_id(queue_id)
            if item == None:
                return False,'wrong queue id'

            if key not in self.HostInterface.done_queue_keys:
                return False,'process not completed yet'

            if not item.has_key('result'):
                return False,'result not available'

            item = item.copy()

            ext_result = None
            if item.has_key('extended_result'):
                ext_result = item['extended_result']

            return True,(item['result'],ext_result,)

        def docmd_remove_queue_ids(self, myargs):

            if not myargs:
                return False,'wrong arguments'

            valid_queue_ids = set()
            for queue_id in myargs:
                item, key = self.HostInterface.get_item_by_queue_id(queue_id)
                if (item != None) and (key in self.HostInterface.removable_queue_keys):
                    valid_queue_ids.add(queue_id)

            if not valid_queue_ids:
                return False,'no valid queue ids'

            # remove
            self.HostInterface.remove_from_queue(valid_queue_ids)

            return True,'ok'

        def docmd_pause_queue(self, myargs):

            if not myargs:
                return False,'wrong arguments'

            do = myargs[0]
            if do:
                self.HostInterface.pause_queue()
            else:
                self.HostInterface.play_queue()

            return self.HostInterface.ManagerQueue['pause'],'ok'

        def docmd_kill_processing_queue_id(self, myargs):

            if not myargs:
                return False,'wrong arguments'

            queue_id = myargs[0]
            self.HostInterface.kill_processing_queue_id(queue_id)
            return True,'ok'

        def docmd_swap_items_in_queue(self, myargs):

            if len(myargs) < 2:
                return False,'wrong arguments'

            queue_id1 = myargs[0]
            queue_id2 = myargs[1]
            status = self.HostInterface.swap_items_in_queue(queue_id1,queue_id2)
            if status:
                return True,'ok'
            return False,'not done'

        def docmd_get_pinboard_data(self):
            data = self.HostInterface.get_pinboard_data()
            return True,data.copy()

        def docmd_add_to_pinboard(self, myargs):

            if not myargs:
                return False,'wrong arguments'

            xml_string = ' '.join(myargs)
            try:
                mydict = self.entropyTools.dict_from_xml(xml_string)
            except Exception, e:
                return None,"error: %s" % (e,)

            if not (mydict.has_key('note') and mydict.has_key('extended_text')):
                return None,'wrong dict arguments, xml must have 2 items with attr value -> note, extended_text'
            note = mydict.get('note')
            extended_text = mydict.get('extended_text')

            self.HostInterface.add_to_pinboard(note, extended_text)
            return True,'ok'

        def docmd_remove_from_pinboard(self, myargs):

            if not myargs:
                return False,'wrong arguments'

            for pinboard_id in myargs:
                try:
                    pinboard_id = int(pinboard_id)
                except ValueError:
                    continue
                self.HostInterface.remove_from_pinboard(pinboard_id)
            return True,'ok'

        def docmd_set_pinboard_items_done(self, myargs):

            if len(myargs) < 2:
                return False,'wrong arguments'

            status = myargs[-1]
            pinboard_ids = myargs[:-1]

            for pinboard_id in pinboard_ids:
                try:
                    pinboard_id = int(pinboard_id)
                except ValueError:
                    continue
                self.HostInterface.set_pinboard_item_status(pinboard_id, status)
            return True,'ok'

        def docmd_write_to_running_command_pipe(self, myargs):

            if len(myargs) < 2:
                return False, 'wrong arguments'

            try:
                queue_id = int(myargs[0])
            except ValueError:
                return False,'invalid queue id'

            try:
                write_stdout = bool(myargs[1])
            except ValueError:
                write_stdout = False

            txt = ' '.join(myargs[2:])+'\n'
            item, key = self.HostInterface.get_item_by_queue_id(queue_id)
            if key not in self.HostInterface.processing_queue_keys:
                return False,'not running'
            mypipe = self.HostInterface.ManagerQueueStdInOut.get(queue_id)
            if mypipe == None:
                return False,'pipe not available'
            try:
                w_fd = mypipe[1]
            except (IndexError, ValueError,):
                return False,'pipe vanished'
            if not isinstance(w_fd,int):
                return False,'stdout fd not an int'

            if write_stdout:
                stdout = open(item['stdout'],"a+")
            try:
                os.write(w_fd,txt)
                if write_stdout:
                    stdout.write(txt)
            except OSError, e:
                return False,'OSError: %s' % (e,)
            except IOError, e:
                return False,'IOError: %s' % (e,)
            finally:
                if write_stdout:
                    stdout.flush()
                    stdout.close()

            return True,'ok'


    class FakeServiceInterface:
        def __init__(self, *args, **kwargs):
            pass

    class BuiltInSystemManagerExecutorCommands:

        def __init__(self, SystemManagerExecutorInstance, *args, **kwargs):
            self.SystemManagerExecutor = SystemManagerExecutorInstance
            self.available_commands = {
                'hello_world': {
                    'func': self.hello_world,
                    'args': 0,
                }
            }

        def hello_world(self):
            rc = subprocess.call('echo hello world', shell = True)
            return True,rc


    queue_file = 'system_manager_queue'
    pinboard_file = "system_manager_pinboard"
    STDOUT_STORAGE_DIR = os.path.join(etpConst['dumpstoragedir'],'system_manager_stdout')
    def __init__(self, EntropyInterface, do_ssl = False, stdout_logging = True, fork_requests = False, entropy_interface_kwargs = {}, **kwargs):

        #nocolor()
        self.queue_loaded = False
        import entropyTools, dumpTools
        self.entropyTools, self.dumpTools = entropyTools, dumpTools
        from datetime import datetime
        self.datetime = datetime

        self.setup_stdout_storage_dir()

        if not kwargs.has_key('external_cmd_classes'):
            kwargs['external_cmd_classes'] = []
        kwargs['external_cmd_classes'].insert(0,self.SystemCommands)

        self.Entropy = EntropyInterface(**entropy_interface_kwargs)
        self.Text = TextInterface()
        self.SystemExecutor = SystemManagerExecutorInterface(self, self.Entropy)

        self.ExecutorCommandClasses = [(self.BuiltInSystemManagerExecutorCommands,[],{},)]
        self.ExecutorCommandInstances = []
        if kwargs.has_key('external_executor_cmd_classes'):
            self.ExecutorCommandClasses += kwargs.pop('external_executor_cmd_classes')
        self.handle_executor_command_classes_initialization()

        self.QueueProcessor = None
        self.QueueProcessorParallel = None
        self.QueueLock = thread.allocate_lock()
        self.PinboardLock = thread.allocate_lock()
        self.do_ssl = do_ssl
        self.ServiceInterface = EntropyInterface

        self.PinboardData = {}
        self.load_pinboard()

        self.done_queue_keys = ['processed','errored']
        self.removable_queue_keys = ['processed','errored','queue']
        self.processing_queue_keys = ['processing']
        self.dict_queue_keys = ['queue','processing','processed','errored']
        self.ManagerQueueStdInOut = {}
        self.ManagerQueue = {
            'queue': {},
            'queue_order': [],
            'processing': {},
            'processing_order': [],
            'processed': {},
            'processed_order': [],
            'errored' : {},
            'errored_order': [],
            'pause': True
        }
        self.load_queue()
        self.queue_loaded = True
        if self.ManagerQueue['processing'] or self.ManagerQueue['processing_order']:
            self.ManagerQueue['processing'].clear()
            del self.ManagerQueue['processing_order'][:]
            self.store_queue()

        SocketHostInterface.__init__(
            self,
            self.FakeServiceInterface, #self.ServiceInterface,
            sock_output = self.Text,
            ssl = do_ssl,
            **kwargs
        )
        self.stdout_logging = stdout_logging
        self.fork_requests = fork_requests
        self.load_queue_processors()
        # here we can put anything that must be loaded before the queue processor execution
        self.play_queue()

    def __del__(self):
        if hasattr(self,'queue_loaded'):
            if self.queue_loaded:
                self.store_queue()

    def handle_executor_command_classes_initialization(self):
        for myclass, args, kwargs in self.ExecutorCommandClasses:
            myintf = myclass(self.SystemExecutor, *args,**kwargs)
            if hasattr(myintf,'available_commands'):
                self.SystemExecutor.register(myintf.available_commands)
                self.ExecutorCommandInstances.append(myintf)
            else:
                del myintf

    def setup_stdout_storage_dir(self):
        if os.path.isfile(self.STDOUT_STORAGE_DIR) or os.path.islink(self.STDOUT_STORAGE_DIR):
            os.remove(self.STDOUT_STORAGE_DIR)
        if not os.path.isdir(self.STDOUT_STORAGE_DIR):
            os.makedirs(self.STDOUT_STORAGE_DIR,0775)
            if etpConst['entropygid'] != None:
                const_setup_perms(self.STDOUT_STORAGE_DIR,etpConst['entropygid'])

    def pinboard_lock_acquire(self):
        self.PinboardLock.acquire()

    def pinboard_lock_release(self):
        self.PinboardLock.release()

    def load_pinboard(self):
        obj = self.get_stored_pinboard()
        if isinstance(obj,dict):
            self.PinboardData = obj
            return True
        return False

    def get_stored_pinboard(self):
        return self.dumpTools.loadobj(self.pinboard_file)

    def store_pinboard(self):
        self.dumpTools.dumpobj(self.pinboard_file, self.PinboardData)

    def add_to_pinboard(self, note, extended_text):
        self.pinboard_lock_acquire()
        try:
            mydata = {
                'note': note,
                'extended_text': extended_text,
                'ts': self.get_ts(),
                'done': False,
            }
            pinboard_id = self.get_pinboard_id()
            self.PinboardData[pinboard_id] = mydata
            self.store_pinboard()
        finally:
            self.pinboard_lock_release()

    def remove_from_pinboard(self, pinboard_id):
        self.pinboard_lock_acquire()
        try:
            if self.PinboardData.has_key(pinboard_id):
                self.PinboardData.pop(pinboard_id)
                self.store_pinboard()
                return True
            return False
        finally:
            self.pinboard_lock_release()

    def set_pinboard_item_status(self, pinboard_id, status):
        self.pinboard_lock_acquire()
        try:
            if self.PinboardData.has_key(pinboard_id):
                self.PinboardData[pinboard_id]['done'] = status
                self.store_pinboard()
                return True
            return False
        finally:
            self.pinboard_lock_release()

    def get_pinboard_id(self):
        numbers = self.PinboardData.keys()
        if numbers:
            number = max(numbers)+1
        else:
            number = 1
        return number

    def get_pinboard_data(self):
        self.pinboard_lock_acquire()
        try:
            return self.PinboardData.copy()
        finally:
            self.pinboard_lock_release()

    def load_queue_processors(self):
        self.QueueProcessor = self.entropyTools.parallelTask(self.queue_processor, False)
        self.QueueProcessor.start()
        self.QueueProcessorParallel = self.entropyTools.parallelTask(self.queue_processor, True)
        self.QueueProcessorParallel.start()

    def queue_lock_acquire(self):
        self.QueueLock.acquire()

    def queue_lock_release(self):
        self.QueueLock.release()

    def get_stored_queue(self):
        return self.dumpTools.loadobj(self.queue_file)

    def load_queue(self):
        obj = self.get_stored_queue()
        if isinstance(obj,dict):
            for key in self.ManagerQueue.keys():
                if not obj.has_key(key):
                    return False
            self.ManagerQueue = obj
            return True
        return False

    def store_queue(self):
        self.dumpTools.dumpobj(self.queue_file, self.ManagerQueue)

    def get_ts(self):
        return self.datetime.fromtimestamp(time.time())

    def swap_items_in_queue(self, queue_id1, queue_id2):
        self.queue_lock_acquire()
        try:
            item1, key1 = self.get_item_by_queue_id(queue_id1)
            item2, key2 = self.get_item_by_queue_id(queue_id2)
            if key1 != key2:
                return False
            t_item = item1.copy()
            item1.clear()
            item1.update(item2)
            item2.clear()
            item2.update(t_item)
            # fix the _order
            queue_id1_idx = self.ManagerQueue[key1+"_order"].index(queue_id1)
            queue_id2_idx = self.ManagerQueue[key2+"_order"].index(queue_id2)
            self.ManagerQueue[key1+"_order"][queue_id1_idx] = queue_id2
            self.ManagerQueue[key2+"_order"][queue_id2_idx] = queue_id1
            self.store_queue()
        finally:
            self.queue_lock_release()
        return True


    def add_to_queue(self, command_name, command_text, user_id, group_id, function, args, kwargs, do_parallel, extended_result, interactive = False):

        if function not in self.SystemExecutor.available_commands:
            return -1

        self.queue_lock_acquire()
        try:
            queue_id = self.generate_unique_queue_id()
            if interactive:
                self.ManagerQueueStdInOut[queue_id] = os.pipe()
            myqueue_dict = {
                'queue_id': queue_id,
                'command_name': command_name,
                'command_desc': self.valid_commands[command_name]['desc'],
                'command_text': command_text,
                'call': function,
                'args': copy.deepcopy(args),
                'kwargs': copy.deepcopy(kwargs),
                'user_id': user_id,
                'group_id': group_id,
                'stdout': self.assign_unique_stdout_file(queue_id),
                'queue_ts': "%s" % (self.get_ts(),),
                'kill': False,
                'processing_pid': None,
                'do_parallel': do_parallel,
                'interactive': False,
            }
            if extended_result:
                myqueue_dict['extended_result'] = None
            self.ManagerQueue['queue'][queue_id] = myqueue_dict
            self.ManagerQueue['queue_order'].append(queue_id)
            self.store_queue()
        finally:
            self.queue_lock_release()
        return queue_id

    def remove_from_queue(self, queue_ids):
        self.queue_lock_acquire()
        removed = False
        try:
            for key in self.ManagerQueue:
                if key not in self.dict_queue_keys:
                    continue
                for queue_id in queue_ids:
                    item = None
                    try:
                        item = self.ManagerQueue[key].pop(queue_id)
                    except KeyError:
                        continue
                    if item:
                        self.flush_item(item, queue_id)
                        if queue_id in self.ManagerQueue[key+"_order"]:
                            self.ManagerQueue[key+"_order"].remove(queue_id)
                    removed = True
            if removed:
                self.store_queue()
        finally:
            self.queue_lock_release()
        return removed

    def kill_processing_queue_id(self, queue_id):
        self.queue_lock_acquire()
        try:
            item, key = self.get_item_by_queue_id(queue_id)
            if key in self.processing_queue_keys:
                item['kill'] = True
        finally:
            self.queue_lock_release()

    def pause_queue(self):
        self.queue_lock_acquire()
        try:
            self.ManagerQueue['pause'] = True
        finally:
            self.queue_lock_release()

    def play_queue(self):
        self.queue_lock_acquire()
        try:
            self.ManagerQueue['pause'] = False
        finally:
            self.queue_lock_release()

    def flush_item(self, item, queue_id):
        if not isinstance(item,dict):
            return False
        if item.has_key('stdout'):
            stdout = item['stdout']
            if (os.path.isfile(stdout) and os.access(stdout,os.W_OK)):
                os.remove(stdout)
        if item.has_key('interactive'):
            if item['interactive'] and (queue_id in self.ManagerQueueStdInOut):
                stdin, stdout = self.ManagerQueueStdInOut.pop(queue_id)
                os.close(stdin)
                os.close(stdout)
        return True

    def assign_unique_stdout_file(self, queue_id):
        stdout = os.path.join(self.STDOUT_STORAGE_DIR,"%d.%s" % (queue_id,"stdout",))
        if os.path.isfile(stdout):
            os.remove(stdout)
        count = 0
        orig_stdout = stdout
        while os.path.lexists(stdout):
            count += 1
            stdout = "%s.%d" % (orig_stdout,count,)
        return stdout

    def generate_unique_queue_id(self):
        current_ids = set()
        for key in self.ManagerQueue:
            if not key.endswith("_order"):
                continue
            current_ids |= set(self.ManagerQueue[key])
        while 1:
            queue_id = abs(hash(os.urandom(20)))
            if queue_id not in current_ids:
                return queue_id

    def get_item_by_queue_id(self, queue_id):
        for key in self.dict_queue_keys:
            item = self.ManagerQueue[key].get(queue_id)
            if item != None:
                return item, key
        return None, None

    def _pop_item_from_queue_by_parallel(self, parallel):
        self.queue_lock_acquire()
        try:
            for idx in range(len(self.ManagerQueue['queue_order'])):
                queue_id = self.ManagerQueue['queue_order'][idx]
                if parallel == self.ManagerQueue['queue'][queue_id]['do_parallel']:
                    return self.ManagerQueue['queue'].pop(queue_id), self.ManagerQueue['queue_order'].pop(idx)
        finally:
            self.queue_lock_release()
        return None, None

    def queue_processor(self, parallel_mode, fork_data = None):

        def do_store_and_free():
            self.store_queue()
            self.queue_lock_release()

        def wait_and_takeover():
            self.queue_lock_acquire()

        def copy_obj(obj):
            if isinstance(obj,(dict,set,)):
                return obj.copy()
            elif isinstance(obj,(list,tuple,)):
                return obj[:]
            return obj

        while 1:

            try:

                if not fork_data:
                    if self.ManagerQueue['pause']:
                        time.sleep(0.1)
                        continue

                    if not self.ManagerQueue['queue_order']:
                        time.sleep(0.1)
                        continue

                    command_data, queue_id = self._pop_item_from_queue_by_parallel(parallel_mode)
                    if not command_data:
                        time.sleep(0.5)
                        continue

                    wait_and_takeover()
                    command_data['processing_ts'] = "%s" % (self.get_ts(),)
                    self.ManagerQueue['processing'][queue_id] = command_data
                    self.ManagerQueue['processing_order'].append(queue_id)
                    do_store_and_free()

                else:
                    command_data, queue_id = fork_data

                try:
                    if parallel_mode:
                        t = self.entropyTools.parallelTask(self.queue_processor, False, fork_data = (command_data, queue_id,))
                        t.start()
                        continue
                    else:
                        done, result = self.SystemExecutor.execute_task(command_data)
                except Exception, e:
                    self.queue_lock_release()
                    self.entropyTools.printTraceback()
                    done = False
                    result = (False, unicode(e),)

                wait_and_takeover()
                if command_data.has_key('extended_result') and done:
                    try:
                        command_data['result'], command_data['extended_result'] = copy_obj(result)
                    except TypeError:
                        done = False
                        command_data['result'] = 'wrong tuple split from queue processor (1)'
                        command_data['extended_result'] = None
                else:
                    command_data['result'] = copy_obj(result)

                if not done:
                    try:
                        self.ManagerQueue['processing'].pop(queue_id)
                    except KeyError:
                        pass
                    if queue_id in self.ManagerQueue['processing_order']:
                        self.ManagerQueue['processing_order'].remove(queue_id)
                    command_data['errored_ts'] = "%s" % (self.get_ts(),)
                    self.ManagerQueue['errored'][queue_id] = command_data
                    self.ManagerQueue['errored_order'].append(queue_id)
                    do_store_and_free()
                    if fork_data:
                        break
                    continue

                try:
                    done, cmd_result = result
                except TypeError:
                    done = False
                    command_data['result'] = 'wrong tuple split from queue processor (2)'
                if not done:
                    try:
                        self.ManagerQueue['processing'].pop(queue_id)
                    except KeyError:
                        pass
                    if queue_id in self.ManagerQueue['processing_order']:
                        self.ManagerQueue['processing_order'].remove(queue_id)
                    command_data['errored_ts'] = "%s" % (self.get_ts(),)
                    self.ManagerQueue['errored'][queue_id] = command_data
                    self.ManagerQueue['errored_order'].append(queue_id)
                    do_store_and_free()
                    if fork_data:
                        break
                    continue

                try:
                    self.ManagerQueue['processing'].pop(queue_id)
                except KeyError:
                    pass
                if queue_id in self.ManagerQueue['processing_order']:
                    self.ManagerQueue['processing_order'].remove(queue_id)
                command_data['processed_ts'] = "%s" % (self.get_ts(),)
                self.ManagerQueue['processed'][queue_id] = command_data
                self.ManagerQueue['processed_order'].append(queue_id)
                do_store_and_free()

                if fork_data:
                    break

            except:
                try:
                    self.queue_lock_release()
                except thread.error:
                    pass
                raise


    def killall(self):
        SocketHostInterface.killall(self)
        if self.QueueProcessor != None:
            self.QueueProcessor.kill()
        if self.QueueProcessorParallel != None:
            self.QueueProcessorParallel.kill()


class SystemManagerClientCommands(EntropySocketClientCommands):

    import entropyTools
    def __init__(self, *args, **kwargs):
        EntropySocketClientCommands.__init__(self, *args, **kwargs)

    def service_login(self, username, password, session_id):

        cmd = "%s %s %s %s %s" % (
            session_id,
            'login',
            username,
            'plain',
            password,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_queue(self, session_id, extended):

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:get_queue',
            extended,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_queue_item_by_id(self, session_id, queue_id):

        cmd = "%s %s %d" % (
            session_id,
            'systemsrv:get_queue_item_by_id',
            queue_id,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_queue_id_stdout(self, session_id, queue_id, last_bytes):

        cmd = "%s %s %d %d" % (
            session_id,
            'systemsrv:get_queue_id_stdout',
            queue_id,
            last_bytes,
        )

        # enable zlib compression
        compression = self.set_gzip_compression(session_id, True)

        rc = self.do_generic_handler(cmd, session_id, compression = compression)

        # disable compression
        self.set_gzip_compression(session_id, False)

        return rc

    def get_queue_id_result(self, session_id, queue_id):

        cmd = "%s %s %d" % (
            session_id,
            'systemsrv:get_queue_id_result',
            queue_id,
        )
        return self.do_generic_handler(cmd, session_id)

    def remove_queue_ids(self, session_id, queue_ids):

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:remove_queue_ids',
            ' '.join([str(x) for x in queue_ids]),
        )
        return self.do_generic_handler(cmd, session_id)

    def pause_queue(self, session_id, do_pause):

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:pause_queue',
            do_pause,
        )
        return self.do_generic_handler(cmd, session_id)

    def kill_processing_queue_id(self, session_id, queue_id):

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:kill_processing_queue_id',
            queue_id,
        )
        return self.do_generic_handler(cmd, session_id)

    def swap_items_in_queue(self, session_id, queue_id1, queue_id2):

        cmd = "%s %s %d %d" % (
            session_id,
            'systemsrv:swap_items_in_queue',
            queue_id1,
            queue_id2,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_pinboard_data(self, session_id):
        cmd = "%s %s" % (
            session_id,
            'systemsrv:get_pinboard_data',
        )
        return self.do_generic_handler(cmd, session_id)

    def add_to_pinboard(self, session_id, note, extended_text):

        mydict = {
            'note': note,
            'extended_text': extended_text,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:add_to_pinboard',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def remove_from_pinboard(self, session_id, pinboard_ids):

        cmd = "%s %s %s" % (
            session_id,
            'systemsrv:remove_from_pinboard',
            ' '.join([str(x) for x in pinboard_ids]),
        )
        return self.do_generic_handler(cmd, session_id)

    def set_pinboard_items_done(self, session_id, pinboard_ids, status):
        cmd = "%s %s %s %s" % (
            session_id,
            'systemsrv:set_pinboard_items_done',
            ' '.join([str(x) for x in pinboard_ids]),
            status,
        )
        return self.do_generic_handler(cmd, session_id)

    def write_to_running_command_pipe(self, session_id, queue_id, write_to_stdout, txt):
        cmd = "%s %s %s %s %s" % (
            session_id,
            'systemsrv:write_to_running_command_pipe',
            queue_id,
            write_to_stdout,
            txt,
        )
        return self.do_generic_handler(cmd, session_id)

class SystemManagerRepositoryClientCommands(SystemManagerClientCommands):

    import dumpTools

    def sync_spm(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:sync_spm',
        )
        return self.do_generic_handler(cmd, session_id)

    def compile_atoms(self, session_id, atoms, pretend = False, oneshot = False, verbose = False, nocolor = True, fetchonly = False, buildonly = False, nodeps = False, custom_use = '', ldflags = '', cflags = ''):

        s_pretend = "0"
        s_oneshot = "0"
        s_verbose = "0"
        s_nocolor = "0"
        s_fetchonly = "0"
        s_buildonly = "0"
        s_nodeps = "0"
        if pretend: s_pretend = "1"
        if oneshot: s_oneshot = "1"
        if verbose: s_verbose = "1"
        if nocolor: s_nocolor = "1"
        if fetchonly: s_fetchonly = "1"
        if buildonly: s_buildonly = "1"
        if nodeps: s_nodeps = "1"
        mydict = {
            'atoms': ' '.join(atoms),
            'pretend': s_pretend,
            'oneshot': s_oneshot,
            'verbose': s_verbose,
            'nocolor': s_nocolor,
            'fetchonly': s_fetchonly,
            'buildonly': s_buildonly,
            'nodeps': s_nodeps,
            'custom_use': custom_use,
            'ldflags': ldflags,
            'cflags': cflags,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:compile_atoms',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def spm_remove_atoms(self, session_id, atoms, pretend = False, verbose = False, nocolor = True):

        s_pretend = "0"
        s_verbose = "0"
        s_nocolor = "0"
        if pretend: s_pretend = "1"
        if verbose: s_verbose = "1"
        if nocolor: s_nocolor = "1"
        mydict = {
            'atoms': ' '.join(atoms),
            'pretend': s_pretend,
            'verbose': s_verbose,
            'nocolor': s_nocolor,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:spm_remove_atoms',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_spm_categories_updates(self, session_id, categories):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_spm_categories_updates',
            ' '.join(categories),
        )
        return self.do_generic_handler(cmd, session_id)

    def get_spm_categories_installed(self, session_id, categories):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_spm_categories_installed',
            ' '.join(categories),
        )
        return self.do_generic_handler(cmd, session_id)

    def enable_uses_for_atoms(self, session_id, atoms, useflags):

        mydict = {
            'atoms': ' '.join(atoms),
            'useflags': ' '.join(useflags),
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:enable_uses_for_atoms',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def disable_uses_for_atoms(self, session_id, atoms, useflags):

        mydict = {
            'atoms': ' '.join(atoms),
            'useflags': ' '.join(useflags),
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:disable_uses_for_atoms',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_spm_atoms_info(self, session_id, atoms):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_spm_atoms_info',
            ' '.join(atoms),
        )
        return self.do_generic_handler(cmd, session_id)

    def run_spm_info(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:run_spm_info',
        )
        return self.do_generic_handler(cmd, session_id)

    def run_custom_shell_command(self, session_id, command):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:run_custom_shell_command',
            command,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_spm_glsa_data(self, session_id, list_type):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_spm_glsa_data',
            list_type,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_available_repositories(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:get_available_repositories',
        )
        return self.do_generic_handler(cmd, session_id)

    def set_default_repository(self, session_id, repoid):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:set_default_repository',
            repoid,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_available_entropy_packages(self, session_id, repoid):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_available_entropy_packages',
            repoid,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_entropy_idpackage_information(self, session_id, idpackage, repoid):

        cmd = "%s %s %d %s" % (
            session_id,
            'srvrepo:get_entropy_idpackage_information',
            idpackage,
            repoid,
        )
        return self.do_generic_handler(cmd, session_id)

    def remove_entropy_packages(self, session_id, matched_atoms):
        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:remove_entropy_packages',
            ','.join(["%s:%s" % (str(x[0]),str(x[1]),) for x in matched_atoms]), # 1:repoid,2:repoid
        )
        return self.do_generic_handler(cmd, session_id)

    def search_entropy_packages(self, session_id, search_type, search_string, repoid):

        cmd = "%s %s %s %s %s" % (
            session_id,
            'srvrepo:search_entropy_packages',
            repoid,
            search_type,
            search_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def move_entropy_packages_to_repository(self, session_id, idpackages, from_repo, to_repo, do_copy):

        cmd = "%s %s %s %s %s %s" % (
            session_id,
            'srvrepo:move_entropy_packages_to_repository',
            from_repo,
            to_repo,
            do_copy,
            ' '.join([str(x) for x in idpackages]),
        )
        return self.do_generic_handler(cmd, session_id)

    def scan_entropy_packages_database_changes(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:scan_entropy_packages_database_changes',
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_database_updates(self, session_id, to_add, to_remove, to_inject):

        cmd = "%s %s %s %s %s" % (
            session_id,
            'srvrepo:run_entropy_database_updates',
            ','.join(["%s:%s:%s" % (str(x[0]),str(x[1]),str(x[2]),) for x in to_add]),
            ','.join(["%s:%s" % (str(x[0]),str(x[1]),) for x in to_remove]),
            ','.join(["%s:%s" % (str(x[0]),str(x[1]),) for x in to_inject]),
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_dependency_test(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:run_entropy_dependency_test',
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_library_test(self, session_id):

        cmd = "%s %s" % (
            session_id,
            'srvrepo:run_entropy_library_test',
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_treeupdates(self, session_id, repoid):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:run_entropy_treeupdates',
            repoid,
        )
        return self.do_generic_handler(cmd, session_id)

    def scan_entropy_mirror_updates(self, session_id, repositories):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:scan_entropy_mirror_updates',
            ' '.join(repositories),
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_mirror_updates(self, session_id, repository_data):

        serialized_string = self.dumpTools.serialize_string(repository_data)
        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:run_entropy_mirror_updates',
            serialized_string,
        )
        return self.do_generic_handler(cmd, session_id)

    def run_entropy_checksum_test(self, session_id, repoid, mode):

        cmd = "%s %s %s %s" % (
            session_id,
            'srvrepo:run_entropy_checksum_test',
            repoid,
            mode,
        )
        return self.do_generic_handler(cmd, session_id)

    def get_notice_board(self, session_id, repoid):

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:get_notice_board',
            repoid,
        )
        return self.do_generic_handler(cmd, session_id)

    def remove_notice_board_entries(self, session_id, repoid, entry_ids):

        cmd = "%s %s %s %s" % (
            session_id,
            'srvrepo:remove_notice_board_entries',
            repoid,
             ' '.join([str(x) for x in entry_ids]),
        )
        return self.do_generic_handler(cmd, session_id)

    def add_notice_board_entry(self, session_id, repoid, title, notice_text, link):

        mydict = {
            'repoid': repoid,
            'title': title,
            'notice_text': notice_text,
            'link': link,
        }
        xml_string = self.entropyTools.xml_from_dict(mydict)

        cmd = "%s %s %s" % (
            session_id,
            'srvrepo:add_notice_board_entry',
            xml_string,
        )
        return self.do_generic_handler(cmd, session_id)

class SystemManagerMethodsInterface:

    def __init__(self, SystemManagerClientInstance):
        self.Manager = SystemManagerClientInstance
        self.available_commands = {
            'get_available_commands': {
                'desc': _("Get a list of remotely available commands"),
                'params': [],
                'call': self.get_available_commands,
                'private': True,
            },
            'get_queue': {
                'desc': _("Get current queue content"),
                'params': [
                    ('extended',bool,_('Extended results'),False,)
                ],
                'call': self.get_queue,
                'private': True,
            },
            'get_queue_item_by_id': {
                'desc': _("Get queue item using its queue unique identifier"),
                'params': [('queue_id',int,_('Queue Identifier'),True,)],
                'call': self.get_queue_item_by_id,
                'private': True,
            },
            'get_queue_id_stdout': {
                'desc': _("Get queue stdout/stderr using its queue unique identifier"),
                'params': [('queue_id',int,_('Queue Identifier'),True,)],
                'call': self.get_queue_id_stdout,
                'private': True,
            },
            'get_queue_id_stdout': {
                'desc': _("Get queued command result using its queue unique identifier"),
                'params': [('queue_id',int,_('Queue Identifier'),True,)],
                'call': self.get_queue_id_result,
                'private': True,
            },
            'remove_queue_ids': {
                'desc': _("Remove queued commands using their queue unique identifiers"),
                'params': [('queue_ids',list,_('Queue Identifiers'),True,)],
                'call': self.remove_queue_ids,
                'private': True,
            },
            'pause_queue': {
                'desc': _("Toggle queue pause (True/False)"),
                'params': [('do_pause',bool,_('Pause or not'),True,)],
                'call': self.pause_queue,
                'private': True,
            },
            'kill_processing_queue_id': {
                'desc': _("Kill a running process through its queue id"),
                'params': [('queue_id',int,_('Queue Identifier'),True,)],
                'call': self.kill_processing_queue_id,
                'private': True,
            },
            'swap_items_in_queue': {
                'desc': _("Swap items in queue using their queue ids"),
                'params': [
                    ('queue_id1',int,_('Queue Identifier'),True,),
                    ('queue_id2',int,_('Queue Identifier'),True,)
                ],
                'call': self.swap_items_in_queue,
                'private': True,
            },
            'get_pinboard_data': {
                'desc': _("Get pinboard content"),
                'params': [],
                'call': self.get_pinboard_data,
                'private': True,
            },
            'add_to_pinboard': {
                'desc': _("Add item to pinboard"),
                'params': [
                    ('note',basestring,_('Note'),True,),
                    ('extended_text',basestring,_('Extended text'),True,)
                ],
                'call': self.add_to_pinboard,
                'private': True,
            },
            'remove_from_pinboard': {
                'desc': _("Remove item from pinboard"),
                'params': [('pinboard_ids',list,_('Pinboard identifiers'),True,)],
                'call': self.remove_from_pinboard,
                'private': True,
            },
            'set_pinboard_items_done': {
                'desc': _("Set pinboard items status (done/not done)"),
                'params': [
                    ('pinboard_ids',list,_('Pinboard identifiers'),True,),
                    ('done_status',bool,_('Done status'),True,),
                ],
                'call': self.set_pinboard_items_done,
                'private': True,
            },
            'write_to_running_command_pipe': {
                'desc': _("Write to a remote running command stdin"),
                'params': [
                    ('queue_id',int,_('Queue Identifier'),True,),
                    ('write_to_stdout',bool,_('Write to stdout?'),True,),
                    ('txt',basestring,_('Text'),True,),
                ],
                'call': self.write_to_running_command_pipe,
                'private': True,
            },
        }

    def get_available_commands(self):
        return self.Manager.do_cmd(False, "available_commands", [], {})

    def get_queue(self, extended = False):
        return self.Manager.do_cmd(True, "get_queue", [extended], {})

    def get_queue_item_by_id(self, queue_id):
        return self.Manager.do_cmd(True, "get_queue_item_by_id", [queue_id], {})

    def get_queue_id_stdout(self, queue_id, last_bytes = 0):
        return self.Manager.do_cmd(True, "get_queue_id_stdout", [queue_id, last_bytes], {})

    def get_queue_id_result(self, queue_id):
        return self.Manager.do_cmd(True, "get_queue_id_result", [queue_id], {})

    def remove_queue_ids(self, queue_ids):
        return self.Manager.do_cmd(True, "remove_queue_ids", [queue_ids], {})

    def pause_queue(self, do_queue):
        return self.Manager.do_cmd(True, "pause_queue", [do_queue], {})

    def kill_processing_queue_id(self, queue_id):
        return self.Manager.do_cmd(True, "kill_processing_queue_id", [queue_id], {})

    def swap_items_in_queue(self, queue_id1, queue_id2):
        return self.Manager.do_cmd(True, "swap_items_in_queue", [queue_id1,queue_id2], {})

    def get_pinboard_data(self):
        return self.Manager.do_cmd(True, "get_pinboard_data", [], {})

    def add_to_pinboard(self, note, extended_text):
        return self.Manager.do_cmd(True, "add_to_pinboard", [note,extended_text], {})

    def remove_from_pinboard(self, pinboard_ids):
        return self.Manager.do_cmd(True, "remove_from_pinboard", [pinboard_ids], {})

    def set_pinboard_items_done(self, pinboard_ids, done_status):
        return self.Manager.do_cmd(True, "set_pinboard_items_done", [pinboard_ids,done_status], {})

    def write_to_running_command_pipe(self, queue_id, write_to_stdout, txt):
        return self.Manager.do_cmd(True, "write_to_running_command_pipe", [queue_id, write_to_stdout, txt], {})

class SystemManagerRepositoryMethodsInterface(SystemManagerMethodsInterface):

    def __init__(self, *args, **kwargs):
        SystemManagerMethodsInterface.__init__(self, *args, **kwargs)
        self.available_commands.update({
            'sync_spm': {
                'desc': _("Update Spm Repository (emerge --sync)"),
                'params': [],
                'call': self.sync_spm,
                'private': False,
            },
            'compile_atoms': {
                'desc': _("Compile specified atoms with specified parameters"),
                'params': [
                    ('atoms',list,_('Atoms'),True,),
                    ('pretend',bool,_('Pretend'),False,),
                    ('oneshot',bool,_('Oneshot'),False,),
                    ('verbose',bool,_('Verbose'),False,),
                    ('nocolor',bool,_('No color'),False,),
                    ('fetchonly',bool,_('Fetch only'),False,),
                    ('buildonly',bool,_('Build only'),False,),
                    ('nodeps',bool,_('No dependencies'),False,),
                    ('custom_use',basestring,_('Custom USE'),False,),
                    ('ldflags',basestring,_('Custom LDFLAGS'),False,),
                    ('cflags',basestring,_('Custom CFLAGS'),False,),
                ],
                'call': self.compile_atoms,
                'private': False,
            },
            'spm_remove_atoms': {
                'desc': _("Remove specified atoms with specified parameters"),
                'params': [
                    ('atoms',list,_('Atoms'),True,),
                    ('pretend',bool,_('Pretend'),False,),
                    ('verbose',bool,_('Verbose'),False,),
                    ('nocolor',bool,_('No color'),False,),
                ],
                'call': self.spm_remove_atoms,
                'private': False,
            },
            'get_spm_categories_updates': {
                'desc': _("Get SPM updates for the specified categories"),
                'params': [('categories',list,_('Categories'),True,)],
                'call': self.get_spm_categories_updates,
                'private': False,
            },
            'get_spm_categories_installed': {
                'desc': _("Get SPM installed packages for the specified categories"),
                'params': [('categories',list,_('Categories'),True,)],
                'call': self.get_spm_categories_installed,
                'private': False,
            },
            'enable_uses_for_atoms': {
                'desc': _("Enable USE flags for the specified atoms"),
                'params': [
                    ('atoms',list,_('Atoms'),True,),
                    ('useflags',list,_('USE flags'),True,)
                ],
                'call': self.enable_uses_for_atoms,
                'private': False,
            },
            'disable_uses_for_atoms': {
                'desc': _("Disable USE flags for the specified atoms"),
                'params': [
                    ('atoms',list,_('Atoms'),True,),
                    ('useflags',list,_('USE flags'),True,)
                ],
                'call': self.disable_uses_for_atoms,
                'private': False,
            },
            'get_spm_atoms_info': {
                'desc': _("Get info for the specified atoms"),
                'params': [('atoms',list,_('Atoms'),True,)],
                'call': self.get_spm_atoms_info,
                'private': False,
            },
            'run_spm_info': {
                'desc': _("Run SPM info command"),
                'params': [],
                'call': self.run_spm_info,
                'private': False,
            },
            'run_custom_shell_command': {
                'desc': _("Run custom shell command"),
                'params': [
                    ('command',basestring,_('Command'),True,)
                ],
                'call': self.run_custom_shell_command,
                'private': False,
            },
            'get_spm_glsa_data': {
                'desc': _("Get Spm security updates information"),
                'params': [
                    ('list_type',basestring,_('List type (affected,new,all)'),True,)
                ],
                'call': self.get_spm_glsa_data,
                'private': False,
            },
            'get_available_repositories': {
                'desc': _("Get information about available Entropy repositories"),
                'params': [],
                'call': self.get_available_repositories,
                'private': False,
            },
            'set_default_repository': {
                'desc': _("Set default Entropy Server repository"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,)
                ],
                'call': self.set_default_repository,
                'private': False,
            },
            'get_available_entropy_packages': {
                'desc': _("Get available packages inside the specified repository"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,)
                ],
                'call': self.get_available_entropy_packages,
                'private': False,
            },
            'get_entropy_idpackage_information': {
                'desc': _("Get idpackage metadata using its idpackage in the specified repository"),
                'params': [
                    ('idpackage',int,_('Package Identifier'),True,),
                    ('repoid',basestring,_('Repository Identifier'),True,)
                ],
                'call': self.get_entropy_idpackage_information,
                'private': False,
            },
            'remove_entropy_packages': {
                'desc': _("Remove the specified Entropy package matches (idpackage,repoid)"),
                'params': [
                    ('matched_atoms',list,_('Matched atoms'),True,)
                ],
                'call': self.remove_entropy_packages,
                'private': False,
            },
            'search_entropy_packages': {
                'desc': _("Search Entropy packages using a defined set of search types in the specified repository"),
                'params': [
                    ('search_type',basestring,_('Search type'),True,),
                    ('search_string',basestring,_('Search string'),True,),
                    ('repoid',basestring,_('Repository Identifier'),True,)
                ],
                'call': self.search_entropy_packages,
                'private': False,
            },
            'move_entropy_packages_to_repository': {
                'desc': _("Move or copy a package from a repository to another"),
                'params': [
                    ('idpackages',list,_('Package identifiers'),True,),
                    ('from_repo',basestring,_('From repository'),True,),
                    ('to_repo',basestring,_('To repository'),True,),
                    ('do_copy',bool,_('Copy instead of move?'),False,)
                ],
                'call': self.search_entropy_packages,
                'private': False,
            },
            'scan_entropy_packages_database_changes': {
                'desc': _("Scan Spm package changes and retrieve a list of action that should be run on the repositories"),
                'params': [],
                'call': self.scan_entropy_packages_database_changes,
                'private': False,
            },
            'run_entropy_database_updates': {
                'desc': _("Run Entropy database updates"),
                'params': [
                    ('to_add',list,_('Matches to add from Spm'),True,),
                    ('to_remove',list,_('Matches to remove from repository database'),True,),
                    ('to_inject',list,_('Matches to inject on repository database'),True,),
                ],
                'call': self.run_entropy_database_updates,
                'private': False,
            },
            'run_entropy_dependency_test': {
                'desc': _("Run Entropy dependency test"),
                'params': [],
                'call': self.run_entropy_dependency_test,
                'private': False,
            },
            'run_entropy_library_test': {
                'desc': _("Run Entropy library test"),
                'params': [],
                'call': self.run_entropy_library_test,
                'private': False,
            },
            'run_entropy_treeupdates': {
                'desc': _("Run Entropy tree updates"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,),
                ],
                'call': self.run_entropy_treeupdates,
                'private': False,
            },
            'scan_entropy_mirror_updates': {
                'desc': _("Scan for Mirror updates and retrieve a list of action that should be run"),
                'params': [
                    ('repositories',list,_('list of repository identifiers'),True,),
                ],
                'call': self.scan_entropy_mirror_updates,
                'private': False,
            },
            'run_entropy_mirror_updates': {
                'desc': _("Run Mirror updates for the provided repositories and its data"),
                'params': [
                    ('repository_data',dict,_('composed repository data'),True,),
                ],
                'call': self.run_entropy_mirror_updates,
                'private': False,
            },
            'run_entropy_checksum_test': {
                'desc': _("Run Entropy packages digest verification test"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,),
                    ('mode',basestring,_('Check mode'),False,),
                ],
                'call': self.run_entropy_mirror_updates,
                'private': False,
            },
            'get_notice_board': {
                'desc': _("Get repository notice board"),
                'params': [('repoid',basestring,_('Repository Identifier'),True,),],
                'call': self.get_notice_board,
                'private': False,
            },
            'remove_notice_board_entries': {
                'desc': _("Remove notice board entry"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,),
                    ('entry_ids',list,_('Entry Identifiers'),True,),
                ],
                'call': self.remove_notice_board_entries,
                'private': False,
            },
            'add_notice_board_entry': {
                'desc': _("Add notice board entry"),
                'params': [
                    ('repoid',basestring,_('Repository Identifier'),True,),
                    ('title',basestring,_('Title'),True,),
                    ('notice_text',basestring,_('Text'),True,),
                    ('link',basestring,_('Notice link'),True,),
                ],
                'call': self.add_notice_board_entry,
                'private': False,
            },
        })

    def sync_spm(self):
        return self.Manager.do_cmd(True, "sync_spm", [], {})

    def compile_atoms(self, atoms, pretend = False, oneshot = False, verbose = True, nocolor = True, fetchonly = False, buildonly = False, nodeps = False, custom_use = '', ldflags = '', cflags = ''):
        return self.Manager.do_cmd(
            True,
            "compile_atoms",
            [atoms],
            {
                'pretend': pretend,
                'oneshot': oneshot,
                'verbose': verbose,
                'nocolor': nocolor,
                'fetchonly': fetchonly,
                'buildonly': buildonly,
                'nodeps': nodeps,
                'custom_use': custom_use,
                'ldflags': ldflags,
                'cflags': cflags,
            }
        )

    def spm_remove_atoms(self, atoms, pretend = True, verbose = True, nocolor = True):
        return self.Manager.do_cmd(
            True,
            "spm_remove_atoms",
            [atoms],
            {
                'pretend': pretend,
                'verbose': verbose,
                'nocolor': nocolor,
            }
        )

    def get_spm_categories_updates(self, categories):
        return self.Manager.do_cmd(True, "get_spm_categories_updates", [categories], {})

    def get_spm_categories_installed(self, categories):
        return self.Manager.do_cmd(True, "get_spm_categories_installed", [categories], {})

    def enable_uses_for_atoms(self, atoms, useflags):
        return self.Manager.do_cmd(True, "enable_uses_for_atoms", [atoms,useflags], {})

    def disable_uses_for_atoms(self, atoms, useflags):
        return self.Manager.do_cmd(True, "disable_uses_for_atoms", [atoms,useflags], {})

    def get_spm_atoms_info(self, atoms):
        return self.Manager.do_cmd(True, "get_spm_atoms_info", [atoms], {})

    def run_spm_info(self):
        return self.Manager.do_cmd(True, "run_spm_info", [], {})

    def run_custom_shell_command(self, command):
        return self.Manager.do_cmd(True, "run_custom_shell_command", [command], {})

    def get_spm_glsa_data(self, list_type = "affected"):
        return self.Manager.do_cmd(True, "get_spm_glsa_data", [list_type], {})

    def get_available_repositories(self):
        return self.Manager.do_cmd(True, "get_available_repositories", [], {})

    def set_default_repository(self, repoid):
        return self.Manager.do_cmd(True, "set_default_repository", [repoid], {})

    def get_available_entropy_packages(self, repoid):
        return self.Manager.do_cmd(True, "get_available_entropy_packages", [repoid], {})

    def get_entropy_idpackage_information(self, idpackage, repoid):
        return self.Manager.do_cmd(True, "get_entropy_idpackage_information", [idpackage,repoid], {})

    def remove_entropy_packages(self, matched_atoms):
        return self.Manager.do_cmd(True, "remove_entropy_packages", [matched_atoms], {})

    def search_entropy_packages(self, search_type, search_string, repoid):
        return self.Manager.do_cmd(True, "search_entropy_packages", [search_type,search_string,repoid], {})

    def move_entropy_packages_to_repository(self, idpackages, from_repo, to_repo, do_copy = False):
        return self.Manager.do_cmd(True, "move_entropy_packages_to_repository", [idpackages,from_repo,to_repo, do_copy], {})

    def scan_entropy_packages_database_changes(self):
        return self.Manager.do_cmd(True, "scan_entropy_packages_database_changes", [], {})

    def run_entropy_database_updates(self, to_add, to_remove, to_inject):
        return self.Manager.do_cmd(True, "run_entropy_database_updates", [to_add,to_remove,to_inject], {})

    def run_entropy_dependency_test(self):
        return self.Manager.do_cmd(True, "run_entropy_dependency_test", [], {})

    def run_entropy_library_test(self):
        return self.Manager.do_cmd(True, "run_entropy_library_test", [], {})

    def run_entropy_treeupdates(self, repoid):
        return self.Manager.do_cmd(True, "run_entropy_treeupdates", [repoid], {})

    def scan_entropy_mirror_updates(self, repositories):
        return self.Manager.do_cmd(True, "scan_entropy_mirror_updates", [repositories], {})

    def run_entropy_mirror_updates(self, repository_data):
        return self.Manager.do_cmd(True, "run_entropy_mirror_updates", [repository_data], {})

    def run_entropy_checksum_test(self, repoid, mode = "local"):
        return self.Manager.do_cmd(True, "run_entropy_checksum_test", [repoid, mode], {})

    def get_notice_board(self, repoid):
        return self.Manager.do_cmd(True, "get_notice_board", [repoid], {})

    def remove_notice_board_entries(self, repoid, entry_ids):
        return self.Manager.do_cmd(True, "remove_notice_board_entries", [repoid,entry_ids], {})

    def add_notice_board_entry(self, repoid, title, notice_text, link):
        return self.Manager.do_cmd(True, "add_notice_board_entry", [repoid,title,notice_text,link], {})

class SystemManagerClientInterface:

    ssl_connection = True
    def __init__(self, EntropyInstance, MethodsInterface = None, ClientCommandsInterface = None, quiet = True, show_progress = False, do_cache_connection = False):

        # FIXME: if you enable cache connection, you should also consider to clear the socket buffer
        if not isinstance(EntropyInstance, (EquoInterface, ServerInterface)) and \
            not issubclass(EntropyInstance, (EquoInterface, ServerInterface)):
                mytxt = _("A valid EquoInterface/ServerInterface based instance is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInstance,mytxt,))

        if ClientCommandsInterface != None:
            if not issubclass(ClientCommandsInterface, SystemManagerClientCommands):
                mytxt = _("A valid SystemManagerClientCommands class/subclass is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
            self.ClientCommandsInterface = ClientCommandsInterface
        else:
            self.ClientCommandsInterface = SystemManagerClientCommands

        if MethodsInterface != None:
            if not issubclass(MethodsInterface, SystemManagerMethodsInterface):
                mytxt = _("A valid SystemManagerMethodsInterface class/subclass is needed")
                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
            self.MethodsInterface = MethodsInterface
        else:
            self.MethodsInterface = SystemManagerMethodsInterface

        import socket, struct, entropyTools
        self.socket, self.struct, self.entropyTools = socket, struct, entropyTools
        from datetime import datetime
        self.datetime = datetime
        self.Entropy = EntropyInstance
        self.hostname = None
        self.hostport = None
        self.username = None
        self.password = None
        self.quiet = quiet
        self.do_cache_connection = do_cache_connection
        self.show_progress = show_progress
        self.ClientCommandsInterface = ClientCommandsInterface
        self.Methods = self.MethodsInterface(self)
        self.connection_cache = {}
        self.CacheLock = thread.allocate_lock()
        self.shutdown = False
        self.connection_killer = None
        if self.do_cache_connection:
            self.connection_killer = self.entropyTools.TimeScheduled(self.connection_killer_handler, 2)
            self.connection_killer.start()

    def __del__(self):
        if hasattr(self,'shutdown'):
            self.shutdown = True
        if hasattr(self,'connection_killer'):
            if self.connection_killer != None:
                self.connection_killer.kill()

    def _validate_credentials(self):
        if not isinstance(self.hostname,basestring):
            raise exceptionTools.IncorrectParameter("IncorrectParameter: hostname: %s. %s" % (_('not a string'),_('Please use setup_connection() properly'),))
        if not isinstance(self.username,basestring):
            raise exceptionTools.IncorrectParameter("IncorrectParameter: username: %s. %s" % (_('not a string'),_('Please use setup_connection() properly'),))
        if not isinstance(self.password,basestring):
            raise exceptionTools.IncorrectParameter("IncorrectParameter: password: %s. %s" % (_('not a string'),_('Please use setup_connection() properly'),))
        if not isinstance(self.hostport,int):
            raise exceptionTools.IncorrectParameter("IncorrectParameter: port: %s. %s" % (_('not an int'),_('Please use setup_connection() properly'),))
        if not isinstance(self.ssl_connection,bool):
            raise exceptionTools.IncorrectParameter("IncorrectParameter: ssl_connection: %s. %s" % (_('not a bool'),_('Please use setup_connection() properly'),))

    def get_connection_cache_key(self):
        return hash((self.hostname, self.hostport, self.username, self.password, self.ssl_connection,))

    def get_connection_cache(self):
        if self.do_cache_connection:
            key = self.get_connection_cache_key()
            srv = self.connection_cache.get(key)
            # FIXME: if you enable cache connection, you should also consider to clear the socket buffer
            #  srv.sock_conn
            #  srv.real_sock_conn
            return srv

    def cache_connection(self, srv):
        if self.do_cache_connection:
            key = self.get_connection_cache_key()
            self.connection_cache[key] = {
                'conn': srv,
                'ts': self.get_ts(),
            }

    def update_connection_ts(self):
        if self.do_cache_connection:
            key = self.get_connection_cache_key()
            if key not in self.connection_cache:
                return
            self.connection_cache[key]['ts'] = self.get_ts()

    def kill_all_connections(self):
        if self.do_cache_connection:
            self.CacheLock.acquire()
            try:
                keys = self.connection_cache.keys()
                for key in keys:
                    data = self.connection_cache.pop(key)
                    data['conn'].disconnect()
            finally:
                self.CacheLock.release()

    def connection_killer_handler(self):

        if not self.do_cache_connection: return
        if self.shutdown: return
        if not self.connection_cache: return

        keys = self.connection_cache.keys()
        for key in keys:
            curr_ts = self.get_ts()
            ts = self.connection_cache[key]['ts']
            delta = curr_ts - ts
            if delta.seconds < 60:
                continue
            self.CacheLock.acquire()
            try:
                data = self.connection_cache.pop(key)
            finally:
                self.CacheLock.release()
            srv = data['conn']
            srv.disconnect()

    def get_ts(self):
        return self.datetime.fromtimestamp(time.time())

    def setup_connection(self, hostname, port, username, password, ssl):
        self.hostname = hostname
        self.hostport = port
        self.username = username
        self.password = password
        self.ssl_connection = ssl
        self._validate_credentials()

    def connect_to_service(self, timeout = None):
        self._validate_credentials()
        args = [self.Entropy, self.ClientCommandsInterface]
        kwargs = {
            'ssl': self.ssl_connection,
            'quiet': self.quiet,
            'show_progress': self.show_progress
        }
        if timeout != None: kwargs['socket_timeout'] = timeout
        srv = SystemSocketClientInterface(*args,**kwargs)
        srv.connect(self.hostname, self.hostport)
        return srv

    def get_service_connection(self, timeout = None):
        try:
            srv = self.connect_to_service(timeout = timeout)
        except (exceptionTools.ConnectionError,self.socket.error,self.struct.error,):
            return None
        return srv

    def logout(self, srv, session_id):
        self._validate_credentials()
        return srv.CmdInterface.service_logout(self.username, session_id)

    def login(self, srv, session_id):
        self._validate_credentials()
        return srv.CmdInterface.service_login(self.username, self.password, session_id)

    # eval(func) must have session as first param
    def do_cmd(self, login_required, func, args, kwargs):

        self.CacheLock.acquire()
        try:
            srv = self.get_connection_cache()
            if srv == None:
                srv = self.get_service_connection(timeout = 10)
                if srv != None: self.cache_connection(srv)
            else:
                srv = srv['conn']

            if srv == None:
                return False, 'no connection'
            session = srv.open_session()
            if session == None:
                return False, 'no session'

            self.update_connection_ts()
            args.insert(0,session)

            if login_required:
                logged, error = self.login(srv, session)
                if not logged:
                    srv.close_session(session)
                    if not self.do_cache_connection:
                        srv.disconnect()
                    return False, error

            rslt = eval("srv.CmdInterface.%s" % (func,))(*args,**kwargs)
            if login_required:
                self.logout(srv, session)
            srv.close_session(session)
            if not self.do_cache_connection:
                srv.disconnect()
            return rslt
        finally:
            self.CacheLock.release()

    def get_available_client_commands(self):
        return self.Methods.available_commands.copy()

class UGCClientAuthStore:

    access_file = etpConst['ugc_accessfile']
    def __init__(self):

        from xml.dom import minidom
        from xml.parsers import expat
        self.expat = expat
        self.minidom = minidom
        self.setup_store_paths()
        try:
            self.setup_permissions()
        except IOError:
            pass
        self.store = {}
        try:
            self.xmldoc = self.minidom.parse(self.access_file)
        except (self.expat.ExpatError,IOError,):
            self.xmldoc = None
        if self.xmldoc != None:
            try:
                self.parse_document()
            except self.expat.ExpatError:
                self.xmldoc = None
                self.store = {}

    def setup_store_paths(self):
        myhome = os.getenv("HOME")
        if myhome != None:
            if os.path.isdir(myhome) and os.access(myhome,os.W_OK):
                self.access_file = os.path.join(myhome,".config/entropy",os.path.basename(self.access_file))
        self.access_dir = os.path.dirname(self.access_file)

    def setup_permissions(self):
        if not os.path.isdir(self.access_dir):
            os.makedirs(self.access_dir)
        if not os.path.isfile(self.access_file):
            f = open(self.access_file,"w")
            f.close()
        if etpConst['entropygid'] != None:
            try:
                const_setup_file(self.access_dir, etpConst['entropygid'], 0600)
            except OSError:
                pass

    def parse_document(self):
        self.store.clear()
        store = self.xmldoc.getElementsByTagName("store")[0]
        repositories = store.getElementsByTagName("repository")
        for repository in repositories:
            repoid = repository.getAttribute("id")
            if not repoid: continue
            username = repository.getElementsByTagName("username")[0].firstChild.data.strip()
            password = repository.getElementsByTagName("password")[0].firstChild.data.strip()
            self.store[repoid] = {'username': username, 'password': password}

    def store_login(self, username, password, repository, save = True):
        self.store[repository] = {'username': username, 'password': password}
        if save:
            self.save_store()

    def save_store(self):

        self.xmldoc = self.minidom.Document()
        store = self.xmldoc.createElement("store")

        for repository in self.store:
            repo = self.xmldoc.createElement("repository")
            repo.setAttribute('id',repository)
            # username
            username = self.xmldoc.createElement("username")
            username_value = self.xmldoc.createTextNode(self.store[repository]['username'])
            username.appendChild(username_value)
            repo.appendChild(username)
            # password
            password = self.xmldoc.createElement("password")
            password_value = self.xmldoc.createTextNode(self.store[repository]['password'])
            password.appendChild(password_value)
            repo.appendChild(password)
            store.appendChild(repo)

        self.xmldoc.appendChild(store)
        f = open(self.access_file,"w")
        f.writelines(self.xmldoc.toprettyxml(indent="    "))
        f.flush()
        f.close()
        self.setup_permissions()
        self.parse_document()

    def remove_login(self, repository, save = True):
        if repository in self.store:
            del self.store[repository]
            if save:
                self.save_store()

    def read_login(self, repository):
        if repository in self.store:
            return self.store[repository]['username'],self.store[repository]['password']

class UGCCacheInterface:

    def __init__(self, UGCClientInstance):

        if not isinstance(UGCClientInstance,UGCClientInterface):
            mytxt = _("A valid UGCClientInterface based instance is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        import dumpTools
        self.CacheLock = thread.allocate_lock()
        self.dumpTools = dumpTools
        self.Service = UGCClientInstance
        self.xcache = {}

    def acquire_lock(self):
        self.CacheLock.acquire()

    def release_lock(self):
        self.CacheLock.release()

    def _get_live_cache_item(self, repository, item):
        if repository not in self.xcache:
            return None
        return self.xcache[repository].get(item)

    def _set_live_cache_item(self, repository, item, obj):
        if repository not in self.xcache:
            self.xcache[repository] = {}
        if type(obj) in (list,tuple,):
            my_obj = obj[:]
        elif type(obj) in (set,frozenset,dict,):
            my_obj = obj.copy()
        else:
            my_obj = obj
        self.xcache[repository][item] = my_obj

    def _clear_live_cache_item(self, repository, item):
        if not self.xcache.has_key(repository):
            return
        if not self.xcache[repository].has_key(item):
            return
        del self.xcache[repository][item]

    def _get_store_cache_file(self, iddoc, repository, doc_url):
        return "%s/%s/iddoc_%s/%s" % (etpCache['ugc_docs'], repository, iddoc, doc_url,)

    def _get_vote_cache_file(self, repository):
        return self._get_vote_cache_dir(repository)

    def _get_downloads_cache_file(self, repository):
        return self._get_downloads_cache_dir(repository)

    def _get_alldocs_cache_file(self, pkgkey, repository):
        return self._get_alldocs_cache_dir(repository)+"/"+pkgkey

    def _get_alldocs_cache_dir(self, repository):
        return etpCache['ugc_docs']+"/"+repository

    def _get_downloads_cache_dir(self, repository):
        return etpCache['ugc_downloads']+"/"+repository

    def _get_vote_cache_dir(self, repository):
        return etpCache['ugc_votes']+"/"+repository

    def _get_vote_cache_key(self, repository):
        return 'get_vote_cache_'+repository

    def _get_downloads_cache_key(self, repository):
        return 'get_downloads_cache_'+repository

    def _get_alldocs_cache_key(self, repository):
        return 'get_package_alldocs_cache_'+repository

    def store_document(self, iddoc, repository, doc_url):
        cache_file = os.path.join(etpConst['dumpstoragedir'],self._get_store_cache_file(iddoc, repository, doc_url))
        cache_dir = os.path.dirname(cache_file)

        try:
            if not os.path.isdir(cache_dir):
                os.makedirs(cache_dir,0775)
                if etpConst['entropygid'] != None:
                    const_setup_perms(cache_dir,etpConst['entropygid'])
        except OSError:
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (_("Cannot setup cache directory"),cache_dir,))
        if not os.access(cache_dir,os.W_OK):
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (_("Cannot write to cache directory"),cache_dir,))

        if os.path.isfile(cache_file) or os.path.islink(cache_file):
            try:
                os.remove(cache_file)
            except OSError:
                raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (_("Cannot remove cache file"),cache_file,))

        fetcher = self.Service.Entropy.urlFetcher(doc_url, cache_file, resume = False)
        rc = fetcher.download()
        if rc in ("-1","-2","-3"): return None
        if not os.path.isfile(cache_file): return None

        try:
            os.chmod(cache_file,0664)
            if etpConst['entropygid'] != None:
                os.chown(cache_file,-1,etpConst['entropygid'])
        except OSError:
            raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (_("Cannot write to cache file"),cache_file,))

        del fetcher
        return cache_file

    def get_stored_document(self, iddoc, repository, doc_url):
        cache_file = os.path.join(etpConst['dumpstoragedir'],self._get_store_cache_file(iddoc, repository, doc_url))
        if os.path.isfile(cache_file) and os.access(cache_file,os.R_OK):
            return cache_file

    def update_vote_cache(self, repository, vote_dict):
        cached = self.get_vote_cache(repository)
        if cached == None:
            cached = vote_dict.copy()
        else:
            cached.update(vote_dict)
        self.save_vote_cache(repository,cached)

    def update_downloads_cache(self, repository, down_dict):
        cached = self.get_downloads_cache(repository)
        if cached == None:
            cached = down_dict.copy()
        else:
            cached.update(down_dict)
        self.save_downloads_cache(repository,cached)

    def update_alldocs_cache(self, pkgkey, repository, alldocs_dict):
        self.save_alldocs_cache(pkgkey, repository, alldocs_dict)

    def clear_alldocs_cache(self, repository):
        self.acquire_lock()
        try:
            self.Service.Entropy.clear_dump_cache(self._get_alldocs_cache_dir(repository))
            self._clear_live_cache_item(repository, self._get_alldocs_cache_key(repository))
        finally:
            self.release_lock()

    def clear_downloads_cache(self, repository):
        self.acquire_lock()
        try:
            self.Service.Entropy.clear_dump_cache(self._get_alldocs_cache_dir(repository))
            self._clear_live_cache_item(repository, self._get_downloads_cache_key(repository))
        finally:
            self.release_lock()

    def clear_vote_cache(self, repository):
        self.acquire_lock()
        try:
            self.Service.Entropy.clear_dump_cache(self._get_vote_cache_dir(repository))
            self._clear_live_cache_item(repository, self._get_vote_cache_key(repository))
        finally:
            self.release_lock()

    def clear_cache(self, repository):
        self.clear_alldocs_cache(repository)
        self.clear_downloads_cache(repository)
        self.clear_vote_cache(repository)
        self.xcache.clear()

    def get_vote_cache(self, repository):
        cache_key = self._get_vote_cache_key(repository)
        cached = self._get_live_cache_item(repository, cache_key)
        if cached != None:
            return cached
        self.acquire_lock()
        cache_file = self._get_vote_cache_file(repository)
        try:
            data = self.dumpTools.loadobj(cache_file)
            if data != None:
                self._set_live_cache_item(repository, cache_key, data)
        except (IOError,EOFError,OSError):
            data = None
        finally:
            self.release_lock()
        return data

    def get_downloads_cache(self, repository):
        cache_key = self._get_downloads_cache_key(repository)
        cached = self._get_live_cache_item(repository, cache_key)
        if cached != None:
            return cached
        self.acquire_lock()
        cache_file = self._get_downloads_cache_file(repository)
        try:
            data = self.dumpTools.loadobj(cache_file)
            if data != None:
                self._set_live_cache_item(repository, cache_key, data)
        except (IOError,EOFError,OSError):
            data = None
        finally:
            self.release_lock()
        return data

    def get_alldocs_cache(self, pkgkey, repository):
        cache_key = self._get_alldocs_cache_key(repository)
        cached = self._get_live_cache_item(repository, cache_key)
        if isinstance(cached,dict):
            if cached.has_key(pkgkey): return cached[pkgkey]
        else:
            cached = {}
        self.acquire_lock()
        cache_file = self._get_alldocs_cache_file(pkgkey, repository)
        try:
            data = self.dumpTools.loadobj(cache_file)
            if data != None:
                cached[pkgkey] = data
                self._set_live_cache_item(repository, cache_key, cached)
        except (IOError,EOFError,OSError):
            data = None
        finally:
            self.release_lock()
        return data

    def save_vote_cache(self, repository, vote_dict):
        self.acquire_lock()
        try:
            self._clear_live_cache_item(repository, self._get_vote_cache_key(repository))
            self.dumpTools.dumpobj(self._get_vote_cache_file(repository), vote_dict)
        finally:
            self.release_lock()

    def save_downloads_cache(self, repository, down_dict):
        self.acquire_lock()
        try:
            self._clear_live_cache_item(repository, self._get_downloads_cache_key(repository))
            self.dumpTools.dumpobj(self._get_downloads_cache_file(repository), down_dict)
        finally:
            self.release_lock()

    def save_alldocs_cache(self, pkgkey, repository, alldocs_dict):
        self.acquire_lock()
        try:
            self._clear_live_cache_item(repository, self._get_alldocs_cache_key(repository))
            self.dumpTools.dumpobj(self._get_alldocs_cache_file(pkgkey, repository), alldocs_dict)
        finally:
            self.release_lock()

    def get_package_vote(self, repository, pkgkey):
        cache = self.get_vote_cache(repository)
        if not cache:
            return 0.0
        elif not isinstance(cache,dict):
            return 0.0
        elif not cache.has_key(pkgkey):
            return 0.0
        return cache[pkgkey]

    def get_package_downloads(self, repository, pkgkey):
        cache = self.get_downloads_cache(repository)
        if not cache:
            return 0
        elif not isinstance(cache,dict):
            return 0
        elif not cache.has_key(pkgkey):
            return 0
        try:
            return int(cache[pkgkey])
        except ValueError:
            return 0

class UGCClientInterface:

    ssl_connection = True
    def __init__(self, EquoInstance, quiet = True, show_progress = False):

        if not isinstance(EquoInstance,EquoInterface):
            mytxt = _("A valid EquoInterface based instance is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        import socket, threading
        self.socket, self.threading = socket, threading
        import struct
        self.struct = struct
        self.Entropy = EquoInstance
        self.store = UGCClientAuthStore()
        self.quiet = quiet
        self.show_progress = show_progress
        self.UGCCache = UGCCacheInterface(self)
        self.TxLocks = {}

    def connect_to_service(self, repository, timeout = None):
        if repository not in etpRepositories:
            raise exceptionTools.RepositoryError("RepositoryError: %s" % (_('repository is not available'),))

        try:
            url = etpRepositories[repository]['plain_database'].split("/")[2]
            port = etpRepositories[repository]['service_port']
            if self.ssl_connection: port = etpRepositories[repository]['ssl_service_port']
        except (IndexError,KeyError,):
            raise exceptionTools.RepositoryError("RepositoryError: %s" % (_('repository metadata is malformed'),))

        args = [self.Entropy, RepositorySocketClientCommands]
        kwargs = {
            'ssl': self.ssl_connection,
            'quiet': self.quiet,
            'show_progress': self.show_progress
        }
        if timeout != None: kwargs['socket_timeout'] = timeout
        srv = SystemSocketClientInterface(*args,**kwargs)
        srv.connect(url, port)
        return srv

    def get_service_connection(self, repository, check = True, timeout = None):
        if check:
            if not self.is_repository_eapi3_aware(repository):
                return None
        try:
            srv = self.connect_to_service(repository, timeout = timeout)
        except (exceptionTools.RepositoryError,exceptionTools.ConnectionError,):
            return None
        except (self.socket.error,self.struct.error,):
            return None
        return srv

    def is_repository_eapi3_aware(self, repository):

        aware = self.UGCCache._get_live_cache_item(repository, 'is_repository_eapi3_aware')
        if aware != None:
            return aware

        srv = self.get_service_connection(repository, check = False, timeout = 3)
        if srv == None:
            aware = False
        else:
            session = srv.open_session()
            if session != None:
                srv.close_session(session)
                srv.disconnect()
                aware = True
            else:
                aware = False

        self.UGCCache._set_live_cache_item(repository, 'is_repository_eapi3_aware', aware)
        return aware

    def read_login(self, repository):
        return self.store.read_login(repository)

    def remove_login(self, repository):
        return self.store.remove_login(repository)

    def do_login(self, repository, force = False):

        login_data = self.read_login(repository)
        if (login_data != None) and not force:
            return True,_('ok')

        aware = self.is_repository_eapi3_aware(repository)
        if not aware:
            return False,_('repository does not support EAPI3')

        def fake_callback(*args,**kwargs):
            return True

        attempts = 3
        while attempts:

            # use input box to read login
            input_params = [
                ('username',_('Username'),fake_callback,False),
                ('password',_('Password'),fake_callback,True)
            ]
            login_data = self.Entropy.inputBox(
                "%s %s %s" % (_('Please login against'),repository,_('repository'),),
                input_params,
                cancel_button = True
            )
            if not login_data:
                return False,_('login abort')

            # now verify
            srv = self.get_service_connection(repository)
            if srv == None:
                return False,_('connection issues')
            session = srv.open_session()
            login_status, login_msg = srv.CmdInterface.service_login(login_data['username'], login_data['password'], session)
            if not login_status:
                srv.close_session(session)
                srv.disconnect()
                self.Entropy.askQuestion("%s: %s" % (_("Access denied. Login failed"),login_msg,), responses = ["Ok"])
                attempts -= 1
                continue

            # login accepted, store it?
            srv.close_session(session)
            srv.disconnect()
            rc = self.Entropy.askQuestion(_("Login successful. Do you want to save these credentials ?"))
            save = False
            if rc == "Yes": save = True
            self.store.store_login(login_data['username'], login_data['password'], repository, save = save)
            return True,_('ok')


    def login(self, repository, force = False):

        if not self.TxLocks.has_key(repository):
            self.TxLocks[repository] = self.threading.Lock()

        with self.TxLocks[repository]:
            return self.do_login(repository, force = force)


    def logout(self, repository):
        return self.store.remove_login(repository)

    # eval(func) must have session as first param
    def do_cmd(self, repository, login_required, func, args, kwargs):

        if not self.TxLocks.has_key(repository):
            self.TxLocks[repository] = self.threading.Lock()

        with self.TxLocks[repository]:

            if login_required:
                status, err_msg = self.do_login(repository)
                if not status:
                    return False,err_msg

            srv = self.get_service_connection(repository)
            if srv == None:
                return False,'no connection'
            session = srv.open_session()
            if session == None:
                return False,'no session'
            args.insert(0,session)

            if login_required:
                stored_pass = False
                while 1:
                    # login
                    login_data = self.read_login(repository)
                    if login_data == None:
                        status, msg = self.login(repository)
                        if not status: return status, msg
                        username, password = self.read_login(repository)
                    else:
                        stored_pass = True
                        username, password = login_data
                    logged, error = srv.CmdInterface.service_login(username, password, session)
                    if not logged:
                        if stored_pass:
                            stored_pass = False
                            self.remove_login(repository)
                            continue
                        srv.close_session(session)
                        srv.disconnect()
                        return logged, error
                    break

            rslt = eval("srv.CmdInterface.%s" % (func,))(*args,**kwargs)
            srv.close_session(session)
            srv.disconnect()
            return rslt

    def get_comments(self, repository, pkgkey):
        return self.do_cmd(repository, False, "ugc_get_textdocs", [pkgkey], {})

    def get_comments_by_identifiers(self, repository, identifiers):
        return self.do_cmd(repository, False, "ugc_get_textdocs_by_identifiers", [identifiers], {})

    def get_documents_by_identifiers(self, repository, identifiers):
        return self.do_cmd(repository, False, "ugc_get_documents_by_identifiers", [identifiers], {})

    def add_comment(self, repository, pkgkey, comment, title, keywords):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_add_comment", [pkgkey, comment, title, keywords], {})

    def edit_comment(self, repository, iddoc, new_comment, new_title, new_keywords):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_edit_comment", [iddoc, new_comment, new_title, new_keywords], {})

    def remove_comment(self, repository, iddoc):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_remove_comment", [iddoc], {})

    def add_vote(self, repository, pkgkey, vote):
        data = self.do_cmd(repository, True, "ugc_do_vote", [pkgkey, vote], {})
        if isinstance(data,tuple): voted, add_err_msg = data
        else: return False,'wrong server answer'
        if voted: self.get_vote(repository, pkgkey)
        return voted, add_err_msg

    def get_vote(self, repository, pkgkey):
        vote, err_msg = self.do_cmd(repository, False, "ugc_get_vote", [pkgkey], {})
        if isinstance(vote,float):
            mydict = {pkgkey: vote}
            self.UGCCache.update_vote_cache(repository, mydict)
        return vote, err_msg

    def get_all_votes(self, repository):
        votes_dict, err_msg = self.do_cmd(repository, False, "ugc_get_allvotes", [], {})
        if isinstance(votes_dict,dict):
            self.UGCCache.update_vote_cache(repository, votes_dict)
        return votes_dict, err_msg

    def add_download(self, repository, pkgkey):
        return self.do_cmd(repository, False, "ugc_do_download", [pkgkey], {})

    def add_downloads(self, repository, pkgkeys):
        return self.do_cmd(repository, False, "ugc_do_downloads", [pkgkeys], {})

    def get_downloads(self, repository, pkgkey):
        data = self.do_cmd(repository, False, "ugc_get_downloads", [pkgkey], {})
        if isinstance(data,tuple): downloads, err_msg = data
        else: return False,'wrong server answer'
        if downloads:
            mydict = {pkgkey: downloads}
            self.UGCCache.update_downloads_cache(repository, mydict)
        return downloads, err_msg

    def get_all_downloads(self, repository):
        down_dict, err_msg = self.do_cmd(repository, False, "ugc_get_alldownloads", [], {})
        if isinstance(down_dict,dict):
            self.UGCCache.update_downloads_cache(repository, down_dict)
        return down_dict, err_msg

    def send_file(self, repository, pkgkey, file_path, title, description, keywords):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_send_file", [pkgkey, file_path, etpConst['ugc_doctypes']['generic_file'], title, description, keywords], {})

    def remove_file(self, repository, iddoc):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_remove_file", [iddoc], {})

    def send_image(self, repository, pkgkey, image_path, title, description, keywords):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_send_file", [pkgkey, image_path, etpConst['ugc_doctypes']['image'], title, description, keywords], {})

    def remove_image(self, repository, iddoc):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_remove_image", [iddoc], {})

    def send_youtube_video(self, repository, pkgkey, video_path, title, description, keywords):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_send_file", [pkgkey, video_path, etpConst['ugc_doctypes']['youtube_video'], title, description, keywords], {})

    def remove_youtube_video(self, repository, iddoc):
        self.UGCCache.clear_alldocs_cache(repository)
        return self.do_cmd(repository, True, "ugc_remove_youtube_video", [iddoc], {})

    def get_docs(self, repository, pkgkey):
        data = self.do_cmd(repository, False, "ugc_get_docs", [pkgkey], {})
        if isinstance(data,tuple): docs_data, err_msg = data
        else: return False,'wrong server answer'
        if err_msg == 'ok':
            self.UGCCache.update_alldocs_cache(pkgkey, repository, docs_data)
        return docs_data, err_msg

    def send_document_autosense(self, repository, pkgkey, ugc_type, data, title, description, keywords):
        if ugc_type == etpConst['ugc_doctypes']['generic_file']:
            return self.send_file(repository, pkgkey, data, title, description, keywords)
        elif ugc_type == etpConst['ugc_doctypes']['image']:
            return self.send_image(repository, pkgkey, data, title, description, keywords)
        elif ugc_type == etpConst['ugc_doctypes']['youtube_video']:
            return self.send_youtube_video(repository, pkgkey, data, title, description, keywords)
        elif ugc_type == etpConst['ugc_doctypes']['comments']:
            return self.add_comment(repository, pkgkey, description, title, keywords)
        return None,'type not supported locally'

    def remove_document_autosense(self, repository, iddoc, ugc_type):
        if ugc_type == etpConst['ugc_doctypes']['generic_file']:
            return self.remove_file(repository, iddoc)
        elif ugc_type == etpConst['ugc_doctypes']['image']:
            return self.remove_image(repository, iddoc)
        elif ugc_type == etpConst['ugc_doctypes']['youtube_video']:
            return self.remove_youtube_video(repository, iddoc)
        elif ugc_type == etpConst['ugc_doctypes']['comments']:
            return self.remove_comment(repository, iddoc)
        return None,'type not supported locally'

class ServerMirrorsInterface:

    import entropyTools, dumpTools, socket
    def __init__(self,  ServerInstance, repo = None):

        if not isinstance(ServerInstance,ServerInterface):
            mytxt = _("A valid ServerInterface based instance is needed")
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (mytxt,))

        self.Entropy = ServerInstance
        self.FtpInterface = self.Entropy.FtpInterface
        self.rssFeed = self.Entropy.rssFeed

        mytxt = blue("%s:") % (_("Entropy Server Mirrors Interface loaded"),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 2,
            type = "info",
            header = red(" @@ ")
        )
        mytxt = _("mirror")
        for mirror in self.Entropy.get_remote_mirrors(repo):
            mirror = self.entropyTools.hideFTPpassword(mirror)
            self.Entropy.updateProgress(
                blue("%s: %s") % (mytxt,darkgreen(mirror),),
                importance = 0,
                type = "info",
                header = brown("   # ")
            )


    def lock_mirrors(self, lock = True, mirrors = [], repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        if not mirrors:
            mirrors = self.Entropy.get_remote_mirrors(repo)

        issues = False
        for uri in mirrors:

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

            lock_text = _("unlocking")
            if lock: lock_text = _("locking")
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s %s" % (
                    brown(repo),
                    darkgreen(crippled_uri),
                    bold(lock_text),
                    blue("%s...") % (_("mirror"),),
                ),
                importance = 1,
                type = "info",
                header = brown(" * "),
                back = True
            )

            ftp = self.FtpInterface(uri, self.Entropy)
            my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
            ftp.setCWD(my_path, dodir = True)

            if lock and ftp.isFileAvailable(etpConst['etpdatabaselockfile']):
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s" % (
                            brown(repo),
                            darkgreen(crippled_uri),
                            blue(_("mirror already locked")),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                ftp.closeConnection()
                continue
            elif not lock and not ftp.isFileAvailable(etpConst['etpdatabaselockfile']):
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s" % (
                            brown(repo),
                            darkgreen(crippled_uri),
                            blue(_("mirror already unlocked")),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                ftp.closeConnection()
                continue

            if lock:
                rc = self.do_mirror_lock(uri, ftp, repo = repo)
            else:
                rc = self.do_mirror_unlock(uri, ftp, repo = repo)
            ftp.closeConnection()
            if not rc: issues = True

        if not issues:
            database_taint_file = self.Entropy.get_local_database_taint_file(repo)
            if os.path.isfile(database_taint_file):
                os.remove(database_taint_file)

        return issues

    # this functions makes entropy clients to not download anything from the chosen
    # mirrors. it is used to avoid clients to download databases while we're uploading
    # a new one.
    def lock_mirrors_for_download(self, lock = True, mirrors = [], repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        if not mirrors:
            mirrors = self.Entropy.get_remote_mirrors(repo)

        issues = False
        for uri in mirrors:

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

            lock_text = _("unlocking")
            if lock: lock_text = _("locking")
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s %s..." % (
                            blue(repo),
                            red(crippled_uri),
                            bold(lock_text),
                            blue(_("mirror for download")),
                    ),
                importance = 1,
                type = "info",
                header = red(" @@ "),
                back = True
            )

            ftp = self.FtpInterface(uri, self.Entropy)
            my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
            ftp.setCWD(my_path, dodir = True)

            if lock and ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile']):
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue(_("mirror already locked for download")),
                        ),
                    importance = 1,
                    type = "info",
                    header = red(" @@ ")
                )
                ftp.closeConnection()
                continue
            elif not lock and not ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile']):
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue(_("mirror already unlocked for download")),
                        ),
                    importance = 1,
                    type = "info",
                    header = red(" @@ ")
                )
                ftp.closeConnection()
                continue

            if lock:
                rc = self.do_mirror_lock(uri, ftp, dblock = False, repo = repo)
            else:
                rc = self.do_mirror_unlock(uri, ftp, dblock = False, repo = repo)
            ftp.closeConnection()
            if not rc: issues = True

        return issues

    def do_mirror_lock(self, uri, ftp_connection = None, dblock = True, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
        if not ftp_connection:
            ftp_connection = self.FtpInterface(uri, self.Entropy)
            ftp_connection.setCWD(my_path, dodir = True)
        else:
            mycwd = ftp_connection.getCWD()
            if mycwd != my_path:
                ftp_connection.setBasedir()
                ftp_connection.setCWD(my_path, dodir = True)

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
        lock_string = ''
        if dblock:
            self.create_local_database_lockfile(repo)
            lock_file = self.get_database_lockfile(repo)
        else:
            lock_string = _('for download') # locking/unlocking mirror1 for download
            self.create_local_database_download_lockfile(repo)
            lock_file = self.get_database_download_lockfile(repo)

        rc = ftp_connection.uploadFile(lock_file, ascii = True)
        if rc:
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue(_("mirror successfully locked")),
                            blue(lock_string),
                    ),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )
        else:
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s: %s - %s %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue("lock error"),
                            rc,
                            blue(_("mirror not locked")),
                            blue(lock_string),
                    ),
                importance = 1,
                type = "error",
                header = darkred(" * ")
            )
            self.remove_local_database_lockfile(repo)

        return rc


    def do_mirror_unlock(self, uri, ftp_connection, dblock = True, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
        if not ftp_connection:
            ftp_connection = self.FtpInterface(uri, self.Entropy)
            ftp_connection.setCWD(my_path)
        else:
            mycwd = ftp_connection.getCWD()
            if mycwd != my_path:
                ftp_connection.setBasedir()
                ftp_connection.setCWD(my_path)

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

        if dblock:
            dbfile = etpConst['etpdatabaselockfile']
        else:
            dbfile = etpConst['etpdatabasedownloadlockfile']
        rc = ftp_connection.deleteFile(dbfile)
        if rc:
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue(_("mirror successfully unlocked")),
                    ),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
            if dblock:
                self.remove_local_database_lockfile(repo)
            else:
                self.remove_local_database_download_lockfile(repo)
        else:
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s: %s - %s" % (
                            blue(repo),
                            red(crippled_uri),
                            blue(_("unlock error")),
                            rc,
                            blue(_("mirror not unlocked")),
                    ),
                importance = 1,
                type = "error",
                header = darkred(" * ")
            )

        return rc

    def get_database_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        return os.path.join(self.Entropy.get_local_database_dir(repo),etpConst['etpdatabaselockfile'])

    def get_database_download_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        return os.path.join(self.Entropy.get_local_database_dir(repo),etpConst['etpdatabasedownloadlockfile'])

    def create_local_database_download_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        lock_file = self.get_database_download_lockfile(repo)
        f = open(lock_file,"w")
        f.write("download locked")
        f.flush()
        f.close()

    def create_local_database_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        lock_file = self.get_database_lockfile(repo)
        f = open(lock_file,"w")
        f.write("database locked")
        f.flush()
        f.close()

    def remove_local_database_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        lock_file = self.get_database_lockfile(repo)
        if os.path.isfile(lock_file):
            os.remove(lock_file)

    def remove_local_database_download_lockfile(self, repo = None):
        if repo == None:
            repo = self.Entropy.default_repository
        lock_file = self.get_database_download_lockfile(repo)
        if os.path.isfile(lock_file):
            os.remove(lock_file)

    def download_package(self, uri, pkg_relative_path, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

        tries = 0
        while tries < 5:
            tries += 1

            pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:])
            pkg_to_join_dirpath = os.path.dirname(pkg_to_join_path)
            pkgfile = os.path.basename(pkg_relative_path)

            self.Entropy.updateProgress(
                "[repo:%s|%s|#%s] %s: %s" % (
                    brown(repo),
                    darkgreen(crippled_uri),
                    brown(tries),
                    blue(_("connecting to download package")), # connecting to download package xyz
                    darkgreen(pkg_to_join_path),
                ),
                importance = 1,
                type = "info",
                header = darkgreen(" * "),
                back = True
            )

            ftp = self.FtpInterface(uri, self.Entropy)
            dirpath = os.path.join(self.Entropy.get_remote_packages_relative_path(repo),pkg_to_join_dirpath)
            ftp.setCWD(dirpath, dodir = True)

            self.Entropy.updateProgress(
                "[repo:%s|%s|#%s] %s: %s" % (
                    brown(repo),
                    darkgreen(crippled_uri),
                    brown(tries),
                    blue(_("downloading package")),
                    darkgreen(pkg_to_join_path),
                ),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )

            download_path = os.path.join(self.Entropy.get_local_packages_directory(repo),pkg_to_join_dirpath)
            if (not os.path.isdir(download_path)) and (not os.access(download_path,os.R_OK)):
                os.makedirs(download_path)
            rc = ftp.downloadFile(pkgfile,download_path)
            if not rc:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|#%s] %s: %s %s" % (
                        brown(repo),
                        darkgreen(crippled_uri),
                        brown(tries),
                        blue(_("package")),
                        darkgreen(pkg_to_join_path),
                        blue(_("does not exist")),
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )
                ftp.closeConnection()
                return rc

            dbconn = self.Entropy.openServerDatabase(read_only = True, no_upload = True, repo = repo)
            idpackage = dbconn.getIDPackageFromDownload(pkg_relative_path)
            if idpackage == -1:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|#%s] %s: %s %s" % (
                        brown(repo),
                        darkgreen(crippled_uri),
                        brown(tries),
                        blue(_("package")),
                        darkgreen(pkgfile),
                        blue(_("is not listed in the current repository database!!")),
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )
                ftp.closeConnection()
                return 0

            storedmd5 = dbconn.retrieveDigest(idpackage)
            self.Entropy.updateProgress(
                "[repo:%s|%s|#%s] %s: %s" % (
                    brown(repo),
                    darkgreen(crippled_uri),
                    brown(tries),
                    blue(_("verifying checksum of package")),
                    darkgreen(pkgfile),
                ),
                importance = 1,
                type = "info",
                header = darkgreen(" * "),
                back = True
            )

            pkg_path = os.path.join(download_path,pkgfile)
            md5check = self.entropyTools.compareMd5(pkg_path,storedmd5)
            if md5check:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|#%s] %s: %s %s" % (
                        brown(repo),
                        darkgreen(crippled_uri),
                        brown(tries),
                        blue(_("package")),
                        darkgreen(pkgfile),
                        blue(_("downloaded successfully")),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                return True
            else:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|#%s] %s: %s %s" % (
                        brown(repo),
                        darkgreen(crippled_uri),
                        brown(tries),
                        blue(_("package")),
                        darkgreen(pkgfile),
                        blue(_("checksum does not match. re-downloading...")),
                    ),
                    importance = 1,
                    type = "warning",
                    header = darkred(" * ")
                )
                if os.path.isfile(pkg_path):
                    os.remove(pkg_path)

            continue

        # if we get here it means the files hasn't been downloaded properly
        self.Entropy.updateProgress(
            "[repo:%s|%s|#%s] %s: %s %s" % (
                brown(repo),
                darkgreen(crippled_uri),
                brown(tries),
                blue(_("package")),
                darkgreen(pkgfile),
                blue(_("seems broken. Consider to re-package it. Giving up!")),
            ),
            importance = 1,
            type = "error",
            header = darkred(" !!! ")
        )
        return False


    def get_remote_databases_status(self, repo = None, mirrors = []):

        if repo == None:
            repo = self.Entropy.default_repository

        if not mirrors:
            mirrors = self.Entropy.get_remote_mirrors(repo)

        data = []
        for uri in mirrors:

            ftp = self.FtpInterface(uri, self.Entropy)
            try:
                my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
                ftp.setCWD(my_path)
            except ftp.ftplib.error_perm:
                crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s !" % (
                            brown(repo),
                            darkgreen(crippled_uri),
                            blue(_("mirror doesn't have a valid directory structure")),
                    ),
                    importance = 1,
                    type = "warning",
                    header = darkred(" !!! ")
                )
                ftp.closeConnection()
                continue
            cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
            if cmethod == None:
                raise exceptionTools.InvalidDataType("InvalidDataType: %s." % (
                        _("Wrong database compression method passed"),
                    )
                )
            compressedfile = etpConst[cmethod[2]]

            revision = 0
            rc1 = ftp.isFileAvailable(compressedfile)
            revfilename = os.path.basename(self.Entropy.get_local_database_revision_file(repo))
            rc2 = ftp.isFileAvailable(revfilename)
            if rc1 and rc2:
                revision_localtmppath = os.path.join(etpConst['packagestmpdir'],revfilename)
                ftp.downloadFile(revfilename,etpConst['packagestmpdir'],True)
                f = open(revision_localtmppath,"r")
                try:
                    revision = int(f.readline().strip())
                except ValueError:
                    crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
                    self.Entropy.updateProgress(
                        "[repo:%s|%s] %s: %s" % (
                                brown(repo),
                                darkgreen(crippled_uri),
                                blue(_("mirror doesn't have a valid database revision file")),
                                bold(revision),
                        ),
                        importance = 1,
                        type = "error",
                        header = darkred(" !!! ")
                    )
                    revision = 0
                f.close()
                if os.path.isfile(revision_localtmppath):
                    os.remove(revision_localtmppath)

            info = [uri,revision]
            data.append(info)
            ftp.closeConnection()

        return data

    def is_local_database_locked(self, repo = None):
        x = repo
        if x == None:
            x = self.Entropy.default_repository
        lock_file = self.get_database_lockfile(x)
        return os.path.isfile(lock_file)

    def get_mirrors_lock(self, repo = None):

        dbstatus = []
        for uri in self.Entropy.get_remote_mirrors(repo):
            data = [uri,False,False]
            ftp = FtpInterface(uri, self.Entropy)
            try:
                my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
                ftp.setCWD(my_path)
            except ftp.ftplib.error_perm:
                ftp.closeConnection()
                continue
            if ftp.isFileAvailable(etpConst['etpdatabaselockfile']):
                # upload locked
                data[1] = True
            if ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile']):
                # download locked
                data[2] = True
            ftp.closeConnection()
            dbstatus.append(data)
        return dbstatus

    def download_notice_board(self, repo = None):

        if repo == None: repo = self.Entropy.default_repository
        mirrors = self.Entropy.get_remote_mirrors(repo)
        rss_path = self.Entropy.get_local_database_notice_board_file(repo)
        mytmpdir = self.entropyTools.getRandomTempFile()
        os.makedirs(mytmpdir)

        self.Entropy.updateProgress(
            "[repo:%s] %s %s" % (
                    brown(repo),
                    blue(_("downloading notice board from mirrors to")),
                    red(rss_path),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )

        downloaded = False
        for uri in mirrors:
            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            downloader = self.FileTransceiver(
                self.FtpInterface, self.Entropy, [uri],
                [rss_path], download = True,
                local_basedir = mytmpdir, critical_files = [rss_path], repo = repo
            )
            errors, m_fine_uris, m_broken_uris = downloader.go()
            if not errors:
                self.Entropy.updateProgress(
                    "[repo:%s] %s: %s" % (
                            brown(repo),
                            blue(_("notice board downloaded successfully from")),
                            red(crippled_uri),
                    ),
                    importance = 1,
                    type = "info",
                    header = blue(" @@ ")
                )
                downloaded = True
                break

        if downloaded:
            shutil.move(os.path.join(mytmpdir,os.path.basename(rss_path)),rss_path)

        return downloaded

    def upload_notice_board(self, repo = None):

        if repo == None: repo = self.Entropy.default_repository
        mirrors = self.Entropy.get_remote_mirrors(repo)
        rss_path = self.Entropy.get_local_database_notice_board_file(repo)

        self.Entropy.updateProgress(
            "[repo:%s] %s %s" % (
                    brown(repo),
                    blue(_("uploading notice board from")),
                    red(rss_path),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )

        uploader = self.FileTransceiver(
            self.FtpInterface,
            self.Entropy,
            mirrors,
            [rss_path],
            critical_files = [rss_path],
            repo = repo
        )
        errors, m_fine_uris, m_broken_uris = uploader.go()
        if errors:
            m_broken_uris = sorted(list(m_broken_uris))
            m_broken_uris = [self.entropyTools.extractFTPHostFromUri(x) for x in m_broken_uris]
            self.Entropy.updateProgress(
                "[repo:%s] %s %s" % (
                        brown(repo),
                        blue(_("notice board upload failed on")),
                        red(', '.join(m_broken_uris)),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            return False
        self.Entropy.updateProgress(
            "[repo:%s] %s" % (
                    brown(repo),
                    blue(_("notice board upload success")),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )
        return True


    def update_notice_board(self, title, notice_text, link = None, repo = None):

        rss_title = "%s Notice Board" % (etpConst['systemname'],)
        rss_description = "Inform about important distribution activities."
        rss_path = self.Entropy.get_local_database_notice_board_file(repo)
        if not link: link = etpConst['rss-website-url']

        self.download_notice_board(repo)
        Rss = self.rssFeed(rss_path, rss_title, rss_description, maxentries = 20)
        Rss.addItem(title, link, description = notice_text)
        Rss.writeChanges()
        status = self.upload_notice_board(repo)
        return status

    def read_notice_board(self, do_download = True, repo = None):

        rss_path = self.Entropy.get_local_database_notice_board_file(repo)
        if do_download: self.download_notice_board(repo)
        if not (os.path.isfile(rss_path) and os.access(rss_path,os.R_OK)):
            return None
        Rss = self.rssFeed(rss_path, '', '')
        return Rss.getEntries()

    def remove_from_notice_board(self, identifier, repo = None):

        rss_path = self.Entropy.get_local_database_notice_board_file(repo)
        rss_title = "%s Notice Board" % (etpConst['systemname'],)
        rss_description = "Inform about important distribution activities."
        if not (os.path.isfile(rss_path) and os.access(rss_path,os.R_OK)):
            return 0
        Rss = self.rssFeed(rss_path, rss_title, rss_description)
        data = Rss.removeEntry(identifier)
        Rss.writeChanges()
        return data

    def update_rss_feed(self, repo = None):

        #db_dir = self.Entropy.get_local_database_dir(repo)
        rss_path = self.Entropy.get_local_database_rss_file(repo)
        rss_light_path = self.Entropy.get_local_database_rsslight_file(repo)
        rss_dump_name = etpConst['rss-dump-name']
        db_revision_path = self.Entropy.get_local_database_revision_file(repo)

        rss_title = "%s Online Repository Status" % (etpConst['systemname'],)
        rss_description = "Keep you updated on what's going on in the %s Repository." % (etpConst['systemname'],)
        Rss = self.rssFeed(rss_path, rss_title, rss_description, maxentries = etpConst['rss-max-entries'])
        # load dump
        db_actions = self.Entropy.Cacher.pop(rss_dump_name)
        if db_actions:
            try:
                f = open(db_revision_path)
                revision = f.readline().strip()
                f.close()
            except (IOError, OSError):
                revision = "N/A"
            commitmessage = ''
            if self.Entropy.rssMessages['commitmessage']:
                commitmessage = ' :: '+self.Entropy.rssMessages['commitmessage']
            title = ": "+etpConst['systemname']+" "+etpConst['product'][0].upper()+etpConst['product'][1:]+" "+etpConst['branch']+" :: Revision: "+revision+commitmessage
            link = etpConst['rss-base-url']
            # create description
            added_items = db_actions.get("added")
            if added_items:
                for atom in added_items:
                    mylink = link+"?search="+atom.split("~")[0]+"&arch="+etpConst['currentarch']+"&product="+etpConst['product']
                    description = atom+": "+added_items[atom]['description']
                    Rss.addItem(title = "Added/Updated"+title, link = mylink, description = description)
            removed_items = db_actions.get("removed")
            if removed_items:
                for atom in removed_items:
                    description = atom+": "+removed_items[atom]['description']
                    Rss.addItem(title = "Removed"+title, link = link, description = description)
            light_items = db_actions.get('light')
            if light_items:
                rssLight = self.rssFeed(rss_light_path, rss_title, rss_description, maxentries = etpConst['rss-light-max-entries'])
                for atom in light_items:
                    mylink = link+"?search="+atom.split("~")[0]+"&arch="+etpConst['currentarch']+"&product="+etpConst['product']
                    description = light_items[atom]['description']
                    rssLight.addItem(title = "["+revision+"] "+atom, link = mylink, description = description)
                rssLight.writeChanges()

        Rss.writeChanges()
        self.Entropy.rssMessages.clear()
        self.dumpTools.removeobj(rss_dump_name)


    # f_out is a file instance
    def dump_database_to_file(self, db_path, destination_path, opener, repo = None):
        f_out = opener(destination_path, "wb")
        dbconn = self.Entropy.openServerDatabase(db_path, just_reading = True, repo = repo)
        dbconn.doDatabaseExport(f_out)
        self.Entropy.close_server_database(dbconn)
        f_out.close()

    def create_file_checksum(self, file_path, checksum_path):
        mydigest = self.entropyTools.md5sum(file_path)
        f = open(checksum_path,"w")
        mystring = "%s  %s\n" % (mydigest,os.path.basename(file_path),)
        f.write(mystring)
        f.flush()
        f.close()

    def compress_file(self, file_path, destination_path, opener):
        f_out = opener(destination_path, "wb")
        f_in = open(file_path,"rb")
        data = f_in.read(8192)
        while data:
            f_out.write(data)
            data = f_in.read(8192)
        f_in.close()
        try:
            f_out.flush()
        except:
            pass
        f_out.close()

    def get_files_to_sync(self, cmethod, download = False, repo = None):

        critical = []
        data = {}
        data['database_revision_file'] = self.Entropy.get_local_database_revision_file(repo)
        critical.append(data['database_revision_file'])

        database_package_mask_file = self.Entropy.get_local_database_mask_file(repo)
        if os.path.isfile(database_package_mask_file) or download:
            data['database_package_mask_file'] = database_package_mask_file
            if not download:
                critical.append(data['database_package_mask_file'])

        database_package_system_mask_file = self.Entropy.get_local_database_system_mask_file(repo)
        if os.path.isfile(database_package_system_mask_file) or download:
            data['database_package_system_mask_file'] = database_package_mask_file
            if not download:
                critical.append(data['database_package_system_mask_file'])

        database_package_confl_tagged_file = self.Entropy.get_local_database_confl_tagged_file(repo)
        if os.path.isfile(database_package_confl_tagged_file) or download:
            data['database_package_confl_tagged_file'] = database_package_confl_tagged_file
            if not download:
                critical.append(data['database_package_confl_tagged_file'])

        database_license_whitelist_file = self.Entropy.get_local_database_licensewhitelist_file(repo)
        if os.path.isfile(database_license_whitelist_file) or download:
            data['database_license_whitelist_file'] = database_license_whitelist_file
            if not download:
                critical.append(data['database_license_whitelist_file'])

        database_rss_file = self.Entropy.get_local_database_rss_file(repo)
        if os.path.isfile(database_rss_file) or download:
            data['database_rss_file'] = database_rss_file
            if not download:
                critical.append(data['database_rss_file'])
        database_rss_light_file = self.Entropy.get_local_database_rsslight_file(repo)
        if os.path.isfile(database_rss_light_file) or download:
            data['database_rss_light_file'] = database_rss_light_file
            if not download:
                critical.append(data['database_rss_light_file'])

        # EAPI 2,3
        if not download: # we don't need to get the dump
            data['dump_path'] = os.path.join(self.Entropy.get_local_database_dir(repo),etpConst[cmethod[3]])
            critical.append(data['dump_path'])
            data['dump_path_digest'] = os.path.join(self.Entropy.get_local_database_dir(repo),etpConst[cmethod[4]])
            critical.append(data['dump_path_digest'])
            data['database_path'] = self.Entropy.get_local_database_file(repo)
            critical.append(data['database_path'])

        # EAPI 1
        data['compressed_database_path'] = os.path.join(self.Entropy.get_local_database_dir(repo),etpConst[cmethod[2]])
        critical.append(data['compressed_database_path'])
        data['compressed_database_path_digest'] = os.path.join(
            self.Entropy.get_local_database_dir(repo),etpConst['etpdatabasehashfile']
        )
        critical.append(data['compressed_database_path_digest'])

        # SSL cert file, just for reference
        ssl_ca_cert = self.Entropy.get_local_database_ca_cert_file()
        if os.path.isfile(ssl_ca_cert):
            data['ssl_ca_cert_file'] = ssl_ca_cert
            if not download:
                critical.append(ssl_ca_cert)
        ssl_server_cert = self.Entropy.get_local_database_server_cert_file()
        if os.path.isfile(ssl_server_cert):
            data['ssl_server_cert_file'] = ssl_server_cert
            if not download:
                critical.append(ssl_server_cert)

        # Some information regarding how packages are built
        spm_files = [
            (etpConst['spm']['global_make_conf'],"global_make_conf"),
            (etpConst['spm']['global_package_keywords'],"global_package_keywords"),
            (etpConst['spm']['global_package_use'],"global_package_use"),
            (etpConst['spm']['global_package_mask'],"global_package_mask"),
            (etpConst['spm']['global_package_unmask'],"global_package_unmask"),
        ]
        for myfile,myname in spm_files:
            if os.path.isfile(myfile) and os.access(myfile,os.R_OK):
                data[myname] = myfile

        make_profile = etpConst['spm']['global_make_profile']
        if os.path.islink(make_profile):
            mylink = os.readlink(make_profile)
            mytmpdir = os.path.dirname(self.Entropy.entropyTools.getRandomTempFile())
            mytmpfile = os.path.join(mytmpdir,'profile.link')
            f = open(mytmpfile,"w")
            f.write(mylink)
            f.flush()
            f.close()
            data['global_make_profile'] = mytmpfile

        return data, critical

    class FileTransceiver:

        import entropyTools
        def __init__(   self,
                        ftp_interface,
                        entropy_interface,
                        uris,
                        files_to_upload,
                        download = False,
                        remove = False,
                        ftp_basedir = None,
                        local_basedir = None,
                        critical_files = [],
                        use_handlers = False,
                        handlers_data = {},
                        repo = None
            ):

            self.FtpInterface = ftp_interface
            self.Entropy = entropy_interface
            if not isinstance(uris,list):
                raise exceptionTools.InvalidDataType("InvalidDataType: %s" % (_("uris must be a list instance"),))
            if not isinstance(files_to_upload,(list,dict)):
                raise exceptionTools.InvalidDataType("InvalidDataType: %s" % (
                        _("files_to_upload must be a list or dict instance"),
                    )
                )
            self.uris = uris
            if isinstance(files_to_upload,list):
                self.myfiles = files_to_upload[:]
            else:
                self.myfiles = sorted([x for x in files_to_upload])
            self.download = download
            self.remove = remove
            self.repo = repo
            if self.repo == None:
                self.repo = self.Entropy.default_repository
            self.use_handlers = use_handlers
            if self.remove:
                self.download = False
                self.use_handlers = False
            if not ftp_basedir:
                # default to database directory
                my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
                self.ftp_basedir = unicode(my_path)
            else:
                self.ftp_basedir = unicode(ftp_basedir)
            if not local_basedir:
                # default to database directory
                self.local_basedir = os.path.dirname(self.Entropy.get_local_database_file(self.repo))
            else:
                self.local_basedir = unicode(local_basedir)
            self.critical_files = critical_files
            self.handlers_data = handlers_data.copy()

        def handler_verify_upload(self, local_filepath, uri, ftp_connection, counter, maxcount, action, tries):

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)

            self.Entropy.updateProgress(
                "[%s|#%s|(%s/%s)] %s: %s" % (
                    blue(crippled_uri),
                    darkgreen(str(tries)),
                    blue(str(counter)),
                    bold(str(maxcount)),
                    darkgreen(_("verifying upload (if supported)")),
                    blue(os.path.basename(local_filepath)),
                ),
                importance = 0,
                type = "info",
                header = red(" @@ "),
                back = True
            )

            checksum = self.Entropy.get_remote_package_checksum(
                self.repo,
                os.path.basename(local_filepath),
                self.handlers_data['branch']
            )
            if checksum == None:
                self.Entropy.updateProgress(
                    "[%s|#%s|(%s/%s)] %s: %s: %s" % (
                        blue(crippled_uri),
                        darkgreen(str(tries)),
                        blue(str(counter)),
                        bold(str(maxcount)),
                        blue(_("digest verification")),
                        os.path.basename(local_filepath),
                        darkred(_("not supported")),
                    ),
                    importance = 0,
                    type = "info",
                    header = red(" @@ ")
                )
                return True
            elif checksum == False:
                self.Entropy.updateProgress(
                    "[%s|#%s|(%s/%s)] %s: %s: %s" % (
                        blue(crippled_uri),
                        darkgreen(str(tries)),
                        blue(str(counter)),
                        bold(str(maxcount)),
                        blue(_("digest verification")),
                        os.path.basename(local_filepath),
                        bold(_("file not found")),
                    ),
                    importance = 0,
                    type = "warning",
                    header = brown(" @@ ")
                )
                return False
            elif len(checksum) == 32:
                # valid? checking
                ckres = self.entropyTools.compareMd5(local_filepath,checksum)
                if ckres:
                    self.Entropy.updateProgress(
                        "[%s|#%s|(%s/%s)] %s: %s: %s" % (
                            blue(crippled_uri),
                            darkgreen(str(tries)),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(_("digest verification")),
                            os.path.basename(local_filepath),
                            darkgreen(_("so far, so good!")),
                        ),
                        importance = 0,
                        type = "info",
                        header = red(" @@ ")
                    )
                    return True
                else:
                    self.Entropy.updateProgress(
                        "[%s|#%s|(%s/%s)] %s: %s: %s" % (
                            blue(crippled_uri),
                            darkgreen(str(tries)),
                            blue(str(counter)),
                            bold(str(maxcount)),
                            blue(_("digest verification")),
                            os.path.basename(local_filepath),
                            darkred(_("invalid checksum")),
                        ),
                        importance = 0,
                        type = "warning",
                        header = brown(" @@ ")
                    )
                    return False
            else:
                self.Entropy.updateProgress(
                    "[%s|#%s|(%s/%s)] %s: %s: %s" % (
                        blue(crippled_uri),
                        darkgreen(str(tries)),
                        blue(str(counter)),
                        bold(str(maxcount)),
                        blue(_("digest verification")),
                        os.path.basename(local_filepath),
                        darkred(_("unknown data returned")),
                    ),
                    importance = 0,
                    type = "warning",
                    header = brown(" @@ ")
                )
                return True

        def go(self):

            broken_uris = set()
            fine_uris = set()
            errors = False
            action = 'upload'
            if self.download:
                action = 'download'
            elif self.remove:
                action = 'remove'

            for uri in self.uris:

                crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
                self.Entropy.updateProgress(
                    "[%s|%s] %s..." % (
                        blue(crippled_uri),
                        brown(action),
                        blue(_("connecting to mirror")),
                    ),
                    importance = 0,
                    type = "info",
                    header = blue(" @@ ")
                )
                ftp = self.FtpInterface(uri, self.Entropy)
                my_path = os.path.join(self.Entropy.get_remote_database_relative_path(self.repo),etpConst['branch'])
                self.Entropy.updateProgress(
                    "[%s|%s] %s %s..." % (
                        blue(crippled_uri),
                        brown(action),
                        blue(_("changing directory to")),
                        darkgreen(my_path),
                    ),
                    importance = 0,
                    type = "info",
                    header = blue(" @@ ")
                )

                ftp.setCWD(self.ftp_basedir, dodir = True)
                maxcount = len(self.myfiles)
                counter = 0

                for mypath in self.myfiles:

                    ftp.setBasedir()
                    ftp.setCWD(self.ftp_basedir, dodir = True)

                    mycwd = None
                    if isinstance(mypath,tuple):
                        if len(mypath) < 2: continue
                        mycwd = mypath[0]
                        mypath = mypath[1]
                        ftp.setCWD(mycwd, dodir = True)

                    syncer = ftp.uploadFile
                    myargs = [mypath]
                    if self.download:
                        syncer = ftp.downloadFile
                        myargs = [os.path.basename(mypath),self.local_basedir]
                    elif self.remove:
                        syncer = ftp.deleteFile

                    counter += 1
                    tries = 0
                    done = False
                    lastrc = None
                    while tries < 5:
                        tries += 1
                        self.Entropy.updateProgress(
                            "[%s|#%s|(%s/%s)] %s: %s" % (
                                blue(crippled_uri),
                                darkgreen(str(tries)),
                                blue(str(counter)),
                                bold(str(maxcount)),
                                blue(action+"ing"),
                                red(os.path.basename(mypath)),
                            ),
                            importance = 0,
                            type = "info",
                            header = red(" @@ ")
                        )
                        rc = syncer(*myargs)
                        if rc and self.use_handlers and not self.download:
                            rc = self.handler_verify_upload(mypath, uri, ftp, counter, maxcount, action, tries)
                        if rc:
                            self.Entropy.updateProgress(
                                "[%s|#%s|(%s/%s)] %s %s: %s" % (
                                            blue(crippled_uri),
                                            darkgreen(str(tries)),
                                            blue(str(counter)),
                                            bold(str(maxcount)),
                                            blue(action),
                                            _("successful"),
                                            red(os.path.basename(mypath)),
                                ),
                                importance = 0,
                                type = "info",
                                header = darkgreen(" @@ ")
                            )
                            done = True
                            break
                        else:
                            self.Entropy.updateProgress(
                                "[%s|#%s|(%s/%s)] %s %s: %s" % (
                                            blue(crippled_uri),
                                            darkgreen(str(tries)),
                                            blue(str(counter)),
                                            bold(str(maxcount)),
                                            blue(action),
                                            brown(_("failed, retrying")),
                                            red(os.path.basename(mypath)),
                                    ),
                                importance = 0,
                                type = "warning",
                                header = brown(" @@ ")
                            )
                            lastrc = rc
                            continue

                    if not done:

                        self.Entropy.updateProgress(
                            "[%s|(%s/%s)] %s %s: %s - %s: %s" % (
                                    blue(crippled_uri),
                                    blue(str(counter)),
                                    bold(str(maxcount)),
                                    blue(action),
                                    darkred("failed, giving up"),
                                    red(os.path.basename(mypath)),
                                    _("error"),
                                    lastrc,
                            ),
                            importance = 1,
                            type = "error",
                            header = darkred(" !!! ")
                        )

                        if mypath not in self.critical_files:
                            self.Entropy.updateProgress(
                                "[%s|(%s/%s)] %s: %s, %s..." % (
                                    blue(crippled_uri),
                                    blue(str(counter)),
                                    bold(str(maxcount)),
                                    blue(_("not critical")),
                                    os.path.basename(mypath),
                                    blue(_("continuing")),
                                ),
                                importance = 1,
                                type = "warning",
                                header = brown(" @@ ")
                            )
                            continue

                        ftp.closeConnection()
                        errors = True
                        broken_uris.add((uri,lastrc))
                        # next mirror
                        break

                # close connection
                ftp.closeConnection()
                fine_uris.add(uri)

            return errors,fine_uris,broken_uris

    def _show_eapi3_upload_messages(self, crippled_uri, database_path, repo):
        self.Entropy.updateProgress(
            "[repo:%s|%s|%s:%s] %s" % (
                brown(repo),
                darkgreen(crippled_uri),
                red("EAPI"),
                bold("3"),
                blue(_("preparing uncompressed database for the upload")),
            ),
            importance = 0,
            type = "info",
            header = darkgreen(" * ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("database path"),blue(database_path),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )

    def _show_eapi2_upload_messages(self, crippled_uri, database_path, upload_data, cmethod, repo):

        if repo == None:
            repo = self.Entropy.default_repository

        self.Entropy.updateProgress(
            "[repo:%s|%s|%s:%s] %s" % (
                brown(repo),
                darkgreen(crippled_uri),
                red("EAPI"),
                bold("2"),
                blue(_("creating compressed database dump + checksum")),
            ),
            importance = 0,
            type = "info",
            header = darkgreen(" * ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("database path"),blue(database_path),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("dump"),blue(upload_data['dump_path']),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("dump checksum"),blue(upload_data['dump_path_digest']),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("opener"),blue(cmethod[0]),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )

    def _show_eapi1_upload_messages(self, crippled_uri, database_path, upload_data, cmethod, repo):

        self.Entropy.updateProgress(
            "[repo:%s|%s|%s:%s] %s" % (
                        brown(repo),
                        darkgreen(crippled_uri),
                        red("EAPI"),
                        bold("1"),
                        blue(_("compressing database + checksum")),
            ),
            importance = 0,
            type = "info",
            header = darkgreen(" * "),
            back = True
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("database path"),blue(database_path),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("compressed database path"),blue(upload_data['compressed_database_path']),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("compressed checksum"),blue(upload_data['compressed_database_path_digest']),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )
        self.Entropy.updateProgress(
            "%s: %s" % (_("opener"),blue(cmethod[0]),),
            importance = 0,
            type = "info",
            header = brown("    # ")
        )

    def create_mirror_directories(self, ftp_connection, path_to_create):
        bdir = ""
        for mydir in path_to_create.split("/"):
            bdir += "/"+mydir
            if not ftp_connection.isFileAvailable(bdir):
                try:
                    ftp_connection.mkdir(bdir)
                except Exception, e:
                    error = unicode(e)
                    if (error.find("550") == -1) and (error.find("File exist") == -1):
                        mytxt = "%s %s, %s: %s" % (
                            _("cannot create mirror directory"),
                            bdir,
                            _("error"),
                            e,
                        )
                        raise exceptionTools.OnlineMirrorError("OnlineMirrorError:  %s" % (mytxt,))

    def mirror_lock_check(self, uri, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        gave_up = False
        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
        ftp = self.FtpInterface(uri, self.Entropy)
        my_path = os.path.join(self.Entropy.get_remote_database_relative_path(repo),etpConst['branch'])
        ftp.setCWD(my_path, dodir = True)

        lock_file = self.get_database_lockfile(repo)
        if not os.path.isfile(lock_file) and ftp.isFileAvailable(os.path.basename(lock_file)):
            self.Entropy.updateProgress(
                red("[repo:%s|%s|%s] %s, %s" % (
                    repo,
                    crippled_uri,
                    _("locking"),
                    _("mirror already locked"),
                    _("waiting up to 2 minutes before giving up"),
                )
                ),
                importance = 1,
                type = "warning",
                header = brown(" * "),
                back = True
            )
            unlocked = False
            count = 0
            while count < 120:
                count += 1
                time.sleep(1)
                if not ftp.isFileAvailable(os.path.basename(lock_file)):
                    self.Entropy.updateProgress(
                        red("[repo:%s|%s|%s] %s !" % (
                                repo,
                                crippled_uri,
                                _("locking"),
                                _("mirror unlocked"),
                            )
                        ),
                        importance = 1,
                        type = "info",
                        header = darkgreen(" * ")
                    )
                    unlocked = True
                    break
            if not unlocked:
                gave_up = True

        ftp.closeConnection()
        return gave_up

    def shrink_database_and_close(self, repo = None):
        dbconn = self.Entropy.openServerDatabase(read_only = False, no_upload = True, repo = repo, indexing = False)
        dbconn.dropAllIndexes()
        dbconn.vacuum()
        dbconn.vacuum()
        dbconn.commitChanges()
        self.Entropy.close_server_database(dbconn)

    def sync_database_treeupdates(self, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository
        dbconn = self.Entropy.openServerDatabase(read_only = False, no_upload = True, repo = repo)
        # grab treeupdates from other databases and inject
        server_repos = etpConst['server_repositories'].keys()
        all_actions = set()
        for myrepo in server_repos:

            # avoid __default__
            if myrepo == etpConst['clientserverrepoid']:
                continue

            mydbc = self.Entropy.openServerDatabase(just_reading = True, repo = myrepo)
            actions = mydbc.listAllTreeUpdatesActions(no_ids_repos = True)
            for data in actions:
                all_actions.add(data)
            if not actions:
                continue
        backed_up_entries = dbconn.listAllTreeUpdatesActions()
        try:
            # clear first
            dbconn.removeTreeUpdatesActions(repo)
            dbconn.insertTreeUpdatesActions(all_actions,repo)
        except Exception, e:
            self.entropyTools.printTraceback()
            mytxt = "%s, %s: %s. %s" % (
                _("Troubles with treeupdates"),
                _("error"),
                e,
                _("Bumping old data back"),
            )
            self.Entropy.updateProgress(
                mytxt,
                importance = 1,
                type = "warning"
            )
            # restore previous data
            dbconn.bumpTreeUpdatesActions(backed_up_entries)

        dbconn.commitChanges()
        self.Entropy.close_server_database(dbconn)

    def upload_database(self, uris, lock_check = False, pretend = False, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        # doing some tests
        import gzip
        myt = type(gzip)
        del myt
        import bz2
        myt = type(bz2)
        del myt

        if etpConst['rss-feed']:
            self.update_rss_feed(repo = repo)

        upload_errors = False
        broken_uris = set()
        fine_uris = set()

        for uri in uris:

            cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
            if cmethod == None:
                raise exceptionTools.InvalidDataType("InvalidDataType: %s." % (
                        _("wrong database compression method passed"),
                    )
                )

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            database_path = self.Entropy.get_local_database_file(repo)
            upload_data, critical = self.get_files_to_sync(cmethod, repo = repo)

            if lock_check:
                given_up = self.mirror_lock_check(uri, repo = repo)
                if given_up:
                    upload_errors = True
                    broken_uris.add(uri)
                    continue

            self.lock_mirrors_for_download(True, [uri], repo = repo)

            self.Entropy.updateProgress(
                "[repo:%s|%s|%s] %s" % (
                    repo,
                    crippled_uri,
                    _("upload"),
                    _("preparing to upload database to mirror"),
                ),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )

            self.sync_database_treeupdates(repo)
            self.Entropy.update_database_package_sets(repo)
            self.Entropy.close_server_databases()

            # backup current database to avoid re-indexing
            old_dbpath = self.Entropy.get_local_database_file(repo)
            backup_dbpath = old_dbpath+".up_backup"
            copy_back = False
            if not pretend:
                try:
                    if os.path.isfile(backup_dbpath):
                        os.remove(backup_dbpath)
                    shutil.copy2(old_dbpath,backup_dbpath)
                    copy_back = True
                except:
                    pass

            self.shrink_database_and_close(repo)

            # EAPI 3
            self._show_eapi3_upload_messages(crippled_uri, database_path, repo)

            # EAPI 2
            self._show_eapi2_upload_messages(crippled_uri, database_path, upload_data, cmethod, repo)
            # create compressed dump + checksum
            self.dump_database_to_file(database_path, upload_data['dump_path'], eval(cmethod[0]), repo = repo)
            self.create_file_checksum(upload_data['dump_path'], upload_data['dump_path_digest'])

            # EAPI 1
            self._show_eapi1_upload_messages(crippled_uri, database_path, upload_data, cmethod, repo)
            # compress the database
            self.compress_file(database_path, upload_data['compressed_database_path'], eval(cmethod[0]))
            self.create_file_checksum(database_path, upload_data['compressed_database_path_digest'])

            if not pretend:
                # upload
                uploader = self.FileTransceiver(
                    self.FtpInterface,
                    self.Entropy,
                    [uri],
                    [upload_data[x] for x in upload_data],
                    critical_files = critical,
                    repo = repo
                )
                errors, m_fine_uris, m_broken_uris = uploader.go()
                if errors:
                    #my_fine_uris = sorted([self.entropyTools.extractFTPHostFromUri(x) for x in m_fine_uris])
                    my_broken_uris = sorted([(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris])
                    self.Entropy.updateProgress(
                        "[repo:%s|%s|%s] %s" % (
                            repo,
                            crippled_uri,
                            _("errors"),
                            _("failed to upload to mirror, not unlocking and continuing"),
                        ),
                        importance = 0,
                        type = "error",
                        header = darkred(" !!! ")
                    )
                    # get reason
                    reason = my_broken_uris[0][1]
                    self.Entropy.updateProgress(
                        blue("%s: %s" % (_("reason"),reason,)),
                        importance = 0,
                        type = "error",
                        header = blue("    # ")
                    )
                    upload_errors = True
                    broken_uris |= m_broken_uris
                    continue

                # copy db back
                if copy_back and os.path.isfile(backup_dbpath):
                    self.Entropy.close_server_databases()
                    further_backup_dbpath = old_dbpath+".security_backup"
                    if os.path.isfile(further_backup_dbpath):
                        os.remove(further_backup_dbpath)
                    shutil.copy2(old_dbpath,further_backup_dbpath)
                    shutil.move(backup_dbpath,old_dbpath)

            # unlock
            self.lock_mirrors_for_download(False,[uri], repo = repo)
            fine_uris |= m_fine_uris

        if not fine_uris:
            upload_errors = True
        return upload_errors, broken_uris, fine_uris


    def download_database(self, uris, lock_check = False, pretend = False, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        # doing some tests
        import gzip
        myt = type(gzip)
        del myt
        import bz2
        myt = type(bz2)
        del myt

        download_errors = False
        broken_uris = set()
        fine_uris = set()

        for uri in uris:

            cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
            if cmethod == None:
                raise exceptionTools.InvalidDataType("InvalidDataType: %s." % (
                        _("wrong database compression method passed"),
                    )
                )

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            database_path = self.Entropy.get_local_database_file(repo)
            database_dir_path = os.path.dirname(self.Entropy.get_local_database_file(repo))
            download_data, critical = self.get_files_to_sync(cmethod, download = True, repo = repo)
            mytmpdir = self.entropyTools.getRandomTempFile()
            os.makedirs(mytmpdir)

            self.Entropy.updateProgress(
                "[repo:%s|%s|%s] %s" % (
                    brown(repo),
                    darkgreen(crippled_uri),
                    red(_("download")),
                    blue(_("preparing to download database from mirror")),
                ),
                importance = 1,
                type = "info",
                header = darkgreen(" * ")
            )
            files_to_sync = sorted(download_data.keys())
            for myfile in files_to_sync:
                self.Entropy.updateProgress(
                    "%s: %s" % (blue(_("download path")),brown(unicode(download_data[myfile])),),
                    importance = 0,
                    type = "info",
                    header = brown("    # ")
                )

            if lock_check:
                given_up = self.mirror_lock_check(uri, repo = repo)
                if given_up:
                    download_errors = True
                    broken_uris.add(uri)
                    continue

            # avoid having others messing while we're downloading
            self.lock_mirrors(True,[uri], repo = repo)

            if not pretend:
                # download
                downloader = self.FileTransceiver(
                    self.FtpInterface, self.Entropy, [uri],
                    [download_data[x] for x in download_data], download = True,
                    local_basedir = mytmpdir, critical_files = critical, repo = repo
                )
                errors, m_fine_uris, m_broken_uris = downloader.go()
                if errors:
                    #my_fine_uris = sorted([self.entropyTools.extractFTPHostFromUri(x) for x in m_fine_uris])
                    my_broken_uris = sorted([(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris])
                    self.Entropy.updateProgress(
                        "[repo:%s|%s|%s] %s" % (
                            brown(repo),
                            darkgreen(crippled_uri),
                            red(_("errors")),
                            blue(_("failed to download from mirror")),
                        ),
                        importance = 0,
                        type = "error",
                        header = darkred(" !!! ")
                    )
                    # get reason
                    reason = my_broken_uris[0][1]
                    self.Entropy.updateProgress(
                        blue("%s: %s" % (_("reason"),reason,)),
                        importance = 0,
                        type = "error",
                        header = blue("    # ")
                    )
                    download_errors = True
                    broken_uris |= m_broken_uris
                    self.lock_mirrors(False,[uri], repo = repo)
                    continue

                # all fine then, we need to move data from mytmpdir to database_dir_path

                # EAPI 1
                # unpack database
                compressed_db_filename = os.path.basename(download_data['compressed_database_path'])
                uncompressed_db_filename = os.path.basename(database_path)
                compressed_file = os.path.join(mytmpdir,compressed_db_filename)
                uncompressed_file = os.path.join(mytmpdir,uncompressed_db_filename)
                self.entropyTools.uncompress_file(compressed_file, uncompressed_file, eval(cmethod[0]))
                # now move
                for myfile in os.listdir(mytmpdir):
                    fromfile = os.path.join(mytmpdir,myfile)
                    tofile = os.path.join(database_dir_path,myfile)
                    shutil.move(fromfile,tofile)
                    self.Entropy.ClientService.setup_default_file_perms(tofile)

            if os.path.isdir(mytmpdir):
                shutil.rmtree(mytmpdir)
            if os.path.isdir(mytmpdir):
                os.rmdir(mytmpdir)


            fine_uris.add(uri)
            self.lock_mirrors(False,[uri], repo = repo)

        return download_errors, fine_uris, broken_uris

    def calculate_database_sync_queues(self, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        remote_status =  self.get_remote_databases_status(repo)
        local_revision = self.Entropy.get_local_database_revision(repo)
        upload_queue = []
        download_latest = ()

        # all mirrors are empty ? I rule
        if not [x for x in remote_status if x[1]]:
            upload_queue = remote_status[:]
        else:
            highest_remote_revision = max([x[1] for x in remote_status])

            if local_revision < highest_remote_revision:
                for x in remote_status:
                    if x[1] == highest_remote_revision:
                        download_latest = x
                        break

            if download_latest:
                upload_queue = [x for x in remote_status if (x[1] < highest_remote_revision)]
            else:
                upload_queue = [x for x in remote_status if (x[1] < local_revision)]

        return download_latest, upload_queue

    def sync_databases(self, no_upload = False, unlock_mirrors = False, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        while 1:

            db_locked = False
            if self.is_local_database_locked(repo):
                db_locked = True

            lock_data = self.get_mirrors_lock(repo)
            mirrors_locked = [x for x in lock_data if x[1]]

            if not mirrors_locked and db_locked:
                # mirrors not locked remotely but only locally
                mylock_file = self.get_database_lockfile(repo)
                if os.path.isfile(mylock_file) and os.access(mylock_file,os.W_OK):
                    os.remove(mylock_file)
                    continue

            break

        if mirrors_locked and not db_locked:
            mytxt = "%s, %s %s" % (
                _("At the moment, mirrors are locked, someone is working on their databases"),
                _("try again later"),
                "...",
            )
            raise exceptionTools.OnlineMirrorError("OnlineMirrorError: %s" % (mytxt,))

        download_latest, upload_queue = self.calculate_database_sync_queues(repo)

        if not download_latest and not upload_queue:
            self.Entropy.updateProgress(
                "[repo:%s|%s] %s" % (
                    brown(repo),
                    red(_("sync")), # something short please
                    blue(_("database already in sync")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            return 0, set(), set()

        if download_latest:
            download_uri = download_latest[0]
            download_errors, fine_uris, broken_uris = self.download_database([download_uri], repo = repo)
            if download_errors:
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s: %s" % (
                        brown(repo),
                        red(_("sync")),
                        blue(_("database sync failed")),
                        red(_("download issues")),
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )
                return 1,fine_uris,broken_uris
            # XXX: reload revision settings?

        if upload_queue and not no_upload:

            deps_not_found = self.Entropy.dependencies_test()
            if deps_not_found and not self.Entropy.community_repo:
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s: %s" % (
                        brown(repo),
                        red(_("sync")),
                        blue(_("database sync forbidden")),
                        red(_("dependencies_test() reported errors")),
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )
                return 3,set(),set()

            problems = self.Entropy.check_config_file_updates()
            if problems:
                return 4,set(),set()

            self.Entropy.updateProgress(
                "[repo:%s|%s] %s" % (
                    brown(repo),
                    red(_("config files")), # something short please
                    blue(_("no configuration files to commit. All fine.")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ "),
                back = True
            )
            #             for x in scandata:
            #    

            uris = [x[0] for x in upload_queue]
            errors, fine_uris, broken_uris = self.upload_database(uris, repo = repo)
            if errors:
                self.Entropy.updateProgress(
                    "[repo:%s|%s] %s: %s" % (
                        brown(repo),
                        red(_("sync")),
                        blue(_("database sync failed")),
                        red(_("upload issues")),
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )
                return 2,fine_uris,broken_uris


        self.Entropy.updateProgress(
            "[repo:%s|%s] %s" % (
                brown(repo),
                red(_("sync")),
                blue(_("database sync completed successfully")),
            ),
            importance = 1,
            type = "info",
            header = darkgreen(" * ")
        )

        if unlock_mirrors:
            self.lock_mirrors(False, repo = repo)
        return 0, set(), set()


    def calculate_local_upload_files(self, branch, repo = None):
        upload_files = 0
        upload_packages = set()
        upload_dir = os.path.join(self.Entropy.get_local_upload_directory(repo),branch)

        for package in os.listdir(upload_dir):
            if package.endswith(etpConst['packagesext']) or package.endswith(etpConst['packageshashfileext']):
                upload_packages.add(package)
                if package.endswith(etpConst['packagesext']):
                    upload_files += 1

        return upload_files, upload_packages

    def calculate_local_package_files(self, branch, repo = None):
        local_files = 0
        local_packages = set()
        packages_dir = os.path.join(self.Entropy.get_local_packages_directory(repo),branch)

        if not os.path.isdir(packages_dir):
            os.makedirs(packages_dir)

        for package in os.listdir(packages_dir):
            if package.endswith(etpConst['packagesext']) or package.endswith(etpConst['packageshashfileext']):
                local_packages.add(package)
                if package.endswith(etpConst['packagesext']):
                    local_files += 1

        return local_files, local_packages


    def _show_local_sync_stats(self, upload_files, local_files):
        self.Entropy.updateProgress(
            "%s:" % ( blue(_("Local statistics")),),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )
        self.Entropy.updateProgress(
            red("%s:\t\t%s %s" % (
                    blue(_("upload directory")),
                    bold(str(upload_files)),
                    red(_("files ready")),
                )
            ),
            importance = 0,
            type = "info",
            header = red(" @@ ")
        )
        self.Entropy.updateProgress(
            red("%s:\t\t%s %s" % (
                    blue(_("packages directory")),
                    bold(str(local_files)),
                    red(_("files ready")),
                )
            ),
            importance = 0,
            type = "info",
            header = red(" @@ ")
        )

    def _show_sync_queues(self, upload, download, removal, copy, metainfo, branch):

        # show stats
        for itemdata in upload:
            package = darkgreen(os.path.basename(itemdata[0]))
            size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
            self.Entropy.updateProgress(
                "[branch:%s|%s] %s [%s]" % (
                    brown(branch),
                    blue(_("upload")),
                    darkgreen(package),
                    size,
                ),
                importance = 0,
                type = "info",
                header = red("    # ")
            )
        for itemdata in download:
            package = darkred(os.path.basename(itemdata[0]))
            size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
            self.Entropy.updateProgress(
                "[branch:%s|%s] %s [%s]" % (
                    brown(branch),
                    darkred(_("download")),
                    blue(package),
                    size,
                ),
                importance = 0,
                type = "info",
                header = red("    # ")
            )
        for itemdata in copy:
            package = darkblue(os.path.basename(itemdata[0]))
            size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
            self.Entropy.updateProgress(
                "[branch:%s|%s] %s [%s]" % (
                    brown(branch),
                    darkgreen(_("copy")),
                    brown(package),
                    size,
                ),
                importance = 0,
                type = "info",
                header = red("    # ")
            )
        for itemdata in removal:
            package = brown(os.path.basename(itemdata[0]))
            size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
            self.Entropy.updateProgress(
                "[branch:%s|%s] %s [%s]" % (
                    brown(branch),
                    red(_("remove")),
                    red(package),
                    size,
                ),
                importance = 0,
                type = "info",
                header = red("    # ")
            )

        self.Entropy.updateProgress(
            "%s:\t\t\t%s" % (
                blue(_("Packages to be removed")),
                darkred(str(len(removal))),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )
        self.Entropy.updateProgress(
            "%s:\t\t%s" % (
                darkgreen(_("Packages to be moved locally")),
                darkgreen(str(len(copy))),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )
        self.Entropy.updateProgress(
            "%s:\t\t\t%s" % (
                bold(_("Packages to be uploaded")),
                bold(str(len(upload))),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )

        self.Entropy.updateProgress(
            "%s:\t\t\t%s" % (
                darkred(_("Total removal size")),
                darkred(self.entropyTools.bytesIntoHuman(metainfo['removal'])),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )

        self.Entropy.updateProgress(
            "%s:\t\t\t%s" % (
                blue(_("Total upload size")),
                blue(self.entropyTools.bytesIntoHuman(metainfo['upload'])),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )
        self.Entropy.updateProgress(
            "%s:\t\t\t%s" % (
                brown(_("Total download size")),
                brown(self.entropyTools.bytesIntoHuman(metainfo['download'])),
            ),
            importance = 0,
            type = "info",
            header = blue(" @@ ")
        )


    def calculate_remote_package_files(self, uri, branch, ftp_connection = None, repo = None):

        remote_files = 0
        close_conn = False
        remote_packages_data = {}

        my_path = os.path.join(self.Entropy.get_remote_packages_relative_path(repo),branch)
        if ftp_connection == None:
            close_conn = True
            ftp_connection = self.FtpInterface(uri, self.Entropy)
        ftp_connection.setCWD(my_path, dodir = True)

        remote_packages = ftp_connection.listDir()
        remote_packages_info = ftp_connection.getRoughList()
        if close_conn:
            ftp_connection.closeConnection()

        for tbz2 in remote_packages:
            if tbz2.endswith(etpConst['packagesext']):
                remote_files += 1

        for remote_package in remote_packages_info:
            remote_packages_data[remote_package.split()[8]] = int(remote_package.split()[4])

        return remote_files, remote_packages, remote_packages_data

    def calculate_packages_to_sync(self, uri, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
        upload_files, upload_packages = self.calculate_local_upload_files(branch, repo)
        local_files, local_packages = self.calculate_local_package_files(branch, repo)
        self._show_local_sync_stats(upload_files, local_files)

        self.Entropy.updateProgress(
            "%s: %s" % (blue(_("Remote statistics for")),red(crippled_uri),),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )
        remote_files, remote_packages, remote_packages_data = self.calculate_remote_package_files(
            uri,
            branch,
            repo = repo
        )
        self.Entropy.updateProgress(
            "%s:\t\t\t%s %s" % (
                blue(_("remote packages")),
                bold(str(remote_files)),
                red(_("files stored")),
            ),
            importance = 0,
            type = "info",
            header = red(" @@ ")
        )

        mytxt = blue("%s ...") % (_("Calculating queues"),)
        self.Entropy.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        uploadQueue, downloadQueue, removalQueue, fineQueue = self.calculate_sync_queues(
                upload_packages,
                local_packages,
                remote_packages,
                remote_packages_data,
                branch,
                repo
        )
        return uploadQueue, downloadQueue, removalQueue, fineQueue, remote_packages_data

    def calculate_sync_queues(
            self,
            upload_packages,
            local_packages,
            remote_packages,
            remote_packages_data,
            branch,
            repo = None
        ):

        uploadQueue = set()
        downloadQueue = set()
        removalQueue = set()
        fineQueue = set()

        for local_package in upload_packages:
            if local_package in remote_packages:
                local_filepath = os.path.join(self.Entropy.get_local_upload_directory(repo),branch,local_package)
                local_size = self.entropyTools.get_file_size(local_filepath)
                remote_size = remote_packages_data.get(local_package)
                if remote_size == None:
                    remote_size = 0
                if (local_size != remote_size):
                    # size does not match, adding to the upload queue
                    uploadQueue.add(local_package)
                else:
                    fineQueue.add(local_package) # just move from upload to packages
            else:
                # always force upload of packages in uploaddir
                uploadQueue.add(local_package)

        # if a package is in the packages directory but not online, we have to upload it
        # we have local_packages and remotePackages
        for local_package in local_packages:
            if local_package in remote_packages:
                local_filepath = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,local_package)
                local_size = self.entropyTools.get_file_size(local_filepath)
                remote_size = remote_packages_data.get(local_package)
                if remote_size == None:
                    remote_size = 0
                if (local_size != remote_size) and (local_size != 0):
                    # size does not match, adding to the upload queue
                    if local_package not in fineQueue:
                        uploadQueue.add(local_package)
            else:
                # this means that the local package does not exist
                # so, we need to download it
                uploadQueue.add(local_package)

        # Fill downloadQueue and removalQueue
        for remote_package in remote_packages:
            if remote_package in local_packages:
                local_filepath = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,remote_package)
                local_size = self.entropyTools.get_file_size(local_filepath)
                remote_size = remote_packages_data.get(remote_package)
                if remote_size == None:
                    remote_size = 0
                if (local_size != remote_size) and (local_size != 0):
                    # size does not match, remove first
                    if remote_package not in uploadQueue: # do it only if the package has not been added to the uploadQueue
                        removalQueue.add(remote_package) # remotePackage == localPackage # just remove something that differs from the content of the mirror
                        # then add to the download queue
                        downloadQueue.add(remote_package)
            else:
                # this means that the local package does not exist
                # so, we need to download it
                if not remote_package.endswith(".tmp"): # ignore .tmp files
                    downloadQueue.add(remote_package)

        # Collect packages that don't exist anymore in the database
        # so we can filter them out from the download queue
        dbconn = self.Entropy.openServerDatabase(just_reading = True, repo = repo)
        db_files = dbconn.listBranchPackagesTbz2(branch, do_sort = False, full_path = True)
        db_files = set([os.path.basename(x) for x in db_files if (self.Entropy.get_branch_from_download_relative_uri(x) == branch)])

        exclude = set()
        for myfile in downloadQueue:
            if myfile.endswith(etpConst['packagesext']):
                if myfile not in db_files:
                    exclude.add(myfile)
        downloadQueue -= exclude

        exclude = set()
        for myfile in uploadQueue:
            if myfile.endswith(etpConst['packagesext']):
                if myfile not in db_files:
                    exclude.add(myfile)
        uploadQueue -= exclude

        exclude = set()
        for myfile in downloadQueue:
            if myfile in uploadQueue:
                exclude.add(myfile)
        downloadQueue -= exclude

        return uploadQueue, downloadQueue, removalQueue, fineQueue


    def expand_queues(self, uploadQueue, downloadQueue, removalQueue, remote_packages_data, branch, repo):

        metainfo = {
            'removal': 0,
            'download': 0,
            'upload': 0,
        }
        removal = []
        download = []
        copy = []
        upload = []

        for item in removalQueue:
            if not item.endswith(etpConst['packagesext']):
                continue
            local_filepath = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,item)
            size = self.entropyTools.get_file_size(local_filepath)
            metainfo['removal'] += size
            removal.append((local_filepath,size))

        for item in downloadQueue:
            if not item.endswith(etpConst['packagesext']):
                continue
            local_filepath = os.path.join(self.Entropy.get_local_upload_directory(repo),branch,item)
            if not os.path.isfile(local_filepath):
                size = remote_packages_data.get(item)
                if size == None:
                    size = 0
                size = int(size)
                metainfo['removal'] += size
                download.append((local_filepath,size))
            else:
                size = self.entropyTools.get_file_size(local_filepath)
                copy.append((local_filepath,size))

        for item in uploadQueue:
            if not item.endswith(etpConst['packagesext']):
                continue
            local_filepath = os.path.join(self.Entropy.get_local_upload_directory(repo),branch,item)
            local_filepath_pkgs = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,item)
            if os.path.isfile(local_filepath):
                size = self.entropyTools.get_file_size(local_filepath)
                upload.append((local_filepath,size))
            else:
                size = self.entropyTools.get_file_size(local_filepath_pkgs)
                upload.append((local_filepath_pkgs,size))
            metainfo['upload'] += size


        return upload, download, removal, copy, metainfo


    def _sync_run_removal_queue(self, removal_queue, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        for itemdata in removal_queue:

            remove_filename = itemdata[0]
            remove_filepath = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,remove_filename)
            remove_filepath_hash = remove_filepath+etpConst['packageshashfileext']
            self.Entropy.updateProgress(
                "[repo:%s|%s|%s] %s: %s [%s]" % (
                        brown(repo),
                        red("sync"),
                        brown(branch),
                        blue(_("removing package+hash")),
                        darkgreen(remove_filename),
                        blue(self.entropyTools.bytesIntoHuman(itemdata[1])),
                ),
                importance = 0,
                type = "info",
                header = darkred(" * ")
            )

            if os.path.isfile(remove_filepath):
                os.remove(remove_filepath)
            if os.path.isfile(remove_filepath_hash):
                os.remove(remove_filepath_hash)

        self.Entropy.updateProgress(
            "[repo:%s|%s|%s] %s" % (
                    brown(repo),
                    red(_("sync")),
                    brown(branch),
                    blue(_("removal complete")),
            ),
            importance = 0,
            type = "info",
            header = darkred(" * ")
        )


    def _sync_run_copy_queue(self, copy_queue, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        for itemdata in copy_queue:

            from_file = itemdata[0]
            from_file_hash = from_file+etpConst['packageshashfileext']
            to_file = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,os.path.basename(from_file))
            to_file_hash = to_file+etpConst['packageshashfileext']
            expiration_file = to_file+etpConst['packagesexpirationfileext']
            self.Entropy.updateProgress(
                "[repo:%s|%s|%s] %s: %s" % (
                        brown(repo),
                        red("sync"),
                        brown(branch),
                        blue(_("copying file+hash to repository")),
                        darkgreen(from_file),
                ),
                importance = 0,
                type = "info",
                header = darkred(" * ")
            )

            if not os.path.isdir(os.path.dirname(to_file)):
                os.makedirs(os.path.dirname(to_file))

            shutil.copy2(from_file,to_file)
            if not os.path.isfile(from_file_hash):
                self.create_file_checksum(from_file, from_file_hash)
            shutil.copy2(from_file_hash,to_file_hash)

            # clear expiration file
            if os.path.isfile(expiration_file):
                os.remove(expiration_file)


    def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
        myqueue = []
        for itemdata in upload_queue:
            x = itemdata[0]
            hash_file = x+etpConst['packageshashfileext']
            if not os.path.isfile(hash_file):
                self.entropyTools.createHashFile(x)
            myqueue.append(hash_file)
            myqueue.append(x)

        ftp_basedir = os.path.join(self.Entropy.get_remote_packages_relative_path(repo),branch)
        uploader = self.FileTransceiver(    self.FtpInterface,
                                            self.Entropy,
                                            [uri],
                                            myqueue,
                                            critical_files = myqueue,
                                            use_handlers = True,
                                            ftp_basedir = ftp_basedir,
                                            handlers_data = {'branch': branch },
                                            repo = repo
                                        )
        errors, m_fine_uris, m_broken_uris = uploader.go()
        if errors:
            my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
            reason = my_broken_uris[0][1]
            self.Entropy.updateProgress(
                "[branch:%s] %s: %s, %s: %s" % (
                            brown(branch),
                            blue(_("upload errors")),
                            red(crippled_uri),
                            blue(_("reason")),
                            darkgreen(unicode(reason)),
                ),
                importance = 1,
                type = "error",
                header = darkred(" !!! ")
            )
            return errors, m_fine_uris, m_broken_uris

        self.Entropy.updateProgress(
            "[branch:%s] %s: %s" % (
                        brown(branch),
                        blue(_("upload completed successfully")),
                        red(crippled_uri),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )
        return errors, m_fine_uris, m_broken_uris


    def _sync_run_download_queue(self, uri, download_queue, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
        myqueue = []
        for itemdata in download_queue:
            x = itemdata[0]
            hash_file = x+etpConst['packageshashfileext']
            myqueue.append(x)
            myqueue.append(hash_file)

        ftp_basedir = os.path.join(self.Entropy.get_remote_packages_relative_path(repo),branch)
        local_basedir = os.path.join(self.Entropy.get_local_packages_directory(repo),branch)
        downloader = self.FileTransceiver(
            self.FtpInterface,
            self.Entropy,
            [uri],
            myqueue,
            critical_files = myqueue,
            use_handlers = True,
            ftp_basedir = ftp_basedir,
            local_basedir = local_basedir,
            handlers_data = {'branch': branch },
            download = True,
            repo = repo
        )
        errors, m_fine_uris, m_broken_uris = downloader.go()
        if errors:
            my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
            reason = my_broken_uris[0][1]
            self.Entropy.updateProgress(
                "[repo:%s|%s|%s] %s: %s, %s: %s" % (
                    brown(repo),
                    red(_("sync")),
                    brown(branch),
                    blue(_("download errors")),
                    darkgreen(crippled_uri),
                    blue(_("reason")),
                    reason,
                ),
                importance = 1,
                type = "error",
                header = darkred(" !!! ")
            )
            return errors, m_fine_uris, m_broken_uris

        self.Entropy.updateProgress(
            "[repo:%s|%s|%s] %s: %s" % (
                brown(repo),
                red(_("sync")),
                brown(branch),
                blue(_("download completed successfully")),
                darkgreen(crippled_uri),
            ),
            importance = 1,
            type = "info",
            header = darkgreen(" * ")
        )
        return errors, m_fine_uris, m_broken_uris


    def sync_packages(self, ask = True, pretend = False, packages_check = False, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        self.Entropy.updateProgress(
            "[repo:%s|%s] %s" % (
                repo,
                red(_("sync")),
                darkgreen(_("starting packages sync")),
            ),
            importance = 1,
            type = "info",
            header = red(" @@ "),
            back = True
        )

        successfull_mirrors = set()
        broken_mirrors = set()
        check_data = ()
        mirrors_tainted = False
        mirror_errors = False
        mirrors_errors = False

        for uri in self.Entropy.get_remote_mirrors(repo):

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            mirror_errors = False

            self.Entropy.updateProgress(
                "[repo:%s|%s|branch:%s] %s: %s" % (
                    repo,
                    red(_("sync")),
                    brown(etpConst['branch']),
                    blue(_("packages sync")),
                    bold(crippled_uri),
                ),
                importance = 1,
                type = "info",
                header = red(" @@ ")
            )

            try:
                uploadQueue, downloadQueue, removalQueue, fineQueue, remote_packages_data = self.calculate_packages_to_sync(uri, etpConst['branch'], repo)
                del fineQueue
            except self.socket.error, e:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|branch:%s] %s: %s, %s %s" % (
                        repo,
                        red(_("sync")),
                        etpConst['branch'],
                        darkred(_("socket error")),
                        e,
                        darkred(_("on")),
                        crippled_uri,
                    ),
                    importance = 1,
                    type = "error",
                    header = darkgreen(" * ")
                )
                continue

            if (not uploadQueue) and (not downloadQueue) and (not removalQueue):
                self.Entropy.updateProgress(
                    "[repo:%s|%s|branch:%s] %s: %s" % (
                        repo,
                        red(_("sync")),
                        etpConst['branch'],
                        darkgreen(_("nothing to do on")),
                        crippled_uri,
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                successfull_mirrors.add(uri)
                continue

            self.Entropy.updateProgress(
                "%s:" % (blue(_("Expanding queues")),),
                importance = 1,
                type = "info",
                header = red(" ** ")
            )

            upload, download, removal, copy, metainfo = self.expand_queues(
                        uploadQueue,
                        downloadQueue,
                        removalQueue,
                        remote_packages_data,
                        etpConst['branch'],
                        repo
            )
            del uploadQueue, downloadQueue, removalQueue, remote_packages_data
            self._show_sync_queues(upload, download, removal, copy, metainfo, etpConst['branch'])

            if not len(upload)+len(download)+len(removal)+len(copy):

                self.Entropy.updateProgress(
                    "[repo:%s|%s|branch:%s] %s %s" % (
                        self.Entropy.default_repository,
                        red(_("sync")),
                        etpConst['branch'],
                        blue(_("nothing to sync for")),
                        crippled_uri,
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" @@ ")
                )

                successfull_mirrors.add(uri)
                continue

            if pretend:
                successfull_mirrors.add(uri)
                continue

            if ask:
                rc = self.Entropy.askQuestion(_("Would you like to run the steps above ?"))
                if rc == "No":
                    continue

            try:

                if removal:
                    self._sync_run_removal_queue(removal, etpConst['branch'], repo)
                if copy:
                    self._sync_run_copy_queue(copy, etpConst['branch'], repo)
                if upload or download:
                    mirrors_tainted = True
                if upload:
                    d_errors, m_fine_uris, m_broken_uris = self._sync_run_upload_queue(uri, upload, etpConst['branch'], repo)
                    if d_errors: mirror_errors = True
                if download:
                    d_errors, m_fine_uris, m_broken_uris = self._sync_run_download_queue(uri, download, etpConst['branch'], repo)
                    if d_errors: mirror_errors = True
                if not mirror_errors:
                    successfull_mirrors.add(uri)
                else:
                    mirrors_errors = True

            except KeyboardInterrupt:
                self.Entropy.updateProgress(
                    "[repo:%s|%s|branch:%s] %s" % (
                        repo,
                        red(_("sync")),
                        etpConst['branch'],
                        darkgreen(_("keyboard interrupt !")),
                    ),
                    importance = 1,
                    type = "info",
                    header = darkgreen(" * ")
                )
                continue

            except Exception, e:

                self.entropyTools.printTraceback()
                mirrors_errors = True
                broken_mirrors.add(uri)
                self.Entropy.updateProgress(
                    "[repo:%s|%s|branch:%s] %s: %s, %s: %s" % (
                        repo,
                        red(_("sync")),
                        etpConst['branch'],
                        darkred(_("exception caught")),
                        Exception,
                        _("error"),
                        e,
                    ),
                    importance = 1,
                    type = "error",
                    header = darkred(" !!! ")
                )

                exc_txt = self.Entropy.entropyTools.printException(returndata = True)
                for line in exc_txt:
                    self.Entropy.updateProgress(
                        unicode(line),
                        importance = 1,
                        type = "error",
                        header = darkred(":  ")
                    )

                if len(successfull_mirrors) > 0:
                    self.Entropy.updateProgress(
                        "[repo:%s|%s|branch:%s] %s" % (
                            repo,
                            red(_("sync")),
                            etpConst['branch'],
                            darkred(_("at least one mirror has been sync'd properly, hooray!")),
                        ),
                        importance = 1,
                        type = "error",
                        header = darkred(" !!! ")
                    )
                continue

        # if at least one server has been synced successfully, move files
        if (len(successfull_mirrors) > 0) and not pretend:
            self.remove_expiration_files(etpConst['branch'], repo)

        if packages_check:
            check_data = self.Entropy.verify_local_packages([], ask = ask, repo = repo)

        return mirrors_tainted, mirrors_errors, successfull_mirrors, broken_mirrors, check_data

    def remove_expiration_files(self, branch, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        branch_dir = os.path.join(self.Entropy.get_local_upload_directory(repo),branch)
        branchcontent = os.listdir(branch_dir)
        for xfile in branchcontent:
            source = os.path.join(self.Entropy.get_local_upload_directory(repo),branch,xfile)
            destdir = os.path.join(self.Entropy.get_local_packages_directory(repo),branch)
            if not os.path.isdir(destdir):
                os.makedirs(destdir)
            dest = os.path.join(destdir,xfile)
            shutil.move(source,dest)
            # clear expiration file
            dest_expiration = dest+etpConst['packagesexpirationfileext']
            if os.path.isfile(dest_expiration):
                os.remove(dest_expiration)


    def is_package_expired(self, package_file, branch, repo = None):
        pkg_path = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,package_file)
        pkg_path += etpConst['packagesexpirationfileext']
        if not os.path.isfile(pkg_path):
            return False
        mtime = self.entropyTools.getFileUnixMtime(pkg_path)
        delta = int(etpConst['packagesexpirationdays'])*24*3600
        currmtime = time.time()
        file_delta = currmtime - mtime
        if file_delta > delta:
            return True
        return False

    def create_expiration_file(self, package_file, branch, repo = None, gentle = False):
        pkg_path = os.path.join(self.Entropy.get_local_packages_directory(repo),branch,package_file)
        pkg_path += etpConst['packagesexpirationfileext']
        if gentle and os.path.isfile(pkg_path):
            return
        f = open(pkg_path,"w")
        f.flush()
        f.close()


    def collect_expiring_packages(self, branch, repo = None):
        dbconn = self.Entropy.openServerDatabase(just_reading = True, repo = repo)
        database_bins = dbconn.listBranchPackagesTbz2(branch, do_sort = False, full_path = True)
        bins_dir = os.path.join(self.Entropy.get_local_packages_directory(repo),branch)
        repo_bins = set()
        if os.path.isdir(bins_dir):
            repo_bins = os.listdir(bins_dir)
            repo_bins = set([os.path.join('packages',etpSys['arch'],branch,x) for x in repo_bins if x.endswith(etpConst['packagesext'])])
        repo_bins -= database_bins
        return set([os.path.basename(x) for x in repo_bins])


    def tidy_mirrors(self, ask = True, pretend = False, repo = None):

        if repo == None:
            repo = self.Entropy.default_repository

        self.Entropy.updateProgress(
            "[repo:%s|%s|branch:%s] %s" % (
                brown(repo),
                red(_("tidy")),
                blue(etpConst['branch']),
                blue(_("collecting expired packages")),
            ),
            importance = 1,
            type = "info",
            header = red(" @@ ")
        )

        branch_data = {}
        errors = False

        branch_data['errors'] = False

        if etpConst['branch'] != etpConst['branches'][-1]:
            self.Entropy.updateProgress(
                "[branch:%s] %s" % (
                    brown(etpConst['branch']),
                    blue(_("not the latest branch, skipping tidy for consistence. This branch entered the maintenance mode.")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            branch_data['errors'] = True
            return True, branch_data

        self.Entropy.updateProgress(
            "[branch:%s] %s" % (
                brown(etpConst['branch']),
                blue(_("collecting expired packages in the selected branches")),
            ),
            importance = 1,
            type = "info",
            header = blue(" @@ ")
        )

        # collect removed packages
        expiring_packages = self.collect_expiring_packages(etpConst['branch'], repo)

        removal = []
        for package in expiring_packages:
            expired = self.is_package_expired(package, etpConst['branch'], repo)
            if expired:
                removal.append(package)
            else:
                self.create_expiration_file(package, etpConst['branch'], repo, gentle = True)

        # fill returning data
        branch_data['removal'] = removal[:]

        if not removal:
            self.Entropy.updateProgress(
                "[branch:%s] %s" % (
                        brown(etpConst['branch']),
                        blue(_("nothing to remove on this branch")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            return errors, branch_data
        else:
            self.Entropy.updateProgress(
                "[branch:%s] %s:" % (
                    brown(etpConst['branch']),
                    blue(_("these are the expired packages")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )
            for package in removal:
                self.Entropy.updateProgress(
                    "[branch:%s] %s: %s" % (
                                brown(etpConst['branch']),
                                blue(_("remove")),
                                darkgreen(package),
                        ),
                    importance = 1,
                    type = "info",
                    header = brown("    # ")
                )

        if pretend:
            return errors,branch_data

        if ask:
            rc = self.Entropy.askQuestion(_("Would you like to continue ?"))
            if rc == "No":
                return errors, branch_data

        myqueue = []
        for package in removal:
            myqueue.append(package+etpConst['packageshashfileext'])
            myqueue.append(package)
        ftp_basedir = os.path.join(self.Entropy.get_remote_packages_relative_path(repo),etpConst['branch'])
        for uri in self.Entropy.get_remote_mirrors(repo):

            self.Entropy.updateProgress(
                "[branch:%s] %s..." % (
                    brown(etpConst['branch']),
                    blue(_("removing packages remotely")),
                ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )

            crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
            destroyer = self.FileTransceiver(
                self.FtpInterface,
                self.Entropy,
                [uri],
                myqueue,
                critical_files = [],
                ftp_basedir = ftp_basedir,
                remove = True,
                repo = repo
            )
            errors, m_fine_uris, m_broken_uris = destroyer.go()
            if errors:
                my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
                reason = my_broken_uris[0][1]
                self.Entropy.updateProgress(
                    "[branch:%s] %s: %s, %s: %s" % (
                        brown(etpConst['branch']),
                        blue(_("remove errors")),
                        red(crippled_uri),
                        blue(_("reason")),
                        reason,
                    ),
                    importance = 1,
                    type = "warning",
                    header = brown(" !!! ")
                )
                branch_data['errors'] = True
                errors = True

            self.Entropy.updateProgress(
                "[branch:%s] %s..." % (
                        brown(etpConst['branch']),
                        blue(_("removing packages locally")),
                    ),
                importance = 1,
                type = "info",
                header = blue(" @@ ")
            )

            branch_data['removed'] = set()
            for package in removal:
                package_path = os.path.join(self.Entropy.get_local_packages_directory(repo),etpConst['branch'],package)
                package_path_hash = package_path+etpConst['packageshashfileext']
                package_path_expired = package_path+etpConst['packagesexpirationfileext']
                for myfile in [package_path_hash,package_path,package_path_expired]:
                    if os.path.isfile(myfile):
                        self.Entropy.updateProgress(
                            "[branch:%s] %s: %s" % (
                                        brown(etpConst['branch']),
                                        blue(_("removing")),
                                        darkgreen(myfile),
                                ),
                            importance = 1,
                            type = "info",
                            header = brown(" @@ ")
                        )
                        os.remove(myfile)
                        branch_data['removed'].add(myfile)


        return errors, branch_data


class EntropyDatabaseInterface:

    import entropyTools, dumpTools, threading
    def __init__(
            self,
            readOnly = False,
            noUpload = False,
            dbFile = None,
            clientDatabase = False,
            xcache = False,
            dbname = etpConst['serverdbid'],
            indexing = True,
            OutputInterface = None,
            ServiceInterface = None,
            skipChecks = False, # dangerous!
            useBranch = None,
            lockRemote = True
        ):

        self.dbMatchCacheKey = etpCache['dbMatch']
        self.dbSearchCacheKey = etpCache['dbSearch']
        self.dbname = dbname
        self.lockRemote = lockRemote
        self.db_branch = etpConst['branch']
        if self.dbname == etpConst['clientdbid']:
            self.db_branch = None
        if useBranch != None: self.db_branch = useBranch

        if OutputInterface == None:
            OutputInterface = TextInterface()

        if dbFile == None:
            raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (_("valid database path needed"),) )

        self.WriteLock = self.threading.Lock()
        self.dbapi2 = dbapi2
        # setup output interface
        self.OutputInterface = OutputInterface
        self.updateProgress = self.OutputInterface.updateProgress
        self.askQuestion = self.OutputInterface.askQuestion
        # setup service interface
        self.ServiceInterface = ServiceInterface
        self.readOnly = readOnly
        self.noUpload = noUpload
        self.clientDatabase = clientDatabase
        self.xcache = xcache
        self.indexing = indexing
        self.skipChecks = skipChecks
        if not self.skipChecks:
            if not self.entropyTools.is_user_in_entropy_group():
                # forcing since we won't have write access to db
                self.indexing = False
            # live systems don't like wasting RAM
            if self.entropyTools.islive():
                self.indexing = False
        self.dbFile = dbFile
        self.dbclosed = True
        self.server_repo = None

        if not self.clientDatabase:
            self.server_repo = self.dbname[len(etpConst['serverdbid']):]
            self.create_dbstatus_data()

        if not self.skipChecks:
            # no caching for non root and server connections
            if (self.dbname.startswith(etpConst['serverdbid'])) or (not self.entropyTools.is_user_in_entropy_group()):
                self.xcache = False
        self.live_cache = {}

        # create connection
        self.connection = self.dbapi2.connect(dbFile,timeout=300.0, check_same_thread = False)
        self.cursor = self.connection.cursor()

        if not self.skipChecks:
            if os.access(self.dbFile,os.W_OK) and self.doesTableExist('baseinfo') and self.doesTableExist('extrainfo'):
                if self.entropyTools.islive():
                    # check where's the file
                    if etpConst['systemroot']:
                        self.databaseStructureUpdates()
                else:
                    self.databaseStructureUpdates()


        # now we can set this to False
        self.dbclosed = False

    def setCacheSize(self, size):
        self.cursor.execute('PRAGMA cache_size = '+str(size))

    def setDefaultCacheSize(self, size):
        self.cursor.execute('PRAGMA default_cache_size = '+str(size))


    def __del__(self):
        if not self.dbclosed:
            self.closeDB()

    def create_dbstatus_data(self):
        taint_file = self.ServiceInterface.get_local_database_taint_file(self.server_repo)
        if not etpDbStatus.has_key(self.dbFile):
            etpDbStatus[self.dbFile] = {}
            etpDbStatus[self.dbFile]['tainted'] = False
            etpDbStatus[self.dbFile]['bumped'] = False
        if os.path.isfile(taint_file):
            etpDbStatus[self.dbFile]['tainted'] = True
            etpDbStatus[self.dbFile]['bumped'] = True

    def closeDB(self):

        self.dbclosed = True

        # if the class is opened readOnly, close and forget
        if self.readOnly:
            self.cursor.close()
            self.connection.close()
            return

        if self.clientDatabase:
            self.commitChanges()
            self.cursor.close()
            self.connection.close()
            return

        if not etpDbStatus[self.dbFile]['tainted']:
            # we can unlock it, no changes were made
            self.ServiceInterface.MirrorsService.lock_mirrors(False, repo = self.server_repo)
        else:
            self.updateProgress(
                darkgreen(_("Mirrors have not been unlocked. Remember to sync them.")),
                importance = 1,
                type = "info",
                header = brown(" * ")
            )

        self.commitChanges()
        #self.vacuum()
        self.cursor.close()
        self.connection.close()

    def vacuum(self):
        self.cursor.execute("vacuum")

    def commitChanges(self):

        if self.readOnly:
            return

        try:
            self.connection.commit()
        except:
            pass

        if not self.clientDatabase:
            self.taintDatabase()
            if (etpDbStatus[self.dbFile]['tainted']) and \
                (not etpDbStatus[self.dbFile]['bumped']):
                    # bump revision, setting DatabaseBump causes the session to just bump once
                    etpDbStatus[self.dbFile]['bumped'] = True
                    self.revisionBump()

    def taintDatabase(self):
        # if it's equo to open it, this should be avoided
        if self.clientDatabase:
            return
        # taint the database status
        taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
        f = open(taint_file,"w")
        f.write(etpConst['currentarch']+" database tainted\n")
        f.flush()
        f.close()
        etpDbStatus[self.dbFile]['tainted'] = True

    def untaintDatabase(self):
        if (self.clientDatabase): # if it's equo to open it, this should be avoided
            return
        etpDbStatus[self.dbFile]['tainted'] = False
        # untaint the database status
        taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
        if os.path.isfile(taint_file):
            os.remove(taint_file)

    def revisionBump(self):
        revision_file = self.ServiceInterface.get_local_database_revision_file(repo = self.server_repo)
        if not os.path.isfile(revision_file):
            revision = 1
        else:
            f = open(revision_file,"r")
            revision = int(f.readline().strip())
            revision += 1
            f.close()
        f = open(revision_file,"w")
        f.write(str(revision)+"\n")
        f.flush()
        f.close()

    def isDatabaseTainted(self):
        taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
        if os.path.isfile(taint_file):
            return True
        return False

    # never use this unless you know what you're doing
    def initializeDatabase(self):
        self.checkReadOnly()
        self.cursor.executescript(etpConst['sql_destroy'])
        self.cursor.executescript(etpConst['sql_init'])
        self.databaseStructureUpdates()
        # set cache size
        self.setCacheSize(6000)
        self.setDefaultCacheSize(6000)
        self.commitChanges()

    def checkReadOnly(self):
        if (self.readOnly):
            raise exceptionTools.OperationNotPermitted("OperationNotPermitted: %s." % (
                    _("can't do that on a readonly database"),
                )
            )

    # check for /usr/portage/profiles/updates changes
    def serverUpdatePackagesData(self):

        etpConst['server_treeupdatescalled'].add(self.server_repo)

        repo_updates_file = self.ServiceInterface.get_local_database_treeupdates_file(self.server_repo)
        doRescan = False

        stored_digest = self.retrieveRepositoryUpdatesDigest(self.server_repo)
        if stored_digest == -1:
            doRescan = True

        # check portage files for changes if doRescan is still false
        portage_dirs_digest = "0"
        if not doRescan:

            treeupdates_dict = self.ServiceInterface.repository_treeupdate_digests

            if treeupdates_dict.has_key(self.server_repo):
                portage_dirs_digest = treeupdates_dict.get(self.server_repo)
            else:
                SpmIntf = SpmInterface(self.OutputInterface)
                Spm = SpmIntf.intf
                # grab portdir
                updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates"
                if os.path.isdir(updates_dir):
                    # get checksum
                    mdigest = self.entropyTools.md5sum_directory(updates_dir, get_obj = True)
                    # also checksum etpConst['etpdatabaseupdatefile']
                    if os.path.isfile(repo_updates_file):
                        f = open(repo_updates_file)
                        block = f.read(1024)
                        while block:
                            mdigest.update(block)
                            block = f.read(1024)
                        f.close()
                    portage_dirs_digest = mdigest.hexdigest()
                    treeupdates_dict[self.server_repo] = portage_dirs_digest
                del updates_dir

        if doRescan or (str(stored_digest) != str(portage_dirs_digest)):

            # force parameters
            self.readOnly = False
            self.noUpload = True

            # reset database tables
            self.clearTreeupdatesEntries(self.server_repo)

            SpmIntf = SpmInterface(self.OutputInterface)
            Spm = SpmIntf.intf
            updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates"
            update_files = self.entropyTools.sortUpdateFiles(os.listdir(updates_dir))
            update_files = [os.path.join(updates_dir,x) for x in update_files]
            # now load actions from files
            update_actions = []
            for update_file in update_files:
                f = open(update_file,"r")
                mycontent = f.readlines()
                f.close()
                lines = [x.strip() for x in mycontent if x.strip()]
                update_actions.extend(lines)

            # add entropy packages.db.repo_updates content
            if os.path.isfile(repo_updates_file):
                f = open(repo_updates_file,"r")
                mycontent = f.readlines()
                f.close()
                lines = [x.strip() for x in mycontent if x.strip() and not x.strip().startswith("#")]
                update_actions.extend(lines)
            # now filter the required actions
            update_actions = self.filterTreeUpdatesActions(update_actions)
            if update_actions:

                mytxt = "%s: %s. %s %s" % (
                    bold(_("ATTENTION")),
                    red(_("forcing package updates")),
                    red(_("Syncing with")),
                    blue(updates_dir),
                )
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = brown(" * ")
                )
                # lock database
                if self.lockRemote:
                    self.ServiceInterface.doServerDatabaseSyncLock(self.server_repo, self.noUpload)
                # now run queue
                try:
                    self.runTreeUpdatesActions(update_actions)
                except:
                    # destroy digest
                    self.setRepositoryUpdatesDigest(self.server_repo, "-1")
                    raise

                # store new actions
                self.addRepositoryUpdatesActions(self.server_repo, update_actions, self.db_branch)

            # store new digest into database
            self.setRepositoryUpdatesDigest(self.server_repo, portage_dirs_digest)

    def clientUpdatePackagesData(self, clientDbconn, force = False):

        if clientDbconn == None:
            return

        repository = self.dbname[len(etpConst['dbnamerepoprefix']):]
        etpConst['client_treeupdatescalled'].add(repository)

        doRescan = False
        shell_rescan = os.getenv("ETP_TREEUPDATES_RESCAN")
        if shell_rescan: doRescan = True

        # check database digest
        stored_digest = self.retrieveRepositoryUpdatesDigest(repository)
        if stored_digest == -1:
            doRescan = True

        # check stored value in client database
        client_digest = "0"
        if not doRescan:
            client_digest = clientDbconn.retrieveRepositoryUpdatesDigest(repository)

        if doRescan or (str(stored_digest) != str(client_digest)) or force:

            # reset database tables
            clientDbconn.clearTreeupdatesEntries(repository)

            # load updates
            update_actions = self.retrieveTreeUpdatesActions(repository)
            # now filter the required actions
            update_actions = clientDbconn.filterTreeUpdatesActions(update_actions)

            if update_actions:

                mytxt = "%s: %s. %s %s" % (
                    bold(_("ATTENTION")),
                    red(_("forcing packages metadata update")),
                    red(_("Updating system database using repository id")),
                    blue(repository),
                )
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "info",
                    header = darkred(" * ")
                    )
                # run stuff
                clientDbconn.runTreeUpdatesActions(update_actions)

            # store new digest into database
            clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest)

            # store new actions
            clientDbconn.addRepositoryUpdatesActions(etpConst['clientdbid'], update_actions, etpConst['branch'])

            # clear client cache
            clientDbconn.clearCache()
            return True

    # this functions will filter either data from /usr/portage/profiles/updates/*
    # or repository database returning only the needed actions
    def filterTreeUpdatesActions(self, actions):
        new_actions = []
        for action in actions:

            if action in new_actions: # skip dupies
                continue

            doaction = action.split()
            if doaction[0] == "slotmove":

                # slot move
                atom = doaction[1]
                from_slot = doaction[2]
                to_slot = doaction[3]
                atom_key = self.entropyTools.dep_getkey(atom)
                category = atom_key.split("/")[0]
                matches = self.atomMatch(atom, matchSlot = from_slot, multiMatch = True)
                found = False
                if matches[1] == 0:
                    # found atoms, check category
                    for idpackage in matches[0]:
                        myslot = self.retrieveSlot(idpackage)
                        mycategory = self.retrieveCategory(idpackage)
                        if mycategory == category:
                            if  (myslot != to_slot) and (action not in new_actions):
                                new_actions.append(action)
                                found = True
                                break
                    if found:
                        continue
                # if we get here it means found == False
                # search into dependencies
                dep_atoms = self.searchDependency(atom_key, like = True, multi = True, strings = True)
                dep_atoms = [x for x in dep_atoms if x.endswith(":"+from_slot) and self.entropyTools.dep_getkey(x) == atom_key]
                if dep_atoms:
                    new_actions.append(action)

            elif doaction[0] == "move":
                atom = doaction[1] # usually a key
                atom_key = self.entropyTools.dep_getkey(atom)
                category = atom_key.split("/")[0]
                matches = self.atomMatch(atom, multiMatch = True)
                found = False
                if matches[1] == 0:
                    for idpackage in matches[0]:
                        mycategory = self.retrieveCategory(idpackage)
                        if (mycategory == category) and (action not in new_actions):
                            new_actions.append(action)
                            found = True
                            break
                    if found:
                        continue
                # if we get here it means found == False
                # search into dependencies
                dep_atoms = self.searchDependency(atom_key, like = True, multi = True, strings = True)
                dep_atoms = [x for x in dep_atoms if self.entropyTools.dep_getkey(x) == atom_key]
                if dep_atoms:
                    new_actions.append(action)
        return new_actions

    # this is the place to add extra actions support
    def runTreeUpdatesActions(self, actions):

        # just run fixpackages if gentoo-compat is enabled
        if etpConst['gentoo-compat']:

            mytxt = "%s: %s, %s." % (
                bold(_("SPM")),
                blue(_("Running fixpackages")),
                red(_("it could take a while")),
            )
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = darkred(" * ")
            )
            if self.clientDatabase:
                try:
                    Spm = self.ServiceInterface.Spm()
                    Spm.run_fixpackages()
                except:
                    pass
            else:
                self.ServiceInterface.SpmService.run_fixpackages()

        spm_moves = set()
        quickpkg_atoms = set()
        for action in actions:
            command = action.split()
            mytxt = "%s: %s: %s." % (
                bold(_("ENTROPY")),
                red(_("action")),
                blue(action),
            )
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = darkred(" * ")
            )
            if command[0] == "move":
                spm_moves.add(action)
                quickpkg_atoms |= self.runTreeUpdatesMoveAction(command[1:], quickpkg_atoms)
            elif command[0] == "slotmove":
                quickpkg_atoms |= self.runTreeUpdatesSlotmoveAction(command[1:], quickpkg_atoms)

        if quickpkg_atoms and not self.clientDatabase:
            # quickpkg package and packages owning it as a dependency
            try:
                self.runTreeUpdatesQuickpkgAction(quickpkg_atoms)
            except:
                self.entropyTools.printTraceback()
                mytxt = "%s: %s: %s, %s." % (
                    bold(_("WARNING")),
                    red(_("Cannot complete quickpkg for atoms")),
                    blue(str(list(quickpkg_atoms))),
                    _("do it manually"),
                )
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = darkred(" * ")
                )
            self.commitChanges()

        if spm_moves:
            try:
                self.doTreeupdatesSpmCleanup(spm_moves)
            except Exception, e:
                mytxt = "%s: %s: %s, %s." % (
                    bold(_("WARNING")),
                    red(_("Cannot run SPM cleanup, error")),
                    Exception,
                    e,
                )
                self.entropyTools.printTraceback()

        # discard cache
        self.clearCache()


    # -- move action:
    # 1) move package key to the new name: category + name + atom
    # 2) update all the dependencies in dependenciesreference to the new key
    # 3) run fixpackages which will update /var/db/pkg files
    # 4) automatically run quickpkg() to build the new binary and
    #    tainted binaries owning tainted iddependency and taint database
    def runTreeUpdatesMoveAction(self, move_command, quickpkg_queue):

        key_from = move_command[0]
        key_to = move_command[1]
        cat_to = key_to.split("/")[0]
        name_to = key_to.split("/")[1]
        matches = self.atomMatch(key_from, multiMatch = True)
        iddependencies_idpackages = set()

        if matches[1] == 0:

            for idpackage in matches[0]:

                slot = self.retrieveSlot(idpackage)
                old_atom = self.retrieveAtom(idpackage)
                new_atom = old_atom.replace(key_from,key_to)

                ### UPDATE DATABASE
                # update category
                self.setCategory(idpackage, cat_to)
                # update name
                self.setName(idpackage, name_to)
                # update atom
                self.setAtom(idpackage, new_atom)

                # look for packages we need to quickpkg again
                # note: quickpkg_queue is simply ignored if self.clientDatabase
                quickpkg_queue.add(key_to+":"+str(slot))

                if not self.clientDatabase:

                    # check for injection and warn the developer
                    injected = self.isInjected(idpackage)
                    if injected:
                        mytxt = "%s: %s %s. %s !!! %s." % (
                            bold(_("INJECT")),
                            blue(str(new_atom)),
                            red(_("has been injected")),
                            red(_("You need to quickpkg it manually to update the embedded database")),
                            red(_("Repository database will be updated anyway")),
                        )
                        self.updateProgress(
                            mytxt,
                            importance = 1,
                            type = "warning",
                            header = darkred(" * ")
                        )

        iddeps = self.searchDependency(key_from, like = True, multi = True)
        for iddep in iddeps:
            # update string
            mydep = self.retrieveDependencyFromIddependency(iddep)
            mydep_key = self.entropyTools.dep_getkey(mydep)
            if mydep_key != key_from: # avoid changing wrong atoms -> dev-python/qscintilla-python would
                continue              # become x11-libs/qscintilla if we don't do this check
            mydep = mydep.replace(key_from,key_to)
            # now update
            # dependstable on server is always re-generated
            self.setDependency(iddep, mydep)
            # we have to repackage also package owning this iddep
            iddependencies_idpackages |= self.searchIdpackageFromIddependency(iddep)

        self.commitChanges()
        quickpkg_queue = list(quickpkg_queue)
        for x in range(len(quickpkg_queue)):
            myatom = quickpkg_queue[x]
            myatom = myatom.replace(key_from,key_to)
            quickpkg_queue[x] = myatom
        quickpkg_queue = set(quickpkg_queue)
        for idpackage_owner in iddependencies_idpackages:
            myatom = self.retrieveAtom(idpackage_owner)
            myatom = myatom.replace(key_from,key_to)
            quickpkg_queue.add(myatom)
        return quickpkg_queue


    # -- slotmove action:
    # 1) move package slot
    # 2) update all the dependencies in dependenciesreference owning same matched atom + slot
    # 3) run fixpackages which will update /var/db/pkg files
    # 4) automatically run quickpkg() to build the new binary and tainted binaries owning tainted iddependency and taint database
    def runTreeUpdatesSlotmoveAction(self, slotmove_command, quickpkg_queue):

        atom = slotmove_command[0]
        atomkey = self.entropyTools.dep_getkey(atom)
        slot_from = slotmove_command[1]
        slot_to = slotmove_command[2]
        matches = self.atomMatch(atom, multiMatch = True)
        iddependencies_idpackages = set()

        if matches[1] == 0:

            matched_idpackages = matches[0]
            for idpackage in matched_idpackages:

                ### UPDATE DATABASE
                # update slot
                self.setSlot(idpackage, slot_to)

                # look for packages we need to quickpkg again
                # note: quickpkg_queue is simply ignored if self.clientDatabase
                quickpkg_queue.add(atom+":"+str(slot_to))

                if not self.clientDatabase:

                    # check for injection and warn the developer
                    injected = self.isInjected(idpackage)
                    if injected:
                        mytxt = "%s: %s %s. %s !!! %s." % (
                            bold(_("INJECT")),
                            blue(str(atom)),
                            red(_("has been injected")),
                            red(_("You need to quickpkg it manually to update the embedded database")),
                            red(_("Repository database will be updated anyway")),
                        )
                        self.updateProgress(
                            mytxt,
                            importance = 1,
                            type = "warning",
                            header = darkred(" * ")
                        )

            # only if we've found VALID matches !
            iddeps = self.searchDependency(atomkey, like = True, multi = True)
            for iddep in iddeps:
                # update string
                mydep = self.retrieveDependencyFromIddependency(iddep)
                mydep_key = self.entropyTools.dep_getkey(mydep)
                if mydep_key != atomkey:
                    continue
                if not mydep.endswith(":"+slot_from): # probably slotted dep
                    continue
                mydep_match = self.atomMatch(mydep)
                if mydep_match not in matched_idpackages:
                    continue
                mydep = mydep.replace(":"+slot_from,":"+slot_to)
                # now update
                # dependstable on server is always re-generated
                self.setDependency(iddep, mydep)
                # we have to repackage also package owning this iddep
                iddependencies_idpackages |= self.searchIdpackageFromIddependency(iddep)

        self.commitChanges()
        for idpackage_owner in iddependencies_idpackages:
            myatom = self.retrieveAtom(idpackage_owner)
            quickpkg_queue.add(myatom)
        return quickpkg_queue

    def runTreeUpdatesQuickpkgAction(self, atoms):

        self.commitChanges()

        package_paths = set()
        runatoms = set()
        for myatom in atoms:
            mymatch = self.atomMatch(myatom)
            if mymatch[0] == -1:
                continue
            myatom = self.retrieveAtom(mymatch[0])
            runatoms.add(myatom)

        for myatom in runatoms:
            self.updateProgress(
                red("%s: " % (_("repackaging"),) )+blue(myatom),
                importance = 1,
                type = "warning",
                header = blue("  # ")
            )
            mydest = self.ServiceInterface.get_local_store_directory(self.server_repo)
            try:
                mypath = self.ServiceInterface.quickpkg(myatom,mydest)
            except:
                # remove broken bin before raising
                mypath = os.path.join(mydest,os.path.basename(myatom)+etpConst['packagesext'])
                if os.path.isfile(mypath):
                    os.remove(mypath)
                self.entropyTools.printTraceback()
                mytxt = "%s: %s: %s, %s." % (
                    bold(_("WARNING")),
                    red(_("Cannot complete quickpkg for atom")),
                    blue(myatom),
                    _("do it manually"),
                )
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = darkred(" * ")
                )
                continue
            package_paths.add(mypath)
        packages_data = [(x,False) for x in package_paths]
        idpackages = self.ServiceInterface.add_packages_to_repository(packages_data, repo = self.server_repo)

        if not idpackages:

            mytxt = "%s: %s. %s." % (
                bold(_("ATTENTION")),
                red(_("runTreeUpdatesQuickpkgAction did not run properly")),
                red(_("Please update packages manually")),
            )
            self.updateProgress(
                mytxt,
                importance = 1,
                type = "warning",
                header = darkred(" * ")
            )

    def doTreeupdatesSpmCleanup(self, spm_moves):

        # now erase Spm entries if necessary
        for action in spm_moves:
            command = action.split()
            if len(command) < 2:
                continue
            key = command[1]
            name = key.split("/")[1]
            if self.clientDatabase:
                try:
                    Spm = self.ServiceInterface.Spm()
                except:
                    continue
            else:
                Spm = self.ServiceInterface.SpmService
            vdb_path = Spm.get_vdb_path()
            pkg_path = os.path.join(vdb_path,key.split("/")[0])
            try:
                mydirs = [os.path.join(pkg_path,x) for x in os.listdir(pkg_path) if x.startswith(name)]
            except OSError: # no dir, no party!
                continue
            mydirs = [x for x in mydirs if os.path.isdir(x)]
            # now move these dirs
            for mydir in mydirs:
                to_path = os.path.join(etpConst['packagestmpdir'],os.path.basename(mydir))
                mytxt = "%s: %s '%s' %s '%s'" % (
                    bold(_("SPM")),
                    red(_("Moving old entry")),
                    blue(mydir),
                    red(_("to")),
                    blue(to_path),
                )
                self.updateProgress(
                    mytxt,
                    importance = 1,
                    type = "warning",
                    header = darkred(" * ")
                )
                if os.path.isdir(to_path):
                    shutil.rmtree(to_path,True)
                    try:
                        os.rmdir(to_path)
                    except OSError:
                        pass
                shutil.move(mydir,to_path)


    def handlePackage(self, etpData, forcedRevision = -1, formattedContent = False):

        self.checkReadOnly()

        if self.clientDatabase:
            return self.addPackage(etpData, revision = forcedRevision, formatted_content = formattedContent)

        # build atom string, server side
        pkgatom = self.entropyTools.create_package_atom_string(etpData['category'], etpData['name'], etpData['version'], etpData['versiontag'])
        foundid = self.isPackageAvailable(pkgatom)
        if foundid < 0: # same atom doesn't exist in any branch
            return self.addPackage(etpData, revision = forcedRevision, formatted_content = formattedContent)

        idpackage = self.getIDPackage(pkgatom)
        curRevision = forcedRevision
        if forcedRevision == -1:
            curRevision = 0
            if idpackage != -1:
                curRevision = self.retrieveRevision(idpackage)

        if idpackage != -1: # remove old package atom, we do it here because othersie
            # injected packages wouldn't be removed by addPackages
            self.removePackage(idpackage)
            if forcedRevision == -1: curRevision += 1

        # add the new one
        return self.addPackage(etpData, revision = curRevision, formatted_content = formattedContent)

    def retrieve_packages_to_remove(self, name, category, slot, injected):
        removelist = set()

        searchsimilar = self.searchPackagesByNameAndCategory(
            name = name,
            category = category,
            sensitive = True
        )

        if not injected:
            # read: if package has been injected, we'll skip
            # the removal of packages in the same slot, usually used server side btw
            for oldpkg in searchsimilar:
                # get the package slot
                idpackage = oldpkg[1]
                myslot = self.retrieveSlot(idpackage)
                # we merely ignore packages with
                # negative counters, since they're the injected ones
                if self.isInjected(idpackage): continue
                if slot == myslot:
                    # remove!
                    removelist.add(idpackage)

        return removelist

    def addPackage(self, etpData, revision = -1, idpackage = None, do_remove = True, do_commit = True, formatted_content = False):

        self.checkReadOnly()
        self.live_cache.clear()

        if revision == -1:
            try:
                revision = int(etpData['revision'])
            except (KeyError, ValueError):
                etpData['revision'] = 0 # revision not specified
                revision = 0
        elif not etpData.has_key('revision'):
            etpData['revision'] = revision

        if do_remove:
            removelist = self.retrieve_packages_to_remove(
                            etpData['name'],
                            etpData['category'],
                            etpData['slot'],
                            etpData['injected']
            )
            for pkg in removelist:
                self.removePackage(pkg)

        ### create new ids

        # create new category if it doesn't exist
        catid = self.isCategoryAvailable(etpData['category'])
        if catid == -1: catid = self.addCategory(etpData['category'])

        # create new license if it doesn't exist
        licid = self.isLicenseAvailable(etpData['license'])
        if licid == -1: licid = self.addLicense(etpData['license'])

        idprotect = self.isProtectAvailable(etpData['config_protect'])
        if idprotect == -1: idprotect = self.addProtect(etpData['config_protect'])

        idprotect_mask = self.isProtectAvailable(etpData['config_protect_mask'])
        if idprotect_mask == -1: idprotect_mask = self.addProtect(etpData['config_protect_mask'])

        idflags = self.areCompileFlagsAvailable(etpData['chost'],etpData['cflags'],etpData['cxxflags'])
        if idflags == -1: idflags = self.addCompileFlags(etpData['chost'],etpData['cflags'],etpData['cxxflags'])


        trigger = 0
        if etpData['trigger']:
            trigger = 1

        # baseinfo
        pkgatom = self.entropyTools.create_package_atom_string(etpData['category'], etpData['name'], etpData['version'], etpData['versiontag'])

        mybaseinfo_data = [
            pkgatom,
            catid,
            etpData['name'],
            etpData['version'],
            etpData['versiontag'],
            revision,
            etpData['branch'],
            etpData['slot'],
            licid,
            etpData['etpapi'],
            trigger
        ]

        myidpackage_string = 'NULL'
        if isinstance(idpackage,int):
            # does it exist?
            self.removePackage(idpackage, do_cleanup = False, do_commit = False, do_rss = False)
            myidpackage_string = '?'
            mybaseinfo_data.insert(0,idpackage)
        else:
            idpackage = None

        with self.WriteLock:

            self.cursor.execute(
                    'INSERT into baseinfo VALUES '
                    '('+myidpackage_string+',?,?,?,?,?,?,?,?,?,?,?)'
                    , mybaseinfo_data
            )
            if idpackage == None:
                idpackage = self.cursor.lastrowid

            # extrainfo
            self.cursor.execute(
                    'INSERT into extrainfo VALUES '
                    '(?,?,?,?,?,?,?,?)'
                    , (	idpackage,
                            etpData['description'],
                            etpData['homepage'],
                            etpData['download'],
                            etpData['size'],
                            idflags,
                            etpData['digest'],
                            etpData['datecreation'],
                            )
            )
        ### other information iserted below are not as critical as these above

        # tables using a select
        self.insertEclasses(idpackage, etpData['eclasses'])
        self.insertNeeded(idpackage, etpData['needed'])
        self.insertDependencies(idpackage, etpData['dependencies'])
        self.insertSources(idpackage, etpData['sources'])
        self.insertUseflags(idpackage, etpData['useflags'])
        self.insertKeywords(idpackage, etpData['keywords'])
        self.insertLicenses(etpData['licensedata'])
        self.insertMirrors(etpData['mirrorlinks'])

        # not depending on other tables == no select done
        self.insertContent(idpackage, etpData['content'], already_formatted = formatted_content)
        etpData['counter'] = int(etpData['counter']) # cast to integer
        etpData['counter'] = self.insertPortageCounter(
                                idpackage,
                                etpData['counter'],
                                etpData['branch'],
                                etpData['injected']
        )
        self.insertOnDiskSize(idpackage, etpData['disksize'])
        self.insertTrigger(idpackage, etpData['trigger'])
        self.insertConflicts(idpackage, etpData['conflicts'])
        self.insertProvide(idpackage, etpData['provide'])
        self.insertMessages(idpackage, etpData['messages'])
        self.insertConfigProtect(idpackage, idprotect)
        self.insertConfigProtect(idpackage, idprotect_mask, mask = True)
        # injected?
        if etpData.get('injected'):
            self.setInjected(idpackage, do_commit = False)
        # is it a system package?
        if etpData.get('systempackage'):
            self.setSystemPackage(idpackage, do_commit = False)

        self.clearCache()
        if do_commit:
            self.commitChanges()

        ### RSS Atom support
        ### dictionary will be elaborated by activator
        if etpConst['rss-feed'] and not self.clientDatabase:
            self._write_rss_for_added_package(pkgatom, revision, etpData['description'], etpData['homepage'])

        # Update category description
        if not self.clientDatabase:
            mycategory = etpData['category']
            descdata = {}
            try:
                descdata = self.get_category_description_from_disk(mycategory)
            except (IOError,OSError,EOFError):
                pass
            if descdata:
                self.setCategoryDescription(mycategory,descdata)

        return idpackage, revision, etpData

    def _write_rss_for_added_package(self, pkgatom, revision, description, homepage):
        rssAtom = pkgatom+"~"+str(revision)
        rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name'])
        if rssObj: self.ServiceInterface.rssMessages = rssObj.copy()
        if not isinstance(self.ServiceInterface.rssMessages,dict):
            self.ServiceInterface.rssMessages = {}
        if not self.ServiceInterface.rssMessages.has_key('added'):
            self.ServiceInterface.rssMessages['added'] = {}
        if not self.ServiceInterface.rssMessages.has_key('removed'):
            self.ServiceInterface.rssMessages['removed'] = {}
        if rssAtom in self.ServiceInterface.rssMessages['removed']:
            del self.ServiceInterface.rssMessages['removed'][rssAtom]
        self.ServiceInterface.rssMessages['added'][rssAtom] = {}
        self.ServiceInterface.rssMessages['added'][rssAtom]['description'] = description
        self.ServiceInterface.rssMessages['added'][rssAtom]['homepage'] = homepage
        self.ServiceInterface.rssMessages['light'][rssAtom] = {}
        self.ServiceInterface.rssMessages['light'][rssAtom]['description'] = description
        self.dumpTools.dumpobj(etpConst['rss-dump-name'],self.ServiceInterface.rssMessages)

    def _write_rss_for_removed_package(self, idpackage):
        rssObj = self.dumpTools.loadobj(etpConst['rss-dump-name'])
        if rssObj: self.ServiceInterface.rssMessages = rssObj.copy()
        rssAtom = self.retrieveAtom(idpackage)
        rssRevision = self.retrieveRevision(idpackage)
        rssAtom += "~"+str(rssRevision)
        if not isinstance(self.ServiceInterface.rssMessages,dict):
            self.ServiceInterface.rssMessages = {}
        if not self.ServiceInterface.rssMessages.has_key('added'):
            self.ServiceInterface.rssMessages['added'] = {}
        if not self.ServiceInterface.rssMessages.has_key('removed'):
            self.ServiceInterface.rssMessages['removed'] = {}
        if rssAtom in self.ServiceInterface.rssMessages['added']:
            del self.ServiceInterface.rssMessages['added'][rssAtom]
        self.ServiceInterface.rssMessages['removed'][rssAtom] = {}
        try:
            self.ServiceInterface.rssMessages['removed'][rssAtom]['description'] = self.retrieveDescription(idpackage)
        except:
            self.ServiceInterface.rssMessages['removed'][rssAtom]['description'] = "N/A"
        try:
            self.ServiceInterface.rssMessages['removed'][rssAtom]['homepage'] = self.retrieveHomepage(idpackage)
        except:
            self.ServiceInterface.rssMessages['removed'][rssAtom]['homepage'] = ""
        self.dumpTools.dumpobj(etpConst['rss-dump-name'],self.ServiceInterface.rssMessages)

    def removePackage(self, idpackage, do_cleanup = True, do_commit = True, do_rss = True):

        self.checkReadOnly()
        self.live_cache.clear()

        ### RSS Atom support
        ### dictionary will be elaborated by activator
        if etpConst['rss-feed'] and (not self.clientDatabase) and do_rss:
            # store addPackage action
            self._write_rss_for_removed_package(idpackage)

        with self.WriteLock:

            r_tup = (idpackage,)*20
            self.cursor.executescript("""
                DELETE FROM baseinfo WHERE idpackage = %d;
                DELETE FROM extrainfo WHERE idpackage = %d;
                DELETE FROM dependencies WHERE idpackage = %d;
                DELETE FROM provide WHERE idpackage = %d;
                DELETE FROM conflicts WHERE idpackage = %d;
                DELETE FROM configprotect WHERE idpackage = %d;
                DELETE FROM configprotectmask WHERE idpackage = %d;
                DELETE FROM sources WHERE idpackage = %d;
                DELETE FROM useflags WHERE idpackage = %d;
                DELETE FROM keywords WHERE idpackage = %d;
                DELETE FROM content WHERE idpackage = %d;
                DELETE FROM messages WHERE idpackage = %d;
                DELETE FROM counters WHERE idpackage = %d;
                DELETE FROM sizes WHERE idpackage = %d;
                DELETE FROM eclasses WHERE idpackage = %d;
                DELETE FROM needed WHERE idpackage = %d;
                DELETE FROM triggers WHERE idpackage = %d;
                DELETE FROM systempackages WHERE idpackage = %d;
                DELETE FROM injected WHERE idpackage = %d;
                DELETE FROM installedtable WHERE idpackage = %d;
            """ % r_tup)

        # Remove from dependstable if exists
        self.removePackageFromDependsTable(idpackage)

        if do_cleanup:
            # Cleanups if at least one package has been removed
            self.doCleanups()
            # clear caches
            self.clearCache()

        if do_commit:
            self.commitChanges()

    def removeMirrorEntries(self,mirrorname):
        with self.WriteLock:
            self.cursor.execute('DELETE FROM mirrorlinks WHERE mirrorname = (?)',(mirrorname,))

    def addMirrors(self,mirrorname,mirrorlist):
        with self.WriteLock:
            for x in mirrorlist:
                self.cursor.execute(
                    'INSERT into mirrorlinks VALUES '
                    '(?,?)', (mirrorname,x,)
                )

    def addCategory(self,category):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into categories VALUES '
                    '(NULL,?)', (category,)
            )
            return self.cursor.lastrowid


    def addProtect(self,protect):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into configprotectreference VALUES '
                    '(NULL,?)', (protect,)
            )
            return self.cursor.lastrowid


    def addSource(self,source):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into sourcesreference VALUES '
                    '(NULL,?)', (source,)
            )
            return self.cursor.lastrowid

    def addDependency(self,dependency):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into dependenciesreference VALUES '
                    '(NULL,?)', (dependency,)
            )
            return self.cursor.lastrowid

    def addKeyword(self,keyword):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into keywordsreference VALUES '
                    '(NULL,?)', (keyword,)
            )
            return self.cursor.lastrowid

    def addUseflag(self,useflag):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into useflagsreference VALUES '
                    '(NULL,?)', (useflag,)
            )
            return self.cursor.lastrowid

    def addEclass(self,eclass):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into eclassesreference VALUES '
                    '(NULL,?)', (eclass,)
            )
            return self.cursor.lastrowid

    def addNeeded(self,needed):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into neededreference VALUES '
                    '(NULL,?)', (needed,)
            )
            return self.cursor.lastrowid

    def addLicense(self,pkglicense):
        if not self.entropyTools.is_valid_string(pkglicense):
            pkglicense = ' ' # workaround for broken license entries
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into licenses VALUES '
                    '(NULL,?)', (pkglicense,)
            )
            return self.cursor.lastrowid

    def addCompileFlags(self,chost,cflags,cxxflags):
        with self.WriteLock:
            self.cursor.execute(
                    'INSERT into flags VALUES '
                    '(NULL,?,?,?)', (chost,cflags,cxxflags,)
            )
            return self.cursor.lastrowid

    def setSystemPackage(self, idpackage, do_commit = True):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('INSERT into systempackages VALUES (?)', (idpackage,))
            if do_commit:
                self.commitChanges()

    def setInjected(self, idpackage, do_commit = True):
        self.checkReadOnly()
        with self.WriteLock:
            if not self.isInjected(idpackage):
                self.cursor.execute('INSERT into injected VALUES (?)', (idpackage,))
            if do_commit:
                self.commitChanges()

    # date expressed the unix way
    def setDateCreation(self, idpackage, date):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE extrainfo SET datecreation = (?) WHERE idpackage = (?)', (str(date),idpackage,))
            self.commitChanges()

    def setDigest(self, idpackage, digest):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE extrainfo SET digest = (?) WHERE idpackage = (?)', (digest,idpackage,))
            self.commitChanges()

    def setDownloadURL(self, idpackage, url):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (url,idpackage,))
            self.commitChanges()

    def setCategory(self, idpackage, category):
        self.checkReadOnly()
        # create new category if it doesn't exist
        catid = self.isCategoryAvailable(category)
        if (catid == -1):
            # create category
            catid = self.addCategory(category)
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET idcategory = (?) WHERE idpackage = (?)', (catid,idpackage,))
            self.commitChanges()

    def setCategoryDescription(self, category, description_data):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM categoriesdescription WHERE category = (?)', (category,))
            for locale in description_data:
                mydesc = description_data[locale]
                #if type(mydesc) is unicode:
                #    mydesc = mydesc.encode('raw_unicode_escape')
                self.cursor.execute('INSERT INTO categoriesdescription VALUES (?,?,?)', (category,locale,mydesc,))
            self.commitChanges()

    def setName(self, idpackage, name):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET name = (?) WHERE idpackage = (?)', (name,idpackage,))
            self.commitChanges()

    def setDependency(self, iddependency, dependency):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE dependenciesreference SET dependency = (?) WHERE iddependency = (?)', (dependency,iddependency,))
            self.commitChanges()

    def setAtom(self, idpackage, atom):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET atom = (?) WHERE idpackage = (?)', (atom,idpackage,))
            self.commitChanges()

    def setSlot(self, idpackage, slot):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot,idpackage,))
            self.commitChanges()

    def removeLicensedata(self, license_name):
        if not self.doesTableExist("licensedata"):
            return
        with self.WriteLock:
            self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,))

    def removeDependencies(self, idpackage):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,))
            self.commitChanges()

    def insertDependencies(self, idpackage, depdata):

        dcache = set()
        add_dep = self.addDependency
        is_dep_avail = self.isDependencyAvailable
        def mymf(dep):

            if dep in dcache: return 0
            iddep = is_dep_avail(dep)
            if iddep == -1: iddep = add_dep(dep)

            deptype = 0
            if isinstance(depdata,dict):
                deptype = depdata[dep]

            dcache.add(dep)
            return (idpackage,iddep,deptype,)

        # do not place inside the with statement, otherwise there'll be an obvious lockup
        deps = [x for x in map(mymf,depdata) if type(x) != int]
        with self.WriteLock:
            self.cursor.executemany('INSERT into dependencies VALUES (?,?,?)', deps)

    def removeContent(self, idpackage):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,))
            self.commitChanges()

    def insertContent(self, idpackage, content, already_formatted = False):

        with self.WriteLock:
            if already_formatted:
                self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',[(idpackage,b,c,) for a,b,c in content])
                return
            do_encode = [1 for x in content if type(x) is unicode]
            def my_cmap(xfile):
                contenttype = content[xfile]
                if do_encode: xfile = xfile.encode('raw_unicode_escape')
                return (idpackage,xfile,contenttype,)
            self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',map(my_cmap,content))

    def insertLicenses(self, licenses_data):

        mylicenses = licenses_data.keys()
        is_lic_avail = self.isLicensedataKeyAvailable
        def my_mf(mylicense):
            return not is_lic_avail(mylicense)

        def my_mm(mylicense):
            return (mylicense,buffer(licenses_data.get(mylicense)),0,)

        with self.WriteLock:
            self.cursor.executemany('INSERT into licensedata VALUES (?,?,?)',map(my_mm,list(set(filter(my_mf,mylicenses)))))

    def insertConfigProtect(self, idpackage, idprotect, mask = False):

        mytable = 'configprotect'
        if mask: mytable += 'mask'
        with self.WriteLock:
            self.cursor.execute('INSERT into %s VALUES (?,?)' % (mytable,), (idpackage,idprotect,))

    def insertMirrors(self, mirrors):

        for mirrorname,mirrorlist in mirrors:
            # remove old
            self.removeMirrorEntries(mirrorname)
            # add new
            self.addMirrors(mirrorname,mirrorlist)

    def insertKeywords(self, idpackage, keywords):

        mydata = set()
        for key in keywords:
            idkeyword = self.isKeywordAvailable(key)
            if (idkeyword == -1):
                # create category
                idkeyword = self.addKeyword(key)
            mydata.add((idpackage,idkeyword,))

        with self.WriteLock:
            self.cursor.executemany('INSERT into keywords VALUES (?,?)',mydata)

    def insertUseflags(self, idpackage, useflags):

        mydata = set()
        for flag in useflags:
            iduseflag = self.isUseflagAvailable(flag)
            if (iduseflag == -1):
                # create category
                iduseflag = self.addUseflag(flag)
            mydata.add((idpackage,iduseflag,))

        with self.WriteLock:
            self.cursor.executemany('INSERT into useflags VALUES (?,?)',mydata)

    def insertSources(self, idpackage, sources):

        mydata = set()
        for source in sources:
            if (not source) or (source == "") or (not self.entropyTools.is_valid_string(source)):
                continue
            idsource = self.isSourceAvailable(source)
            if (idsource == -1):
                # create category
                idsource = self.addSource(source)
            mydata.add((idpackage,idsource,))

        with self.WriteLock:
            self.cursor.executemany('INSERT into sources VALUES (?,?)',mydata)

    def insertConflicts(self, idpackage, conflicts):

        def myiter():
            for conflict in conflicts:
                yield (idpackage,conflict,)

        with self.WriteLock:
            self.cursor.executemany('INSERT into conflicts VALUES (?,?)',myiter())

    def insertMessages(self, idpackage, messages):

        def myiter():
            for message in messages:
                yield (idpackage,message,)

        with self.WriteLock:
            self.cursor.executemany('INSERT into messages VALUES (?,?)',myiter())

    def insertProvide(self, idpackage, provides):

        def myiter():
            for atom in provides:
                yield (idpackage,atom,)

        with self.WriteLock:
            self.cursor.executemany('INSERT into provide VALUES (?,?)',myiter())

    def insertNeeded(self, idpackage, neededs):

        mydata = set()
        for needed,elfclass in neededs:
            idneeded = self.isNeededAvailable(needed)
            if idneeded == -1:
                # create eclass
                idneeded = self.addNeeded(needed)
            mydata.add((idpackage,idneeded,elfclass))

        with self.WriteLock:
            self.cursor.executemany('INSERT into needed VALUES (?,?,?)',mydata)

    def insertEclasses(self, idpackage, eclasses):

        mydata = set()
        for eclass in eclasses:
            idclass = self.isEclassAvailable(eclass)
            if (idclass == -1):
                # create eclass
                idclass = self.addEclass(eclass)
            mydata.add((idpackage,idclass))

        with self.WriteLock:
            self.cursor.executemany('INSERT into eclasses VALUES (?,?)',mydata)

    def insertOnDiskSize(self, idpackage, mysize):
        with self.WriteLock:
            self.cursor.execute('INSERT into sizes VALUES (?,?)', (idpackage,mysize,))

    def insertTrigger(self, idpackage, trigger):
        with self.WriteLock:
            self.cursor.execute('INSERT into triggers VALUES (?,?)', (idpackage,buffer(trigger),))

    def insertPortageCounter(self, idpackage, counter, branch, injected):

        if (counter != -1) and not injected:

            if counter <= -2:
                # special cases
                counter = self.getNewNegativeCounter()

            with self.WriteLock:
                try:
                    self.cursor.execute(
                    'INSERT into counters VALUES '
                    '(?,?,?)'
                    , ( counter,
                        idpackage,
                        branch,
                        )
                    )
                except self.dbapi2.IntegrityError: # we have a PRIMARY KEY we need to remove
                    self._migrateCountersTable()
                    self.cursor.execute(
                    'INSERT into counters VALUES '
                    '(?,?,?)'
                    , ( counter,
                        idpackage,
                        branch,
                        )
                    )
                except:
                    if self.dbname == etpConst['clientdbid']: # force only for client database
                        if self.doesTableExist("counters"):
                            raise
                        self.cursor.execute(
                        'INSERT into counters VALUES '
                        '(?,?,?)'
                        , ( counter,
                            idpackage,
                            branch,
                            )
                        )
                    elif self.dbname.startswith(etpConst['serverdbid']):
                        raise

        return counter

    def insertCounter(self, idpackage, counter, branch = None):
        self.checkReadOnly()
        if not branch: branch = self.db_branch
        if not branch: branch = etpConst['branch']
        with self.WriteLock:
            self.cursor.execute('DELETE FROM counters WHERE (counter = (?) OR idpackage = (?)) AND branch = (?)', (counter,idpackage,branch,))
            self.cursor.execute('INSERT INTO counters VALUES (?,?,?)', (counter,idpackage,branch,))
            self.commitChanges()

    def setTrashedCounter(self, counter):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM trashedcounters WHERE counter = (?)', (counter,))
            self.cursor.execute('INSERT INTO trashedcounters VALUES (?)', (counter,))
            self.commitChanges()


    def setCounter(self, idpackage, counter, branch = None):
        self.checkReadOnly()

        branchstring = ''
        insertdata = [counter,idpackage]
        if branch:
            branchstring = ', branch = (?)'
            insertdata.insert(1,branch)

        with self.WriteLock:
            try:
                self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata)
                self.commitChanges()
            except:
                if self.dbname == etpConst['clientdbid']:
                    raise

    def contentDiff(self, idpackage, dbconn, dbconn_idpackage):
        self.checkReadOnly()
        self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
        # create a random table and fill
        randomtable = "cdiff%s" % (self.entropyTools.getRandomNumber(),)
        while self.doesTableExist(randomtable):
            randomtable = "cdiff%s" % (self.entropyTools.getRandomNumber(),)
        self.cursor.execute('CREATE TEMPORARY TABLE %s ( file VARCHAR )' % (randomtable,))

        try:
            dbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
            dbconn.cursor.execute('select file from content where idpackage = (?)', (dbconn_idpackage,))
            xfile = dbconn.cursor.fetchone()
            while xfile:
                self.cursor.execute('INSERT INTO %s VALUES (?)' % (randomtable,), (xfile[0],))
                xfile = dbconn.cursor.fetchone()

            # now compare
            self.cursor.execute('SELECT file FROM content WHERE content.idpackage = (?) AND content.file NOT IN (SELECT file from %s)' % (randomtable,), (idpackage,))
            diff = self.fetchall2set(self.cursor.fetchall())
            return diff
        finally:
            self.cursor.execute('DROP TABLE IF EXISTS %s' % (randomtable,))

    def doCleanups(self):
        self.cleanupUseflags()
        self.cleanupSources()
        self.cleanupEclasses()
        self.cleanupNeeded()
        self.cleanupDependencies()

    def cleanupUseflags(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM useflagsreference WHERE idflag NOT IN (SELECT idflag FROM useflags)')
            self.commitChanges()

    def cleanupSources(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM sourcesreference WHERE idsource NOT IN (SELECT idsource FROM sources)')
            self.commitChanges()

    def cleanupEclasses(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM eclassesreference WHERE idclass NOT IN (SELECT idclass FROM eclasses)')
            self.commitChanges()

    def cleanupNeeded(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM neededreference WHERE idneeded NOT IN (SELECT idneeded FROM needed)')
            self.commitChanges()

    def cleanupDependencies(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM dependenciesreference WHERE iddependency NOT IN (SELECT iddependency FROM dependencies)')
            self.commitChanges()

    def getNewNegativeCounter(self):
        counter = -2
        try:
            self.cursor.execute('SELECT min(counter) FROM counters')
            dbcounter = self.cursor.fetchone()
            mycounter = 0
            if dbcounter:
                mycounter = dbcounter[0]

            if mycounter >= -1:
                counter = -2
            else:
                counter = mycounter-1

        except:
            pass
        return counter

    def getApi(self):
        self.cursor.execute('SELECT max(etpapi) FROM baseinfo')
        api = self.cursor.fetchone()
        if api: api = api[0]
        else: api = -1
        return api

    def getCategory(self, idcategory):
        self.cursor.execute('SELECT category from categories WHERE idcategory = (?)', (idcategory,))
        cat = self.cursor.fetchone()
        if cat: cat = cat[0]
        return cat

    def get_category_description_from_disk(self, category):
        if not self.ServiceInterface:
            return {}
        return self.ServiceInterface.SpmService.get_category_description_data(category)

    def getIDPackage(self, atom, branch = None):
        branch_string = ''
        params = [atom]
        if branch:
            branch_string = ' AND branch = (?)'
            params.append(branch)
        self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)'+branch_string, params)
        idpackage = self.cursor.fetchone()
        if idpackage: return idpackage[0]
        return -1

    def getIDPackageFromDownload(self, download_relative_path, endswith = False):
        if endswith:
            self.cursor.execute('SELECT baseinfo.idpackage FROM baseinfo,extrainfo WHERE extrainfo.download LIKE (?)', ("%"+download_relative_path,))
        else:
            self.cursor.execute('SELECT baseinfo.idpackage FROM baseinfo,extrainfo WHERE extrainfo.download = (?)', (download_relative_path,))
        idpackage = self.cursor.fetchone()
        if idpackage: return idpackage[0]
        return -1

    def getIDPackagesFromFile(self, file):
        self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (file,))
        return self.fetchall2list(self.cursor.fetchall())

    def getIDCategory(self, category):
        self.cursor.execute('SELECT "idcategory" FROM categories WHERE category = (?)', (category,))
        idcat = self.cursor.fetchone()
        if idcat: return idcat[0]
        return -1

    def getVersioningData(self, idpackage):
        self.cursor.execute('SELECT version,versiontag,revision FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        return self.cursor.fetchone()

    def getStrictData(self, idpackage):
        self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot,baseinfo.version,baseinfo.versiontag,baseinfo.revision,baseinfo.atom FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
        return self.cursor.fetchone()

    def getStrictScopeData(self, idpackage):
        self.cursor.execute("""
                SELECT 
                        atom,
                        slot,
                        revision
                FROM 
                        baseinfo
                WHERE 
                        idpackage = (?)
        """, (idpackage,))
        rslt = self.cursor.fetchone()
        return rslt

    def getScopeData(self, idpackage):
        self.cursor.execute("""
                SELECT 
                        baseinfo.atom,
                        categories.category,
                        baseinfo.name,
                        baseinfo.version,
                        baseinfo.slot,
                        baseinfo.versiontag,
                        baseinfo.revision,
                        baseinfo.branch
                FROM 
                        baseinfo,
                        categories
                WHERE 
                        baseinfo.idpackage = (?)
                        and baseinfo.idcategory = categories.idcategory
        """, (idpackage,))
        return self.cursor.fetchone()

    def getBaseData(self,idpackage):

        sql = """
                SELECT 
                        baseinfo.atom,
                        baseinfo.name,
                        baseinfo.version,
                        baseinfo.versiontag,
                        extrainfo.description,
                        categories.category,
                        flags.chost,
                        flags.cflags,
                        flags.cxxflags,
                        extrainfo.homepage,
                        licenses.license,
                        baseinfo.branch,
                        extrainfo.download,
                        extrainfo.digest,
                        baseinfo.slot,
                        baseinfo.etpapi,
                        extrainfo.datecreation,
                        extrainfo.size,
                        baseinfo.revision
                FROM 
                        baseinfo,
                        extrainfo,
                        categories,
                        flags,
                        licenses
                WHERE 
                        baseinfo.idpackage = (?) 
                        and baseinfo.idpackage = extrainfo.idpackage 
                        and baseinfo.idcategory = categories.idcategory 
                        and extrainfo.idflags = flags.idflags
                        and baseinfo.idlicense = licenses.idlicense
        """
        self.cursor.execute(sql, (idpackage,))
        return self.cursor.fetchone()

    def getTriggerInfo(self, idpackage):
        data = {}

        mydata = self.getScopeData(idpackage)

        data['atom'] = mydata[0]
        data['category'] = mydata[1]
        data['name'] = mydata[2]
        data['version'] = mydata[3]
        data['versiontag'] = mydata[5]
        flags = self.retrieveCompileFlags(idpackage)
        data['chost'] = flags[0]
        data['cflags'] = flags[1]
        data['cxxflags'] = flags[2]

        data['trigger'] = self.retrieveTrigger(idpackage)
        data['eclasses'] = self.retrieveEclasses(idpackage)
        data['content'] = self.retrieveContent(idpackage)

        return data

    def getPackageData(self, idpackage, get_content = True, content_insert_formatted = False, trigger_unicode = False):
        data = {}

        try:
            data['atom'], data['name'], data['version'], data['versiontag'], \
            data['description'], data['category'], data['chost'], \
            data['cflags'], data['cxxflags'],data['homepage'], \
            data['license'], data['branch'], data['download'], \
            data['digest'], data['slot'], data['etpapi'], \
            data['datecreation'], data['size'], data['revision']  = self.getBaseData(idpackage)
        except TypeError:
            return None

        ### risky to add to the sql above
        data['counter'] = self.retrieveCounter(idpackage)
        data['messages'] = self.retrieveMessages(idpackage)
        data['trigger'] = self.retrieveTrigger(idpackage, get_unicode = trigger_unicode)
        data['disksize'] = self.retrieveOnDiskSize(idpackage)

        data['injected'] = self.isInjected(idpackage)
        data['systempackage'] = False
        if self.isSystemPackage(idpackage):
            data['systempackage'] = True

        data['config_protect'] = self.retrieveProtect(idpackage)
        data['config_protect_mask'] = self.retrieveProtectMask(idpackage)
        data['useflags'] = self.retrieveUseflags(idpackage)
        data['keywords'] = self.retrieveKeywords(idpackage)
        data['sources'] = self.retrieveSources(idpackage)
        data['eclasses'] = self.retrieveEclasses(idpackage)
        data['needed'] = self.retrieveNeeded(idpackage, extended = True)
        data['provide'] = self.retrieveProvide(idpackage)
        data['conflicts'] = self.retrieveConflicts(idpackage)
        data['licensedata'] = self.retrieveLicensedata(idpackage)

        mirrornames = set()
        for x in data['sources']:
            if x.startswith("mirror://"):
                mirrornames.add(x.split("/")[2])
        data['mirrorlinks'] = []
        for mirror in mirrornames:
            data['mirrorlinks'].append([mirror,self.retrieveMirrorInfo(mirror)])

        data['content'] = {}
        if get_content:
            data['content'] = self.retrieveContent(
                idpackage,
                extended = True,
                formatted = True,
                insert_formatted = content_insert_formatted
            )

        mydeps = {}
        depdata = self.retrieveDependencies(idpackage, extended = True)
        for dep,deptype in depdata:
            mydeps[dep] = deptype
        data['dependencies'] = mydeps

        return data

    def fetchall2set(self, item):
        mycontent = set()
        for x in item:
            mycontent |= set(x)
        return mycontent

    def fetchall2list(self, item):
        content = []
        for x in item:
            content += list(x)
        return content

    def fetchone2list(self, item):
        return list(item)

    def fetchone2set(self, item):
        return set(item)

    def clearCache(self, depends = False):
        self.live_cache.clear()
        def do_clear(name):
            dump_path = os.path.join(etpConst['dumpstoragedir'],name)
            dump_dir = os.path.dirname(dump_path)
            if os.path.isdir(dump_dir):
                for item in os.listdir(dump_dir):
                    try: os.remove(os.path.join(dump_dir,item))
                    except OSError: pass

        do_clear("%s/%s/" % (self.dbMatchCacheKey,self.dbname,))
        do_clear("%s/%s/" % (self.dbSearchCacheKey,self.dbname,))
        if depends:
            do_clear(etpCache['depends_tree'])
            do_clear(etpCache['dep_tree'])
            do_clear(etpCache['filter_satisfied_deps'])

    def fetchSearchCache(self, key, function, extra_hash = 0):
        if self.xcache:
            c_hash = "%s/%s/%s/%s" % (self.dbSearchCacheKey,self.dbname,key,"%s%s" % (hash(function),extra_hash,),)
            if self.ServiceInterface: cached = self.ServiceInterface.Cacher.pop(c_hash)
            else: cached = self.dumpTools.loadobj(c_hash)
            if cached != None: return cached

    def storeSearchCache(self, key, function, search_cache_data, extra_hash = 0):
        if self.xcache:
            c_hash = "%s/%s/%s/%s" % (self.dbSearchCacheKey,self.dbname,key,"%s%s" % (hash(function),extra_hash,),)
            if self.ServiceInterface: self.ServiceInterface.Cacher.push(c_hash, search_cache_data.copy())
            else: self.dumpTools.dumpobj(c_hash,search_cache_data)

    def retrieveRepositoryUpdatesDigest(self, repository):
        if not self.doesTableExist("treeupdates"):
            return -1
        self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,))
        mydigest = self.cursor.fetchone()
        if mydigest:
            return mydigest[0]
        else:
            return -1

    def listAllTreeUpdatesActions(self, no_ids_repos = False):
        if no_ids_repos:
            self.cursor.execute('SELECT command,branch,date FROM treeupdatesactions')
        else:
            self.cursor.execute('SELECT * FROM treeupdatesactions')
        return self.cursor.fetchall()

    def retrieveTreeUpdatesActions(self, repository, forbranch = None):

        if not self.doesTableExist("treeupdatesactions"): return []
        if forbranch == None: forbranch = self.db_branch
        params = [repository]
        branch_string = ''
        if forbranch:
            branch_string = 'and branch = (?)'
            params.append(forbranch)

        self.cursor.execute('SELECT command FROM treeupdatesactions where repository = (?) '+branch_string+' order by date', params)
        return self.fetchall2list(self.cursor.fetchall())

    # mainly used to restore a previous table, used by reagent in --initialize
    def bumpTreeUpdatesActions(self, updates):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM treeupdatesactions')
            for update in updates:
                self.cursor.execute('INSERT INTO treeupdatesactions VALUES (?,?,?,?,?)', update)
            self.commitChanges()

    def removeTreeUpdatesActions(self, repository):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM treeupdatesactions WHERE repository = (?)', (repository,))
            self.commitChanges()

    def insertTreeUpdatesActions(self, updates, repository):
        self.checkReadOnly()
        with self.WriteLock:
            for update in updates:
                update = list(update)
                update.insert(0,repository)
                self.cursor.execute('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', update)
            self.commitChanges()

    def setRepositoryUpdatesDigest(self, repository, digest):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety
            self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository,digest,))
            self.commitChanges()

    def addRepositoryUpdatesActions(self, repository, actions, branch):
        self.checkReadOnly()

        mytime = str(self.entropyTools.getCurrentUnixTime())
        with self.WriteLock:
            for command in actions:
                if not self.doesTreeupdatesActionExist(repository, command, branch):
                    self.cursor.execute('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?,?)', (repository,command,branch,mytime,))
            self.commitChanges()

    def doesTreeupdatesActionExist(self, repository, command, branch):
        self.cursor.execute('SELECT * FROM treeupdatesactions WHERE repository = (?) and command = (?) and branch = (?)', (repository,command,branch,))
        result = self.cursor.fetchone()
        if result:
            return True
        return False

    def clearPackageSets(self):
        self.checkReadOnly()
        self.cursor.execute('DELETE FROM packagesets')

    def insertPackageSets(self, sets_data):
        self.checkReadOnly()

        mysets = []
        for setname in sorted(sets_data.keys()):
            for dependency in sorted(sets_data[setname]):
                try:
                    mysets.append((unicode(setname),unicode(dependency),))
                except (UnicodeDecodeError,UnicodeEncodeError,):
                    continue

        with self.WriteLock:
            self.cursor.executemany('INSERT INTO packagesets VALUES (?,?)', mysets)

    def retrievePackageSets(self):
        if not self.doesTableExist("packagesets"): return {}
        self.cursor.execute('SELECT setname,dependency FROM packagesets')
        data = self.cursor.fetchall()
        sets = {}
        for setname, dependency in data:
            if not sets.has_key(setname):
                sets[setname] = set()
            sets[setname].add(dependency)
        return sets

    def retrievePackageSet(self, setname):
        self.cursor.execute('SELECT dependency FROM packagesets WHERE setname = (?)', (setname,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveSystemPackages(self):
        self.cursor.execute('SELECT idpackage FROM systempackages')
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveAtom(self, idpackage):
        self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        atom = self.cursor.fetchone()
        if atom: return atom[0]

    def retrieveBranch(self, idpackage):
        self.cursor.execute('SELECT branch FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        br = self.cursor.fetchone()
        if br: return br[0]

    def retrieveTrigger(self, idpackage, get_unicode = False):
        self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,))
        trigger = self.cursor.fetchone()
        if trigger:
            trigger = trigger[0]
        else:
            trigger = ''
        if get_unicode:
            trigger = unicode(trigger,'raw_unicode_escape')
        return trigger

    def retrieveDownloadURL(self, idpackage):
        self.cursor.execute('SELECT download FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        download = self.cursor.fetchone()
        if download: return download[0]

    def retrieveDescription(self, idpackage):
        self.cursor.execute('SELECT description FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        description = self.cursor.fetchone()
        if description: return description[0]

    def retrieveHomepage(self, idpackage):
        self.cursor.execute('SELECT homepage FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        home = self.cursor.fetchone()
        if home: return home[0]

    def retrieveCounter(self, idpackage):
        counter = -1
        self.cursor.execute('SELECT counters.counter FROM counters,baseinfo WHERE counters.idpackage = (?) AND baseinfo.idpackage = counters.idpackage AND baseinfo.branch = counters.branch', (idpackage,))
        mycounter = self.cursor.fetchone()
        if mycounter: return mycounter[0]
        return counter

    def retrieveMessages(self, idpackage):
        messages = []
        try:
            self.cursor.execute('SELECT message FROM messages WHERE idpackage = (?)', (idpackage,))
            messages = self.fetchall2list(self.cursor.fetchall())
        except:
            pass
        return messages

    # in bytes
    def retrieveSize(self, idpackage):
        self.cursor.execute('SELECT size FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        size = self.cursor.fetchone()
        if size: return size[0]

    # in bytes
    def retrieveOnDiskSize(self, idpackage):
        self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,))
        size = self.cursor.fetchone() # do not use [0]!
        if not size: size = 0
        else: size = size[0]
        return size

    def retrieveDigest(self, idpackage):
        self.cursor.execute('SELECT digest FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        digest = self.cursor.fetchone()
        if digest: return digest[0]

    def retrieveName(self, idpackage):
        self.cursor.execute('SELECT name FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        name = self.cursor.fetchone()
        if name: return name[0]

    def retrieveKeySlot(self, idpackage):
        self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
        data = self.cursor.fetchone()
        return data

    def retrieveKeySlotTag(self, idpackage):
        self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot,baseinfo.versiontag FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
        data = self.cursor.fetchone()
        return data

    def retrieveVersion(self, idpackage):
        self.cursor.execute('SELECT version FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        ver = self.cursor.fetchone()
        if ver: return ver[0]

    def retrieveRevision(self, idpackage):
        self.cursor.execute('SELECT revision FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        rev = self.cursor.fetchone()
        if rev: return rev[0]

    def retrieveDateCreation(self, idpackage):
        self.cursor.execute('SELECT datecreation FROM extrainfo WHERE idpackage = (?)', (idpackage,))
        date = self.cursor.fetchone()
        if date: return date[0]

    def retrieveApi(self, idpackage):
        self.cursor.execute('SELECT etpapi FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        api = self.cursor.fetchone()
        if api: return api[0]

    def retrieveUseflags(self, idpackage):
        self.cursor.execute('SELECT flagname FROM useflags,useflagsreference WHERE useflags.idpackage = (?) and useflags.idflag = useflagsreference.idflag', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveEclasses(self, idpackage):
        self.cursor.execute('SELECT classname FROM eclasses,eclassesreference WHERE eclasses.idpackage = (?) and eclasses.idclass = eclassesreference.idclass', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveNeededRaw(self, idpackage):
        self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveNeeded(self, idpackage, extended = False, format = False):

        if extended:
            self.cursor.execute('SELECT library,elfclass FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded order by library', (idpackage,))
            needed = self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded order by library', (idpackage,))
            needed = self.fetchall2list(self.cursor.fetchall())

        if extended and format:
            data = {}
            for lib,elfclass in needed:
                data[lib] = elfclass
            needed = data

        return needed

    def retrieveConflicts(self, idpackage):
        self.cursor.execute('SELECT conflict FROM conflicts WHERE idpackage = (?)', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveProvide(self, idpackage):
        self.cursor.execute('SELECT atom FROM provide WHERE idpackage = (?)', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveDependenciesList(self, idpackage):
        self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) AND dependencies.iddependency = dependenciesreference.iddependency UNION SELECT "!" || conflict FROM conflicts WHERE idpackage = (?)', (idpackage,idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrievePostDependencies(self, idpackage):
        self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) AND dependencies.iddependency = dependenciesreference.iddependency AND dependencies.type = (?)', (idpackage,etpConst['spm']['pdepend_id'],))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveDependencies(self, idpackage, extended = False, deptype = None):

        searchdata = [idpackage]

        depstring = ''
        if deptype != None:
            depstring = ' and dependencies.type = (?)'
            searchdata.append(deptype)

        if extended:
            self.cursor.execute('SELECT dependenciesreference.dependency,dependencies.type FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) and dependencies.iddependency = dependenciesreference.iddependency'+depstring, searchdata)
            deps = self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) and dependencies.iddependency = dependenciesreference.iddependency'+depstring, searchdata)
            deps = self.fetchall2set(self.cursor.fetchall())

        return deps

    def retrieveIdDependencies(self, idpackage):
        self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = (?)', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveDependencyFromIddependency(self, iddependency):
        self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = (?)', (iddependency,))
        dep = self.cursor.fetchone()
        if dep: dep = dep[0]
        return dep

    def retrieveKeywords(self, idpackage):
        self.cursor.execute('SELECT keywordname FROM keywords,keywordsreference WHERE keywords.idpackage = (?) and keywords.idkeyword = keywordsreference.idkeyword', (idpackage,))
        return self.fetchall2set(self.cursor.fetchall())

    def retrieveProtect(self, idpackage):
        self.cursor.execute('SELECT protect FROM configprotect,configprotectreference WHERE configprotect.idpackage = (?) and configprotect.idprotect = configprotectreference.idprotect', (idpackage,))
        protect = self.cursor.fetchone()
        if not protect: protect = ''
        else: protect = protect[0]
        return protect

    def retrieveProtectMask(self, idpackage):
        self.cursor.execute('SELECT protect FROM configprotectmask,configprotectreference WHERE idpackage = (?) and configprotectmask.idprotect= configprotectreference.idprotect', (idpackage,))
        protect = self.cursor.fetchone()
        if not protect: protect = ''
        else: protect = protect[0]
        return protect

    def retrieveSources(self, idpackage, extended = False):
        self.cursor.execute('SELECT sourcesreference.source FROM sources,sourcesreference WHERE idpackage = (?) and sources.idsource = sourcesreference.idsource', (idpackage,))
        sources = self.fetchall2set(self.cursor.fetchall())
        if not extended:
            return sources

        source_data = {}
        mirror_str = "mirror://"
        for source in sources:
            source_data[source] = set()
            if source.startswith(mirror_str):
                mirrorname = source.split("/")[2]
                mirror_url =  source.split("/",3)[3:][0]
                source_data[source] |= set([os.path.join(url,mirror_url) for url in self.retrieveMirrorInfo(mirrorname)])
            else:
                source_data[source].add(source)

        return source_data



    def retrieveContent(self, idpackage, extended = False, contentType = None, formatted = False, insert_formatted = False, order_by = ''):

        # like portage does
        self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")

        extstring = ''
        if extended:
            extstring = ",type"
        extstring_idpackage = ''
        if insert_formatted:
            extstring_idpackage = 'idpackage,'

        searchkeywords = [idpackage]
        contentstring = ''
        if contentType:
            searchkeywords.append(contentType)
            contentstring = ' and type = (?)'

        order_by_string = ''
        if order_by:
            order_by_string = ' order by %s' % (order_by,)

        self.cursor.execute('SELECT %s file%s FROM content WHERE idpackage = (?) %s%s' % (extstring_idpackage,extstring,contentstring,order_by_string,), searchkeywords)

        if extended and insert_formatted:
            fl = self.cursor.fetchall()
        elif extended and formatted:
            fl = {}
            items = self.cursor.fetchone()
            while items:
                fl[items[0]] = items[1]
                items = self.cursor.fetchone()
        elif extended:
            fl = self.cursor.fetchall()
        else:
            if order_by:
                fl = self.fetchall2list(self.cursor.fetchall())
            else:
                fl = self.fetchall2set(self.cursor.fetchall())

        return fl

    def retrieveSlot(self, idpackage):
        self.cursor.execute('SELECT slot FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        slot = self.cursor.fetchone()
        if slot: return slot[0]

    def retrieveVersionTag(self, idpackage):
        self.cursor.execute('SELECT versiontag FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        vtag = self.cursor.fetchone()
        if vtag: return vtag[0]

    def retrieveMirrorInfo(self, mirrorname):
        self.cursor.execute('SELECT mirrorlink FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,))
        mirrorlist = self.fetchall2set(self.cursor.fetchall())
        return mirrorlist

    def retrieveCategory(self, idpackage):
        self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
        cat = self.cursor.fetchone()
        if cat: return cat[0]

    def retrieveCategoryDescription(self, category):
        data = {}
        if not self.doesTableExist("categoriesdescription"):
            return data
        #self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
        self.cursor.execute('SELECT description,locale FROM categoriesdescription WHERE category = (?)', (category,))
        description_data = self.cursor.fetchall()
        for description,locale in description_data:
            data[locale] = description
        return data

    def retrieveLicensedata(self, idpackage):

        # insert license information
        if not self.doesTableExist("licensedata"):
            return {}
        licenses = self.retrieveLicense(idpackage)
        if licenses == None:
            return {}
        licenses = licenses.split()
        licdata = {}
        for licname in licenses:
            licname = licname.strip()
            if not self.entropyTools.is_valid_string(licname):
                continue

            self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")

            self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,))
            lictext = self.cursor.fetchone()
            if lictext != None:
                licdata[licname] = str(lictext[0])

        return licdata

    def retrieveLicensedataKeys(self, idpackage):

        if not self.doesTableExist("licensedata"):
            return set()
        licenses = self.retrieveLicense(idpackage)
        if licenses == None:
            return set()
        licenses = licenses.split()
        licdata = set()
        for licname in licenses:
            licname = licname.strip()
            if not self.entropyTools.is_valid_string(licname):
                continue
            self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,))
            licidentifier = self.cursor.fetchone()
            if licidentifier:
                licdata.add(licidentifier[0])

        return licdata

    def retrieveLicenseText(self, license_name):

        if not self.doesTableExist("licensedata"):
            return None

        self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")

        self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (license_name,))
        text = self.cursor.fetchone()
        if not text:
            return None
        return str(text[0])

    def retrieveLicense(self, idpackage):
        self.cursor.execute('SELECT license FROM baseinfo,licenses WHERE baseinfo.idpackage = (?) and baseinfo.idlicense = licenses.idlicense', (idpackage,))
        licname = self.cursor.fetchone()
        if licname: return licname[0]

    def retrieveCompileFlags(self, idpackage):
        self.cursor.execute('SELECT chost,cflags,cxxflags FROM flags,extrainfo WHERE extrainfo.idpackage = (?) and extrainfo.idflags = flags.idflags', (idpackage,))
        flags = self.cursor.fetchone()
        if not flags:
            flags = ("N/A","N/A","N/A")
        return flags

    def retrieveDepends(self, idpackage, atoms = False, key_slot = False):

        # XXX: never remove this, otherwise equo.db (client database) dependstable will be always broken (trust me)
        # sanity check on the table
        if not self.isDependsTableSane(): # is empty, need generation
            self.regenerateDependsTable(output = False)

        if atoms:
            self.cursor.execute('SELECT baseinfo.atom FROM dependstable,dependencies,baseinfo WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency and baseinfo.idpackage = dependencies.idpackage', (idpackage,))
            result = self.fetchall2set(self.cursor.fetchall())
        elif key_slot:
            self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories,dependstable,dependencies WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency and baseinfo.idpackage = dependencies.idpackage and categories.idcategory = baseinfo.idcategory', (idpackage,))
            result = self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT dependencies.idpackage FROM dependstable,dependencies WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency', (idpackage,))
            result = self.fetchall2set(self.cursor.fetchall())

        return result

    def retrieveUnusedIdpackages(self):
        # XXX: never remove this, otherwise equo.db (client database) dependstable will be always broken (trust me)
        # sanity check on the table
        if not self.isDependsTableSane(): # is empty, need generation
            self.regenerateDependsTable(output = False)
        self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage NOT IN (SELECT idpackage FROM dependstable) ORDER BY atom')
        return self.fetchall2list(self.cursor.fetchall())

    # You must provide the full atom to this function
    # WARNING: this function does not support branches
    def isPackageAvailable(self,pkgatom):
        pkgatom = self.entropyTools.removePackageOperators(pkgatom)
        self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)', (pkgatom,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isIDPackageAvailable(self,idpackage):
        self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = (?)', (idpackage,))
        result = self.cursor.fetchone()
        if not result:
            return False
        return True

    def isCategoryAvailable(self,category):
        self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isProtectAvailable(self,protect):
        self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isFileAvailable(self, myfile, get_id = False):
        self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (myfile,))
        result = self.cursor.fetchall()
        if get_id:
            return self.fetchall2set(result)
        elif result:
            return True
        return False

    def resolveNeeded(self, needed, elfclass = -1):

        cache = self.fetchSearchCache(needed,'resolveNeeded')
        if cache != None: return cache

        ldpaths = self.entropyTools.collectLinkerPaths()
        mypaths = [os.path.join(x,needed) for x in ldpaths]

        query = """
        SELECT
                idpackage,file
        FROM
                content
        WHERE
                content.file IN (%s)
        """ % ( ('?,'*len(mypaths))[:-1], )

        self.cursor.execute(query,mypaths)
        results = self.cursor.fetchall()

        if elfclass == -1:
            mydata = set(results)
        else:
            mydata = set()
            for data in results:
                if not os.access(data[1],os.R_OK):
                    continue
                myclass = self.entropyTools.read_elf_class(data[1])
                if myclass == elfclass:
                    mydata.add(data)

        self.storeSearchCache(needed,'resolveNeeded',mydata)
        return mydata

    def isSourceAvailable(self,source):
        self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = (?)', (source,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isDependencyAvailable(self,dependency):
        self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = (?)', (dependency,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isKeywordAvailable(self,keyword):
        self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = (?)', (keyword,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isUseflagAvailable(self,useflag):
        self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = (?)', (useflag,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isEclassAvailable(self,eclass):
        self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isNeededAvailable(self,needed):
        self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isCounterAvailable(self, counter, branch = None, branch_operator = "="):
        params = [counter]
        branch_string = ''
        if branch:
            branch_string = ' and branch '+branch_operator+' (?)'
            params = [counter,branch]

        self.cursor.execute('SELECT counter FROM counters WHERE counter = (?)'+branch_string, params)
        result = self.cursor.fetchone()
        if result: return True
        return False

    def isCounterTrashed(self, counter):
        self.cursor.execute('SELECT counter FROM trashedcounters WHERE counter = (?)', (counter,))
        result = self.cursor.fetchone()
        if result: return True
        return False

    def isLicensedataKeyAvailable(self, license_name):
        if not self.doesTableExist("licensedata"):
            return True
        self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,))
        result = self.cursor.fetchone()
        if not result:
            return False
        return True

    def isLicenseAccepted(self, license_name):
        self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,))
        result = self.cursor.fetchone()
        if not result:
            return False
        return True

    def acceptLicense(self, license_name):
        if self.readOnly or (not self.entropyTools.is_user_in_entropy_group()):
            return
        if self.isLicenseAccepted(license_name):
            return
        with self.WriteLock:
            self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,))
            self.commitChanges()

    def isLicenseAvailable(self,pkglicense):
        if not self.entropyTools.is_valid_string(pkglicense):
            pkglicense = ' '
        self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,))
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def isSystemPackage(self,idpackage):
        self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,))
        result = self.cursor.fetchone()
        if result:
            return True
        return False

    def isInjected(self,idpackage):
        self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,))
        result = self.cursor.fetchone()
        if result:
            return True
        return False

    def areCompileFlagsAvailable(self,chost,cflags,cxxflags):

        self.cursor.execute('SELECT idflags FROM flags WHERE chost = (?) AND cflags = (?) AND cxxflags = (?)', 
            (chost,cflags,cxxflags,)
        )
        result = self.cursor.fetchone()
        if result: return result[0]
        return -1

    def searchBelongs(self, file, like = False, branch = None, branch_operator = "="):

        branchstring = ''
        searchkeywords = [file]
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and baseinfo.branch '+branch_operator+' (?)'

        if like:
            self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file LIKE (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords)
        else:
            self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file = (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords)

        return self.fetchall2set(self.cursor.fetchall())

    ''' search packages that uses the eclass provided '''
    def searchEclassedPackages(self, eclass, atoms = False): # atoms = return atoms directly
        if atoms:
            self.cursor.execute('SELECT baseinfo.atom,eclasses.idpackage FROM baseinfo,eclasses,eclassesreference WHERE eclassesreference.classname = (?) and eclassesreference.idclass = eclasses.idclass and eclasses.idpackage = baseinfo.idpackage', (eclass,))
            return self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (eclass,))
            return self.fetchall2set(self.cursor.fetchall())

    ''' search packages whose versiontag matches the one provided '''
    def searchTaggedPackages(self, tag, atoms = False): # atoms = return atoms directly
        if atoms:
            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE versiontag = (?)', (tag,))
            return self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (tag,))
            return self.fetchall2set(self.cursor.fetchall())

    def searchLicenses(self, mylicense, caseSensitive = False, atoms = False):

        if not self.entropyTools.is_valid_string(mylicense):
            return []

        request = "baseinfo.idpackage"
        if atoms:
            request = "baseinfo.atom,baseinfo.idpackage"

        if caseSensitive:
            self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE licenses.license LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%",))
        else:
            self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE LOWER(licenses.license) LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%".lower(),))
        if atoms:
            return self.cursor.fetchall()
        return self.fetchall2set(self.cursor.fetchall())

    ''' search packages whose slot matches the one provided '''
    def searchSlottedPackages(self, slot, atoms = False): # atoms = return atoms directly
        if atoms:
            self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE slot = (?)', (slot,))
            return self.cursor.fetchall()
        else:
            self.cursor.execute('SELECT idpackage FROM baseinfo WHERE slot = (?)', (slot,))
            return self.fetchall2set(self.cursor.fetchall())

    def searchKeySlot(self, key, slot, branch = None):

        branchstring = ''
        cat,name = key.split("/")
        params = [cat,name,slot]
        if branch:
            params.append(branch)
            branchstring = 'and baseinfo.branch = (?)'

        self.cursor.execute('SELECT idpackage FROM baseinfo,categories WHERE baseinfo.idcategory = categories.idcategory AND categories.category = (?) AND baseinfo.name = (?) AND baseinfo.slot = (?) %s' % (branchstring,), params)
        return self.cursor.fetchall()

    ''' search packages that need the specified library (in neededreference table) specified by keyword '''
    def searchNeeded(self, keyword, like = False):
        if like:
            self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library LIKE (?) and needed.idneeded = neededreference.idneeded', (keyword,))
        else:
            self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = (?) and needed.idneeded = neededreference.idneeded', (keyword,))
	return self.fetchall2set(self.cursor.fetchall())

    ''' search dependency string inside dependenciesreference table and retrieve iddependency '''
    def searchDependency(self, dep, like = False, multi = False, strings = False):
        sign = "="
        if like:
            sign = "LIKE"
            dep = "%"+dep+"%"
        item = 'iddependency'
        if strings:
            item = 'dependency'
        self.cursor.execute('SELECT %s FROM dependenciesreference WHERE dependency %s (?)' % (item,sign,), (dep,))
        if multi:
            return self.fetchall2set(self.cursor.fetchall())
        else:
            iddep = self.cursor.fetchone()
            if iddep:
                iddep = iddep[0]
            else:
                iddep = -1
            return iddep

    ''' search iddependency inside dependencies table and retrieve idpackages '''
    def searchIdpackageFromIddependency(self, iddep):
        self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,))
        return self.fetchall2set(self.cursor.fetchall())

    def searchSets(self, keyword):
        self.cursor.execute('SELECT DISTINCT(setname) FROM packagesets WHERE setname LIKE (?)', ("%"+keyword+"%",))
        return self.fetchall2set(self.cursor.fetchall())

    def searchSimilarPackages(self, mystring, atom = False):
        s_item = 'name'
        if atom: s_item = 'atom'
        self.cursor.execute('SELECT idpackage FROM baseinfo WHERE soundex(%s) = soundex((?)) ORDER BY %s' % (s_item,s_item,), (mystring,))
        return self.fetchall2list(self.cursor.fetchall())

    def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None, order_by = 'atom', just_id = False):

        searchkeywords = ["%"+keyword+"%"]
        slotstring = ''
        if slot:
            searchkeywords.append(slot)
            slotstring = ' and slot = (?)'
        tagstring = ''
        if tag:
            searchkeywords.append(tag)
            tagstring = ' and versiontag = (?)'
        branchstring = ''
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and branch = (?)'
        order_by_string = ''
        if order_by in ("atom","idpackage","branch",):
            order_by_string = ' order by %s' % (order_by,)

        search_elements = 'atom,idpackage,branch'
        if just_id: search_elements = 'idpackage'

        if (sensitive):
            self.cursor.execute('SELECT '+search_elements+' FROM baseinfo WHERE atom LIKE (?)'+slotstring+tagstring+branchstring+order_by_string, searchkeywords)
        else:
            self.cursor.execute('SELECT '+search_elements+' FROM baseinfo WHERE LOWER(atom) LIKE (?)'+slotstring+tagstring+branchstring+order_by_string, searchkeywords)
        if just_id:
            return self.fetchall2list(self.cursor.fetchall())
        return self.cursor.fetchall()

    def searchProvide(self, keyword, slot = None, tag = None, branch = None, justid = False):

        slotstring = ''
        searchkeywords = [keyword]
        if slot:
            searchkeywords.append(slot)
            slotstring = ' and baseinfo.slot = (?)'
        tagstring = ''
        if tag:
            searchkeywords.append(tag)
            tagstring = ' and baseinfo.versiontag = (?)'
        branchstring = ''
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and baseinfo.branch = (?)'
        atomstring = ''
        if not justid:
            atomstring = 'baseinfo.atom,'

        self.cursor.execute('SELECT '+atomstring+'baseinfo.idpackage FROM baseinfo,provide WHERE provide.atom = (?) and provide.idpackage = baseinfo.idpackage'+slotstring+tagstring+branchstring, searchkeywords)

        if justid:
            results = self.fetchall2list(self.cursor.fetchall())
        else:
            results = self.cursor.fetchall()
        return results

    def searchPackagesByDescription(self, keyword):
        self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo WHERE LOWER(extrainfo.description) LIKE (?) and baseinfo.idpackage = extrainfo.idpackage', ("%"+keyword.lower()+"%",))
        return self.cursor.fetchall()

    def searchPackagesByName(self, keyword, sensitive = False, branch = None, justid = False):

        if sensitive:
            searchkeywords = [keyword]
        else:
            searchkeywords = [keyword.lower()]
        branchstring = ''
        atomstring = ''
        if not justid:
            atomstring = 'atom,'
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and branch = (?)'

        if sensitive:
            self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords)
        else:
            self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords)

        if justid:
            results = self.fetchall2list(self.cursor.fetchall())
        else:
            results = self.cursor.fetchall()
        return results


    def searchPackagesByCategory(self, keyword, like = False, branch = None):

        searchkeywords = [keyword]
        branchstring = ''
        if branch:
            searchkeywords.append(branch)
            branchstring = 'and branch = (?)'

        if like:
            self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category LIKE (?) and baseinfo.idcategory = categories.idcategory %s' % (branchstring,), searchkeywords)
        else:
            self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category = (?) and baseinfo.idcategory = categories.idcategory %s' % (branchstring,), searchkeywords)

        return self.cursor.fetchall()

    def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None, justid = False):

        myname = name
        mycat = category
        if not sensitive:
            myname = name.lower()
            mycat = category.lower()

        searchkeywords = [myname,mycat]
        branchstring = ''
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and branch = (?)'
        atomstring = ''
        if not justid:
            atomstring = 'atom,'

        if sensitive:
            self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE name = (?) AND idcategory IN (SELECT idcategory FROM categories WHERE category = (?))'+branchstring, searchkeywords)
        else:
            self.cursor.execute('SELECT '+atomstring+'idpackage FROM baseinfo WHERE LOWER(name) = (?) AND idcategory IN (SELECT idcategory FROM categories WHERE LOWER(category) = (?))'+branchstring, searchkeywords)
            ''

        if justid:
            results = self.fetchall2list(self.cursor.fetchall())
        else:
            results = self.cursor.fetchall()
        return results

    def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False):

        searchkeywords = []
        if sensitive:
            searchkeywords.append(key)
        else:
            searchkeywords.append(key.lower())

        searchkeywords.append(version)

        branchstring = ''
        if branch:
            searchkeywords.append(branch)
            branchstring = ' and branch = (?)'

        if (sensitive):
            self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category || "/" || baseinfo.name = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)
        else:
            self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE LOWER(categories.category) || "/" || LOWER(baseinfo.name) = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)

        results = self.cursor.fetchall()

        return results

    def isPackageScopeAvailable(self, atom, slot, revision):
        searchdata = (atom,slot,revision,)
        self.cursor.execute('SELECT idpackage FROM baseinfo where atom = (?) and slot = (?) and revision = (?)',searchdata)
        rslt = self.cursor.fetchone()
        idreason = 0
        idpackage = -1
        if rslt:
            # check if it's masked
            idpackage, idreason = self.idpackageValidator(rslt[0])
        return idpackage,idreason

    def listAllPackages(self, get_scope = False, order_by = None, branch = None, branch_operator = "="):

        branchstring = ''
        searchkeywords = []
        if branch:
            searchkeywords = [branch]
            branchstring = ' where branch %s (?)' % (str(branch_operator),)

        order_txt = ''
        if order_by:
            order_txt = ' order by '+str(order_by)
        if get_scope:
            self.cursor.execute('SELECT idpackage,atom,slot,revision FROM baseinfo'+order_txt+branchstring,searchkeywords)
        else:
            self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo'+order_txt+branchstring,searchkeywords)
        return self.cursor.fetchall()

    def listAllInjectedPackages(self, justFiles = False):
        self.cursor.execute('SELECT idpackage FROM injected')
        injecteds = self.fetchall2set(self.cursor.fetchall())
        results = set()
        # get download
        for injected in injecteds:
            download = self.retrieveDownloadURL(injected)
            if justFiles:
                results.add(download)
            else:
                results.add((download,injected))
        return results

    def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="):

        branchstring = ''
        if branch:
            branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"'
        if onlycounters:
            self.cursor.execute('SELECT counter FROM counters'+branchstring)
            return self.fetchall2set(self.cursor.fetchall())
        else:
            self.cursor.execute('SELECT counter,idpackage FROM counters'+branchstring)
            return self.cursor.fetchall()

    def listAllIdpackages(self, branch = None, branch_operator = "=", order_by = None):

        branchstring = ''
        orderbystring = ''
        searchkeywords = []
        if branch:
            searchkeywords.append(branch)
            branchstring = ' where branch %s (?)' % (str(branch_operator),)
        if order_by:
            orderbystring = ' order by '+order_by

        self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring+orderbystring, searchkeywords)

        try:
            if order_by:
                results = self.fetchall2list(self.cursor.fetchall())
            else:
                results = self.fetchall2set(self.cursor.fetchall())
            return results
        except self.dbapi2.OperationalError:
            if order_by:
                return []
            return set()

    def listAllDependencies(self, only_deps = False):
        if only_deps:
            self.cursor.execute('SELECT dependency FROM dependenciesreference')
            return self.fetchall2set(self.cursor.fetchall())
        else:
            self.cursor.execute('SELECT * FROM dependenciesreference')
            return self.cursor.fetchall()

    def listAllBranches(self):

        cache = self.live_cache.get('listAllBranches')
        if cache != None:
            return cache

        self.cursor.execute('SELECT distinct branch FROM baseinfo')
        results = self.fetchall2set(self.cursor.fetchall())

        self.live_cache['listAllBranches'] = results.copy()
        return results

    def listIdPackagesInIdcategory(self,idcategory, order_by = 'atom'):
        order_by_string = ''
        if order_by in ("atom","name","version",):
            order_by_string = ' ORDER BY %s' % (order_by,)
        self.cursor.execute('SELECT idpackage FROM baseinfo where idcategory = (?)'+order_by_string, (idcategory,))
        return self.fetchall2set(self.cursor.fetchall())

    def listIdpackageDependencies(self, idpackage):
        self.cursor.execute('SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies WHERE dependencies.idpackage = (?) AND dependenciesreference.iddependency = dependencies.iddependency', (idpackage,))
        return set(self.cursor.fetchall())

    def listBranchPackagesTbz2(self, branch, do_sort = True, full_path = False):
        order_string = ''
        if do_sort: order_string = 'ORDER BY extrainfo.download'
        self.cursor.execute('SELECT extrainfo.download FROM baseinfo,extrainfo WHERE baseinfo.branch = (?) AND baseinfo.idpackage = extrainfo.idpackage %s' % (order_string,), (branch,))

        if do_sort: results = self.fetchall2list(self.cursor.fetchall())
        else: results = self.fetchall2set(self.cursor.fetchall())

        if not full_path: results = [os.path.basename(x) for x in results]
        if do_sort: return results
        return set(results)

    def listAllFiles(self, clean = False, count = False):
        self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
        if count:
            self.cursor.execute('SELECT count(file) FROM content')
        else:
            self.cursor.execute('SELECT file FROM content')
        if count:
            return self.cursor.fetchone()[0]
        else:
            if clean:
                return self.fetchall2set(self.cursor.fetchall())
            else:
                return self.fetchall2list(self.cursor.fetchall())

    def listAllCategories(self, order_by = ''):
        order_by_string = ''
        if order_by: order_by_string = ' order by %s' % (order_by,)
        self.cursor.execute('SELECT idcategory,category FROM categories %s' % (order_by_string,))
        return self.cursor.fetchall()

    def listConfigProtectDirectories(self, mask = False):
        mask_t = ''
        if mask: mask_t = 'mask'
        self.cursor.execute('SELECT DISTINCT(protect) FROM configprotectreference where idprotect >= 1 and idprotect <= (SELECT max(idprotect) FROM configprotect%s) order by protect' % (mask_t,))
        results = self.fetchall2set(self.cursor.fetchall())
        dirs = set()
        for mystr in results:
            dirs |= set(map(unicode,mystr.split()))
        return sorted(list(dirs))

    def switchBranch(self, idpackage, tobranch):
        self.checkReadOnly()

        key, slot = self.retrieveKeySlot(idpackage)

        # if there are entries already, remove idpackage directly
        my_idpackage, result = self.atomMatch(key, matchSlot = slot, matchBranches = (tobranch,))
        if my_idpackage != -1: return False

        # otherwise, update the old one (set the new branch)
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET branch = (?) WHERE idpackage = (?)', (tobranch,idpackage,))
            self.commitChanges()
            self.clearCache()
        return True

    def databaseStructureUpdates(self):

        old_readonly = self.readOnly
        self.readOnly = False

        if not self.doesTableExist("licensedata"):
            self.createLicensedataTable()

        if not self.doesTableExist("licenses_accepted") and (self.dbname == etpConst['clientdbid']):
            self.createLicensesAcceptedTable()

        if not self.doesTableExist("trashedcounters"):
            self.createTrashedcountersTable()

        if not self.doesTableExist("counters"):
            self.createCountersTable()

        if not self.doesTableExist("installedtable") and (self.dbname == etpConst['clientdbid']):
            self.createInstalledTable()

        if not self.doesTableExist("categoriesdescription"):
            self.createCategoriesdescriptionTable()

        if not self.doesTableExist('packagesets'):
            self.createPackagesetsTable()

        self.readOnly = old_readonly
        self.connection.commit()

    def validateDatabase(self):
        self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","baseinfo"))
        rslt = self.cursor.fetchone()
        if rslt == None:
            mytxt = _("baseinfo table not found. Either does not exist or corrupted.")
            raise exceptionTools.SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,))
        self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","extrainfo"))
        rslt = self.cursor.fetchone()
        if rslt == None:
            mytxt = _("extrainfo table not found. Either does not exist or corrupted.")
            raise exceptionTools.SystemDatabaseError("SystemDatabaseError: %s" % (mytxt,))

    def getIdpackagesDifferences(self, foreign_idpackages):
        myids = self.listAllIdpackages()
        if type(foreign_idpackages) in (list,tuple,):
            outids = set(foreign_idpackages)
        else:
            outids = foreign_idpackages
        added_ids = outids - myids
        removed_ids = myids - outids
        return added_ids, removed_ids

    def uniformBranch(self, branch):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE baseinfo SET branch = (?)', (branch,))
            self.commitChanges()
            self.clearCache()

    def alignDatabases(self, dbconn, force = False, output_header = "  ", align_limit = 300):

        added_ids, removed_ids = self.getIdpackagesDifferences(dbconn.listAllIdpackages())

        if not force:
            if len(added_ids) > align_limit: # too much hassle
                return 0
            if len(removed_ids) > align_limit: # too much hassle
                return 0

        if not added_ids and not removed_ids:
            return -1

        mytxt = red("%s, %s ...") % (_("Syncing current database"),_("please wait"),)
        self.updateProgress(
            mytxt,
            importance = 1,
            type = "info",
            header = output_header,
            back = True
        )
        maxcount = len(removed_ids)
        mycount = 0
        for idpackage in removed_ids:
            mycount += 1
            mytxt = "%s: %s" % (red(_("Removing entry")),blue(str(self.retrieveAtom(idpackage))),)
            self.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = output_header,
                back = True,
                count = (mycount,maxcount)
            )
            self.removePackage(idpackage, do_cleanup = False, do_commit = False)

        maxcount = len(added_ids)
        mycount = 0
        for idpackage in added_ids:
            mycount += 1
            mytxt = "%s: %s" % (red(_("Adding entry")),blue(str(dbconn.retrieveAtom(idpackage))),)
            self.updateProgress(
                mytxt,
                importance = 0,
                type = "info",
                header = output_header,
                back = True,
                count = (mycount,maxcount)
            )
            mydata = dbconn.getPackageData(idpackage, get_content = True, content_insert_formatted = True)
            self.addPackage(
                mydata,
                revision = mydata['revision'],
                idpackage = idpackage,
                do_remove = False,
                do_commit = False,
                formatted_content = True
            )

        # do some cleanups
        self.doCleanups()
        # clear caches
        self.clearCache()
        self.commitChanges()
        self.regenerateDependsTable(output = False)

        # verify both checksums, if they don't match, bomb out
        mycheck = self.database_checksum(do_order = True, strict = False)
        outcheck = dbconn.database_checksum(do_order = True, strict = False)
        if mycheck == outcheck:
            return 1
        return 0

    def checkDatabaseApi(self):

        dbapi = self.getApi()
        if int(dbapi) > int(etpConst['etpapi']):
            self.updateProgress(
                red(_("Repository EAPI > Entropy EAPI. Please update Equo/Entropy as soon as possible !")),
                importance = 1,
                type = "warning",
                header = " * ! * ! * ! * "
            )

    def doDatabaseImport(self, dumpfile, dbfile):
        sqlite3_exec = "/usr/bin/sqlite3 %s < %s" % (dbfile,dumpfile,)
        retcode = subprocess.call(sqlite3_exec, shell = True)
        return retcode

    def doDatabaseExport(self, dumpfile, gentle_with_tables = True):

        dumpfile.write("BEGIN TRANSACTION;\n")
        self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'")
        for name, x, sql in self.cursor.fetchall():

            self.updateProgress(
                red("%s " % (_("Exporting database table"),) )+"["+blue(str(name))+"]",
                importance = 0,
                type = "info",
                back = True,
                header = "   "
            )

            if name == "sqlite_sequence":
                dumpfile.write("DELETE FROM sqlite_sequence;\n")
            elif name == "sqlite_stat1":
                dumpfile.write("ANALYZE sqlite_master;\n")
            elif name.startswith("sqlite_"):
                continue
            else:
                t_cmd = "CREATE TABLE"
                if sql.startswith(t_cmd) and gentle_with_tables:
                    sql = "CREATE TABLE IF NOT EXISTS"+sql[len(t_cmd):]
                dumpfile.write("%s;\n" % sql)

            self.cursor.execute("PRAGMA table_info('%s')" % name)
            cols = [str(r[1]) for r in self.cursor.fetchall()]
            q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES("
            q += ", ".join(["'||quote(" + x + ")||'" for x in cols])
            q += ")' FROM '%(tbl_name)s'"
            self.cursor.execute(q % {'tbl_name': name})
            self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
            for row in self.cursor:
                dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape')))

        self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'")
        for name, x, sql in self.cursor.fetchall():
            dumpfile.write("%s;\n" % sql)

        dumpfile.write("COMMIT;\n")
        try:
            dumpfile.flush()
        except:
            pass
        self.updateProgress(
            red(_("Database Export completed.")),
            importance = 0,
            type = "info",
            header = "   "
        )
        # remember to close the file


    # FIXME: this is only compatible with SQLITE
    def doesTableExist(self, table):
        self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table",table))
        rslt = self.cursor.fetchone()
        if rslt == None:
            return False
        return True

    # FIXME: this is only compatible with SQLITE
    def doesColumnInTableExist(self, table, column):
        self.cursor.execute('PRAGMA table_info( '+table+' )')
        rslt = self.cursor.fetchall()
        if not rslt:
            return False
        found = False
        for row in rslt:
            if row[1] == column:
                found = True
                break
        return found

    # FIXME: this is only compatible with SQLITE
    def getColumnsInTable(self, table):
        self.cursor.execute('PRAGMA table_info( '+table+' )')
        rslt = self.cursor.fetchall()
        columns = []
        for row in rslt:
            columns.append(row[1])
        return columns

    def database_checksum(self, do_order = False, strict = True, strings = False):

        idpackage_order = ''
        category_order = ''
        license_order = ''
        flags_order = ''
        if do_order:
            idpackage_order = 'order by idpackage'
            category_order = 'order by category'
            license_order = 'order by license'
            flags_order = 'order by chost'

        def do_update_md5(m, cursor):
            mydata = cursor.fetchall()
            for record in mydata:
                for item in record:
                    m.update(str(item))

        if strings:
            import hashlib
            m = hashlib.md5()

        self.cursor.execute('select idpackage,atom,name,version,versiontag,revision,branch,slot,etpapi,trigger from baseinfo %s' % (idpackage_order,))
        if strings:
            do_update_md5(m,self.cursor)
        else:
            a_hash = hash(tuple(self.cursor.fetchall()))
        self.cursor.execute('select idpackage,description,homepage,download,size,digest,datecreation from extrainfo %s' % (idpackage_order,))
        if strings:
            do_update_md5(m,self.cursor)
        else:
            b_hash = hash(tuple(self.cursor.fetchall()))
        self.cursor.execute('select category from categories %s' % (category_order,))
        if strings:
            do_update_md5(m,self.cursor)
        else:
            c_hash = hash(tuple(self.cursor.fetchall()))
        d_hash = '0'
        e_hash = '0'
        if strict:
            self.cursor.execute('select * from licenses %s' % (license_order,))
            if strings:
                do_update_md5(m,self.cursor)
            else:
                d_hash = hash(tuple(self.cursor.fetchall()))
            self.cursor.execute('select * from flags %s' % (flags_order,))
            if strings:
                do_update_md5(m,self.cursor)
            else:
                e_hash = hash(tuple(self.cursor.fetchall()))
        if strings:
            return m.hexdigest()
        return "%s:%s:%s:%s:%s" % (a_hash,b_hash,c_hash,d_hash,e_hash,)


########################################################
####
##   Client Database API / but also used by server part
#

    def addPackageToInstalledTable(self, idpackage, repositoryName):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('INSERT into installedtable VALUES (?,?)', (idpackage, repositoryName,))
            self.commitChanges()

    def retrievePackageFromInstalledTable(self, idpackage):
        self.checkReadOnly()
        result = 'Not available'
        with self.WriteLock:
            try:
                self.cursor.execute('SELECT repositoryname FROM installedtable WHERE idpackage = (?)', (idpackage,))
                return self.cursor.fetchone()[0] # it's ok because it's inside try/except
            except:
                pass
        return result

    def removePackageFromInstalledTable(self, idpackage):
        with self.WriteLock:
            self.cursor.execute('DELETE FROM installedtable WHERE idpackage = (?)', (idpackage,))

    def removePackageFromDependsTable(self, idpackage):
        with self.WriteLock:
            try:
                self.cursor.execute('DELETE FROM dependstable WHERE idpackage = (?)', (idpackage,))
                return 0
            except (self.dbapi2.OperationalError,):
                return 1 # need reinit

    def removeDependencyFromDependsTable(self, iddependency):
        self.checkReadOnly()
        with self.WriteLock:
            try:
                self.cursor.execute('DELETE FROM dependstable WHERE iddependency = (?)',(iddependency,))
                self.commitChanges()
                return 0
            except:
                return 1 # need reinit

    # temporary/compat functions
    def createDependsTable(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.executescript("""
                DROP TABLE IF EXISTS dependstable;
                CREATE TABLE dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER );
                INSERT into dependstable VALUES (-1,-1);
            """)
            if self.indexing:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS dependsindex_idpackage ON dependstable ( idpackage )')
            self.commitChanges()

    def sanitizeDependsTable(self):
        with self.WriteLock:
            self.cursor.execute('DELETE FROM dependstable where iddependency = -1')
            self.commitChanges()

    def isDependsTableSane(self):
        try:
            self.cursor.execute('SELECT iddependency FROM dependstable WHERE iddependency = -1')
        except (self.dbapi2.OperationalError,):
            return False # table does not exist, please regenerate and re-run
        status = self.cursor.fetchone()
        if status: return False

        self.cursor.execute('select count(*) from dependstable')
        dependstable_count = self.cursor.fetchone()
        if dependstable_count < 2:
            return False
        return True

    def createXpakTable(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE xpakdata ( idpackage INTEGER PRIMARY KEY, data BLOB );')
            self.commitChanges()

    def storeXpakMetadata(self, idpackage, blob):
        with self.WriteLock:
            self.cursor.execute('INSERT into xpakdata VALUES (?,?)', (int(idpackage),buffer(blob),))
            self.commitChanges()

    def retrieveXpakMetadata(self, idpackage):
        try:
            self.cursor.execute('SELECT data from xpakdata where idpackage = (?)', (idpackage,))
            mydata = self.cursor.fetchone()
            if not mydata:
                return ""
            return mydata[0]
        except:
            return ""

    def createCountersTable(self):
        with self.WriteLock:
            self.cursor.execute("CREATE TABLE IF NOT EXISTS counters ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );")

    def CreatePackedDataTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE packed_data ( idpack INTEGER PRIMARY KEY, data BLOB );')

    def dropAllIndexes(self):
        self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"')
        indexes = self.fetchall2set(self.cursor.fetchall())
        with self.WriteLock:
            for index in indexes:
                if not index.startswith("sqlite"):
                    self.cursor.execute('DROP INDEX IF EXISTS %s' % (index,))

    def listAllIndexes(self, only_entropy = True):
        self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"')
        indexes = self.fetchall2set(self.cursor.fetchall())
        if not only_entropy:
            return indexes
        myindexes = set()
        for index in indexes:
            if index.startswith("sqlite"):
                continue
            myindexes.add(index)
        return myindexes


    def createAllIndexes(self):
        self.createContentIndex()
        self.createBaseinfoIndex()
        self.createKeywordsIndex()
        self.createDependenciesIndex()
        self.createProvideIndex()
        self.createConflictsIndex()
        self.createExtrainfoIndex()
        self.createNeededIndex()
        self.createUseflagsIndex()
        self.createLicensedataIndex()
        self.createLicensesIndex()
        self.createConfigProtectReferenceIndex()
        self.createMessagesIndex()
        self.createSourcesIndex()
        self.createCountersIndex()
        self.createEclassesIndex()
        self.createCategoriesIndex()
        self.createCompileFlagsIndex()
        self.createPackagesetsIndex()

    def createPackagesetsIndex(self):
        if self.indexing:
            with self.WriteLock:
                try:
                    self.cursor.execute('CREATE INDEX IF NOT EXISTS packagesetsindex ON packagesets ( setname )')
                    self.commitChanges()
                except self.dbapi2.OperationalError:
                    pass

    def createNeededIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library );
                    CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded );
                    CREATE INDEX IF NOT EXISTS neededindex_idpackage ON needed ( idpackage );
                    CREATE INDEX IF NOT EXISTS neededindex_elfclass ON needed ( elfclass );
                """)
                self.commitChanges()

    def createMessagesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS messagesindex ON messages ( idpackage )')
                self.commitChanges()

    def createCompileFlagsIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS flagsindex ON flags ( chost,cflags,cxxflags )')
                self.commitChanges()

    def createUseflagsIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idpackage ON useflags ( idpackage );
                    CREATE INDEX IF NOT EXISTS useflagsindex_useflags_idflag ON useflags ( idflag );
                    CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( flagname );
                """)
                self.commitChanges()

    def createContentIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS contentindex_couple ON content ( idpackage );
                    CREATE INDEX IF NOT EXISTS contentindex_file ON content ( file );
                """)
                self.commitChanges()

    def createConfigProtectReferenceIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )')
                self.commitChanges()

    def createBaseinfoIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS baseindex_atom ON baseinfo ( atom );
                    CREATE INDEX IF NOT EXISTS baseindex_branch_name ON baseinfo ( name,branch );
                    CREATE INDEX IF NOT EXISTS baseindex_branch_name_idcategory ON baseinfo ( name,idcategory,branch );
                    CREATE INDEX IF NOT EXISTS baseindex_idcategory ON baseinfo ( idcategory );
                """)
                self.commitChanges()

    def createLicensedataIndex(self):
        if self.indexing:
            if not self.doesTableExist("licensedata"):
                return
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )')
                self.commitChanges()

    def createLicensesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS licensesindex ON licenses ( license )')
                self.commitChanges()

    def createCategoriesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS categoriesindex_category ON categories ( category )')
                self.commitChanges()

    def createKeywordsIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname );
                    CREATE INDEX IF NOT EXISTS keywordsindex_idpackage ON keywords ( idpackage );
                    CREATE INDEX IF NOT EXISTS keywordsindex_idkeyword ON keywords ( idkeyword );
                """)
                self.commitChanges()

    def createDependenciesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS dependenciesindex_idpackage ON dependencies ( idpackage );
                    CREATE INDEX IF NOT EXISTS dependenciesindex_iddependency ON dependencies ( iddependency );
                    CREATE INDEX IF NOT EXISTS dependenciesreferenceindex_dependency ON dependenciesreference ( dependency );
                """)
                self.commitChanges()

    def createCountersIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS countersindex_idpackage ON counters ( idpackage );
                    CREATE INDEX IF NOT EXISTS countersindex_counter ON counters ( counter );
                """)
                self.commitChanges()

    def createSourcesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS sourcesindex_idpackage ON sources ( idpackage );
                    CREATE INDEX IF NOT EXISTS sourcesindex_idsource ON sources ( idsource );
                    CREATE INDEX IF NOT EXISTS sourcesreferenceindex_source ON sourcesreference ( source );
                """)
                self.commitChanges()

    def createProvideIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS provideindex_idpackage ON provide ( idpackage );
                    CREATE INDEX IF NOT EXISTS provideindex_atom ON provide ( atom );
                """)
                self.commitChanges()

    def createConflictsIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS conflictsindex_idpackage ON conflicts ( idpackage );
                    CREATE INDEX IF NOT EXISTS conflictsindex_atom ON conflicts ( conflict );
                """)
                self.commitChanges()

    def createExtrainfoIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( description )')
                self.commitChanges()

    def createEclassesIndex(self):
        if self.indexing:
            with self.WriteLock:
                self.cursor.executescript("""
                    CREATE INDEX IF NOT EXISTS eclassesindex_idpackage ON eclasses ( idpackage );
                    CREATE INDEX IF NOT EXISTS eclassesindex_idclass ON eclasses ( idclass );
                    CREATE INDEX IF NOT EXISTS eclassesreferenceindex_classname ON eclassesreference ( classname );
                """)
                self.commitChanges()

    def regenerateCountersTable(self, vdb_path, output = False):
        self.checkReadOnly()
        self.createCountersTable()
        # assign a counter to an idpackage
        myids = self.listAllIdpackages()
        for myid in myids:
            # get atom
            myatom = self.retrieveAtom(myid)
            mybranch = self.retrieveBranch(myid)
            myatom = self.entropyTools.remove_tag(myatom)
            myatomcounterpath = "%s%s/%s" % (vdb_path,myatom,etpConst['spm']['xpak_entries']['counter'],)
            if os.path.isfile(myatomcounterpath):
                try:
                    f = open(myatomcounterpath,"r")
                    counter = int(f.readline().strip())
                    f.close()
                except:
                    if output:
                        mytxt = "%s: %s: %s" % (
                            bold(_("ATTENTION")),
                            red(_("cannot open Spm counter file for")),
                            bold(myatom),
                        )
                        self.updateProgress(
                            mytxt,
                            importance = 1,
                            type = "warning"
                        )
                    continue
                # insert id+counter
                with self.WriteLock:
                    try:
                        self.cursor.execute(
                                'INSERT into counters VALUES '
                                '(?,?,?)', ( counter, myid, mybranch )
                        )
                    except self.dbapi2.IntegrityError:
                        if output:
                            mytxt = "%s: %s: %s" % (
                                bold(_("ATTENTION")),
                                red(_("counter for atom is duplicated, ignoring")),
                                bold(myatom),
                            )
                            self.updateProgress(
                                mytxt,
                                importance = 1,
                                type = "warning"
                            )
                        continue
                        # don't trust counters, they might not be unique

        self.commitChanges()

    def clearTreeupdatesEntries(self, repository):
        self.checkReadOnly()
        if not self.doesTableExist("treeupdates"):
            self.createTreeupdatesTable()
        # treeupdates
        with self.WriteLock:
            self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,))
            self.commitChanges()

    def resetTreeupdatesDigests(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('UPDATE treeupdates SET digest = "-1"')
            self.commitChanges()

    #
    # FIXME: remove these when 1.0 will be out
    #

    def migrateCountersTable(self):
        with self.WriteLock:
            self._migrateCountersTable()

    def _migrateCountersTable(self):
        self.cursor.execute('DROP TABLE IF EXISTS counterstemp;')
        self.cursor.execute('CREATE TABLE counterstemp ( counter INTEGER, idpackage INTEGER, branch VARCHAR, PRIMARY KEY(idpackage,branch) );')
        self.cursor.execute('select * from counters')
        countersdata = self.cursor.fetchall()
        self.cursor.executemany('INSERT INTO counterstemp VALUES (?,?,?)',countersdata)
        self.cursor.execute('DROP TABLE counters')
        self.cursor.execute('ALTER TABLE counterstemp RENAME TO counters')
        self.commitChanges()

    def createPackagesetsTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE packagesets ( setname VARCHAR, dependency VARCHAR );')

    def createCategoriesdescriptionTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE categoriesdescription ( category VARCHAR, locale VARCHAR, description VARCHAR );')

    def createTreeupdatesTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );')

    def createTreeupdatesactionsTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE treeupdatesactions ( idupdate INTEGER PRIMARY KEY AUTOINCREMENT, repository VARCHAR, command VARCHAR, branch VARCHAR, date VARCHAR );')

    def createSizesTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );')

    def createEntropyMiscCountersTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE entropy_misc_counters ( idtype INTEGER PRIMARY KEY, counter INTEGER );')

    def createDependenciesTypeColumn(self):
        with self.WriteLock:
            self.cursor.execute('ALTER TABLE dependencies ADD COLUMN type INTEGER;')
            self.cursor.execute('UPDATE dependencies SET type = (?)', (0,))

    def createCountersBranchColumn(self):
        with self.WriteLock:
            self.cursor.execute('ALTER TABLE counters ADD COLUMN branch VARCHAR;')
            idpackages = self.listAllIdpackages()
            for idpackage in idpackages:
                branch = self.retrieveBranch(idpackage)
                self.cursor.execute('UPDATE counters SET branch = (?) WHERE idpackage = (?)', (branch,idpackage,))

    def createTreeupdatesactionsDateColumn(self):
        with self.WriteLock:
            try:
                # if database disk image is malformed, won't raise exception here
                self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN date VARCHAR;')
                mytime = str(self.entropyTools.getCurrentUnixTime())
                self.cursor.execute('UPDATE treeupdatesactions SET date = (?)', (mytime,))
            except:
                pass

    def createTreeupdatesactionsBranchColumn(self):
        with self.WriteLock:
            try: # if database disk image is malformed, won't raise exception here
                self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN branch VARCHAR;')
                self.cursor.execute('UPDATE treeupdatesactions SET branch = (?)', (etpConst['branch'],))
            except:
                pass

    def createNeededElfclassColumn(self):
        with self.WriteLock:
            try: # if database disk image is malformed, won't raise exception here
                self.cursor.execute('ALTER TABLE needed ADD COLUMN elfclass INTEGER;')
                self.cursor.execute('UPDATE needed SET elfclass = -1')
            except:
                pass

    def createContentTypeColumn(self):
        with self.WriteLock:
            try: # if database disk image is malformed, won't raise exception here
                self.cursor.execute('ALTER TABLE content ADD COLUMN type VARCHAR;')
                self.cursor.execute('UPDATE content SET type = "0"')
            except:
                pass

    def createLicensedataTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );')

    def createLicensesAcceptedTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE licenses_accepted ( licensename VARCHAR UNIQUE );')

    def createTrashedcountersTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE trashedcounters ( counter INTEGER );')

    def createTriggerTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );')

    def createTriggerColumn(self):
        self.checkReadOnly()
        with self.WriteLock:
            self.cursor.execute('ALTER TABLE baseinfo ADD COLUMN trigger INTEGER;')
            self.cursor.execute('UPDATE baseinfo SET trigger = 0')

    def createMessagesTable(self):
        with self.WriteLock:
            self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR );")

    def createEclassesTable(self):
        with self.WriteLock:
            self.cursor.execute('DROP TABLE IF EXISTS eclasses;')
            self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;')
            self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );')
            self.cursor.execute('CREATE TABLE eclassesreference ( idclass INTEGER PRIMARY KEY AUTOINCREMENT, classname VARCHAR );')

    def createNeededTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER, elfclass INTEGER );')
            self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY AUTOINCREMENT, library VARCHAR );')

    def createSystemPackagesTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE systempackages ( idpackage INTEGER PRIMARY KEY );')

    def createInjectedTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE injected ( idpackage INTEGER PRIMARY KEY );')

    def createProtectTable(self):
        with self.WriteLock:
            self.cursor.execute('DROP TABLE IF EXISTS configprotect;')
            self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;')
            self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;')
            self.cursor.execute('CREATE TABLE configprotect ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );')
            self.cursor.execute('CREATE TABLE configprotectmask ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );')
            self.cursor.execute('CREATE TABLE configprotectreference ( idprotect INTEGER PRIMARY KEY AUTOINCREMENT, protect VARCHAR );')

    def createInstalledTable(self):
        with self.WriteLock:
            self.cursor.execute('DROP TABLE IF EXISTS installedtable;')
            self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR );')

    def addDependRelationToDependsTable(self, iddependency, idpackage):
        with self.WriteLock:
            self.cursor.execute( 'INSERT into dependstable VALUES (?,?)', (iddependency, idpackage,))
            if (self.entropyTools.is_user_in_entropy_group()) and \
                (self.dbname.startswith(etpConst['serverdbid'])):
                    # force commit even if readonly, this will allow to automagically fix dependstable server side
                    self.connection.commit() # we don't care much about syncing the database since it's quite trivial

    def addDependsRelationToDependsTable(self, iterable):
        with self.WriteLock:
            self.cursor.executemany('INSERT into dependstable VALUES (?,?)', iterable)
            if (self.entropyTools.is_user_in_entropy_group()) and \
                (self.dbname.startswith(etpConst['serverdbid'])):
                    # force commit even if readonly, this will allow to automagically fix dependstable server side
                    self.connection.commit() # we don't care much about syncing the database since it's quite trivial

    def clearDependsTable(self):
        if not self.doesTableExist("dependstable"): return
        self.cursor.execute("DROP TABLE IF EXISTS dependstable")

    '''
       @description: recreate dependstable table in the chosen database, it's used for caching searchDepends requests
       @input Nothing
       @output: Nothing
    '''
    def regenerateDependsTable(self, output = True):

        depends = self.listAllDependencies()
        self.createDependsTable()
        count = 0
        total = len(depends)
        mydata = set()
        am = self.atomMatch
        up = self.updateProgress
        for iddep,atom in depends:
            count += 1
            if output:
                up( red("Resolving %s") % (atom,), importance = 0,
                    type = "info", back = True, count = (count,total)
                )
            idpackage, rc = am(atom)
            if idpackage == -1: continue
            mydata.add((iddep,idpackage))

        if mydata: self.addDependsRelationToDependsTable(mydata)

        # now validate dependstable
        self.sanitizeDependsTable()

    def moveCountersToBranch(self, to_branch):
        with self.WriteLock:
            self.cursor.execute('UPDATE counters SET branch = (?)', to_branch)
            self.commitChanges()
            self.clearCache()

########################################################
####
##   Dependency handling functions
#

    def atomMatchFetchCache(self, *args):
        if self.xcache:
            cached = self.dumpTools.loadobj("%s/%s/%s" % (self.dbMatchCacheKey,self.dbname,hash(tuple(args)),))
            if cached != None: return cached

    def atomMatchStoreCache(self, *args, **kwargs):
        if self.xcache:
            if self.ServiceInterface:
                self.ServiceInterface.Cacher.push("%s/%s/%s" % (self.dbMatchCacheKey,self.dbname,hash(tuple(args)),),kwargs.get('result')[:])
            else:
                self.dumpTools.dumpobj("%s/%s/%s" % (self.dbMatchCacheKey,self.dbname,hash(tuple(args)),),kwargs.get('result')[:])

    def _idpackageValidator_live(self, idpackage, reponame):
        if (idpackage,reponame) in self.ServiceInterface.SystemSettings['live_packagemasking']['mask_matches']:
            # do not cache this
            return -1,self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_live_mask']
        elif (idpackage,reponame) in self.ServiceInterface.SystemSettings['live_packagemasking']['unmask_matches']:
            return idpackage,self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_live_unmask']

    def _idpackageValidator_user_package_mask(self, idpackage, reponame, live):
        # check if user package.mask needs it masked

        mykw = "%smask_ids" % (reponame,)
        user_package_mask_ids = self.ServiceInterface.SystemSettings.get(mykw)
        if not isinstance(user_package_mask_ids,set):
            user_package_mask_ids = set()
            for atom in self.ServiceInterface.SystemSettings['mask']:
                matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                if r != 0: continue
                user_package_mask_ids |= set(matches)
            self.ServiceInterface.SystemSettings[mykw] = user_package_mask_ids
        if idpackage in user_package_mask_ids:
            # sorry, masked
            myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_package_mask']
            self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = -1,myr
            return -1,myr

    def _idpackageValidator_user_package_unmask(self, idpackage, reponame, live):
        # see if we can unmask by just lookin into user package.unmask stuff -> self.ServiceInterface.SystemSettings['unmask']
        mykw = "%sunmask_ids" % (reponame,)
        user_package_unmask_ids = self.ServiceInterface.SystemSettings.get(mykw)
        if not isinstance(user_package_unmask_ids,set):
            user_package_unmask_ids = set()
            for atom in self.ServiceInterface.SystemSettings['unmask']:
                matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                if r != 0: continue
                user_package_unmask_ids |= set(matches)
            self.ServiceInterface.SystemSettings[mykw] = user_package_unmask_ids
        if idpackage in user_package_unmask_ids:
            myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_package_unmask']
            self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = idpackage,myr
            return idpackage,myr

    def _idpackageValidator_packages_db_mask(self, idpackage, reponame, live):
        # check if repository packages.db.mask needs it masked
        repos_mask = self.ServiceInterface.SystemSettings['repos_mask']
        repomask = repos_mask.get(reponame)
        if isinstance(repomask,set):
            # first, seek into generic masking, all branches
            mask_repo_id = "%s_ids@@:of:%s" % (reponame,reponame,) # avoid issues with repository names
            repomask_ids = repos_mask.get(mask_repo_id)
            if not isinstance(repomask_ids,set):
                repomask_ids = set()
                for atom in repomask:
                    matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                    if r != 0: continue
                    repomask_ids |= set(matches)
                repos_mask[mask_repo_id] = repomask_ids
            if idpackage in repomask_ids:
                myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['repository_packages_db_mask']
                self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = -1,myr
                return -1,myr

    def _idpackageValidator_package_license_mask(self, idpackage, reponame, live):
        if self.ServiceInterface.SystemSettings['license_mask']:
            mylicenses = self.retrieveLicense(idpackage)
            mylicenses = mylicenses.strip().split()
            lic_mask = self.ServiceInterface.SystemSettings['license_mask']
            for mylicense in mylicenses:
                if mylicense not in lic_mask: continue
                myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_license_mask']
                self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = -1,myr
                return -1,myr

    def _idpackageValidator_keyword_mask(self, idpackage, reponame, live):

        mykeywords = self.retrieveKeywords(idpackage)
        # XXX WORKAROUND
        if not mykeywords: mykeywords = [''] # ** is fine then
        # firstly, check if package keywords are in etpConst['keywords']
        # (universal keywords have been merged from package.mask)
        for key in etpConst['keywords']:
            if key not in mykeywords: continue
            myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['system_keyword']
            self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = idpackage,myr
            return idpackage,myr

        # if we get here, it means we didn't find mykeywords in etpConst['keywords']
        # we need to seek self.ServiceInterface.SystemSettings['keywords']
        # seek in repository first
        if reponame in self.ServiceInterface.SystemSettings['keywords']['repositories']:
            for keyword in self.ServiceInterface.SystemSettings['keywords']['repositories'][reponame]:
                if keyword not in mykeywords: continue
                keyword_data = self.ServiceInterface.SystemSettings['keywords']['repositories'][reponame].get(keyword)
                if not keyword_data: continue
                if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok
                    myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_repo_package_keywords_all']
                    self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = idpackage,myr
                    return idpackage,myr
                kwd_key = "%s_ids" % (keyword,)
                keyword_data_ids = self.ServiceInterface.SystemSettings['keywords']['repositories'][reponame].get(kwd_key)
                if not isinstance(keyword_data_ids,set):
                    keyword_data_ids = set()
                    for atom in keyword_data:
                        matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                        if r != 0: continue
                        keyword_data_ids |= matches
                    self.ServiceInterface.SystemSettings['keywords']['repositories'][reponame][kwd_key] = keyword_data_ids
                if idpackage in keyword_data_ids:
                    myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_repo_package_keywords']
                    self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = idpackage,myr
                    return idpackage,myr

        # if we get here, it means we didn't find a match in repositories
        # so we scan packages, last chance
        for keyword in self.ServiceInterface.SystemSettings['keywords']['packages']:
            # first of all check if keyword is in mykeywords
            if keyword not in mykeywords: continue
            keyword_data = self.ServiceInterface.SystemSettings['keywords']['packages'].get(keyword)
            if not keyword_data: continue
            kwd_key = "%s_ids" % (keyword,)
            keyword_data_ids = self.ServiceInterface.SystemSettings['keywords']['packages'].get(reponame+kwd_key)
            if not isinstance(keyword_data_ids,set):
                keyword_data_ids = set()
                for atom in keyword_data:
                    # match atom
                    matches, r = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
                    if r != 0: continue
                    keyword_data_ids |= matches
                self.ServiceInterface.SystemSettings['keywords']['packages'][reponame+kwd_key] = keyword_data_ids
            if idpackage in keyword_data_ids:
                # valid!
                myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['user_package_keywords']
                self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = idpackage,myr
                return idpackage,myr



    # function that validate one atom by reading keywords settings
    # self.ServiceInterface.package_match_validator_cache = {} >> function cache
    def idpackageValidator(self,idpackage, live = True):

        if self.dbname == etpConst['clientdbid']:
            return idpackage,0
        elif self.dbname.startswith(etpConst['serverdbid']):
            return idpackage,0

        if self.ServiceInterface == None:
            return idpackage,0
        elif not hasattr(self.ServiceInterface,'SystemSettings'):
            return idpackage,0

        reponame = self.dbname[5:]
        cached = self.ServiceInterface.package_match_validator_cache.get((idpackage,reponame,live))
        if cached != None:
            return cached
        # avoid memleaks
        if len(self.ServiceInterface.package_match_validator_cache) > 10000:
            self.ServiceInterface.package_match_validator_cache.clear()

        if live:
            data = self._idpackageValidator_live(idpackage, reponame)
            if data: return data

        data = self._idpackageValidator_user_package_mask(idpackage, reponame, live)
        if data: return data

        data = self._idpackageValidator_user_package_unmask(idpackage, reponame, live)
        if data: return data

        data = self._idpackageValidator_packages_db_mask(idpackage, reponame, live)
        if data: return data

        data = self._idpackageValidator_package_license_mask(idpackage, reponame, live)
        if data: return data

        data = self._idpackageValidator_keyword_mask(idpackage, reponame, live)
        if data: return data

        # holy crap, can't validate
        myr = self.ServiceInterface.SystemSettings['pkg_masking_reference']['completely_masked']
        self.ServiceInterface.package_match_validator_cache[(idpackage,reponame,live)] = -1,myr
        return -1,myr

    # packages filter used by atomMatch, input must me foundIDs, a list like this:
    # [608,1867]
    def packagesFilter(self, results):
        # keywordsFilter ONLY FILTERS results if
        # self.dbname.startswith(etpConst['dbnamerepoprefix']) => repository database is open
        if not self.dbname.startswith(etpConst['dbnamerepoprefix']):
            return results

        newresults = set()
        for idpackage in results:
            idpackage, reason = self.idpackageValidator(idpackage)
            if idpackage == -1: continue
            newresults.add(idpackage)
        return newresults

    def __filterSlot(self, idpackage, slot):
        if slot == None:
            return idpackage
        dbslot = self.retrieveSlot(idpackage)
        if str(dbslot) == str(slot):
            return idpackage

    def __filterTag(self, idpackage, tag, operators):
        if tag == None:
            return idpackage
        dbtag = self.retrieveVersionTag(idpackage)
        compare = cmp(tag,dbtag)
        if not operators or operators == "=":
            if compare == 0:
                return idpackage
        else:
            return self.__do_operator_compare(idpackage, operators, compare)

    def __filterUse(self, idpackage, use):
        if not use:
            return idpackage
        pkguse = self.retrieveUseflags(idpackage)
        disabled = set([x[1:] for x in use if x.startswith("-")])
        enabled = set([x for x in use if not x.startswith("-")])
        enabled_not_satisfied = enabled - pkguse
        # check enabled
        if enabled_not_satisfied:
            return None
        # check disabled
        disabled_not_satisfied = disabled - pkguse
        if len(disabled_not_satisfied) != len(disabled):
            return None
        return idpackage

    def __do_operator_compare(self, token, operators, compare):
        if operators == ">" and compare == -1:
            return token
        elif operators == ">=" and compare < 1:
            return token
        elif operators == "<" and compare == 1:
            return token
        elif operators == "<=" and compare > -1:
            return token

    def __filterSlotTagUse(self, foundIDs, slot, tag, use, operators):

        def myfilter(idpackage):

            idpackage = self.__filterSlot(idpackage, slot)
            if not idpackage:
                return False

            idpackage = self.__filterUse(idpackage, use)
            if not idpackage:
                return False

            idpackage = self.__filterTag(idpackage, tag, operators)
            if not idpackage:
                return False

            return True

        return set(filter(myfilter,foundIDs))

    '''
       @description: matches the user chosen package name+ver, if possibile, in a single repository
       @input atom: string, atom to match
       @input caseSensitive: bool, should the atom be parsed case sensitive?
       @input matchSlot: string, match atoms with the provided slot
       @input multiMatch: bool, return all the available atoms
       @input matchBranches: tuple or list, match packages only in the specified branches
       @input matchTag: match packages only for the specified tag
       @input matchUse: match packages only if it owns the specified use flags
       @input packagesFilter: enable/disable package.mask/.keywords/.unmask filter
       @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error
    '''
    def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, matchUse = (), packagesFilter = True, matchRevision = None, extendedResults = False, useCache = True):

        if not atom:
            return -1,1

        if useCache:
            cached = self.atomMatchFetchCache(
                atom, caseSensitive, matchSlot,
                multiMatch, matchBranches, matchTag,
                matchUse, packagesFilter, matchRevision,
                extendedResults
            )
            if isinstance(cached,tuple):
                return cached

        atomTag = self.entropyTools.dep_gettag(atom)
        try:
            atomUse = self.entropyTools.dep_getusedeps(atom)
        except exceptionTools.InvalidAtom:
            atomUse = ()
        atomSlot = self.entropyTools.dep_getslot(atom)
        atomRev = self.entropyTools.dep_get_entropy_revision(atom)

        # use match
        scan_atom = self.entropyTools.remove_usedeps(atom)
        if (not matchUse) and (atomUse):
            matchUse = atomUse

        # tag match
        scan_atom = self.entropyTools.remove_tag(scan_atom)
        if (matchTag == None) and (atomTag != None):
            matchTag = atomTag

        # slot match
        scan_atom = self.entropyTools.remove_slot(scan_atom)
        if (matchSlot == None) and (atomSlot != None):
            matchSlot = atomSlot

        # revision match
        scan_atom = self.entropyTools.remove_entropy_revision(scan_atom)
        if (matchRevision == None) and (atomRev != None):
            matchRevision = atomRev

        branch_list = ()
        direction = ''
        justname = True
        pkgkey = ''
        strippedAtom = ''
        foundIDs = []
        dbpkginfo = set()

        if scan_atom:

            while 1:
                pkgversion = ''
                # check for direction
                strippedAtom = self.entropyTools.dep_getcpv(scan_atom)
                if scan_atom[-1] == "*":
                    strippedAtom += "*"
                direction = scan_atom[0:len(scan_atom)-len(strippedAtom)]

                justname = self.entropyTools.isjustname(strippedAtom)
                pkgkey = strippedAtom
                if justname == 0:
                    # get version
                    data = self.entropyTools.catpkgsplit(strippedAtom)
                    if data == None: break # badly formatted
                    pkgversion = data[2]+"-"+data[3]
                    pkgkey = self.entropyTools.dep_getkey(strippedAtom)

                splitkey = pkgkey.split("/")
                if (len(splitkey) == 2):
                    pkgname = splitkey[1]
                    pkgcat = splitkey[0]
                else:
                    pkgname = splitkey[0]
                    pkgcat = "null"

                branch_list = (self.db_branch,)
                if matchBranches:
                    # force to tuple for security
                    branch_list = tuple(matchBranches)
                break


        if branch_list:
            # IDs found in the database that match our search
            foundIDs = self.__generate_found_ids_match(branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch)

        ### FILTERING
        # filter slot and tag
        if foundIDs:
            foundIDs = self.__filterSlotTagUse(foundIDs, matchSlot, matchTag, matchUse, direction)
            if packagesFilter:
                foundIDs = self.packagesFilter(foundIDs)
        ### END FILTERING

        if foundIDs:
            dbpkginfo = self.__handle_found_ids_match(foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, atom, pkgversion)

        if not dbpkginfo:
            if extendedResults:
                x = (-1,1,None,None,None)
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (x,1)
                )
                return x,1
            else:
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (-1,1)
                )
                return -1,1

        if multiMatch:
            if extendedResults:
                x = set([(x[0],0,x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0])) for x in dbpkginfo])
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (x,0)
                )
                return x,0
            else:
                x = set([x[0] for x in dbpkginfo])
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (x,0)
                )
                return x,0

        if len(dbpkginfo) == 1:
            x = dbpkginfo.pop()
            if extendedResults:
                x = (x[0],0,x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0]))
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (x,0)
                )
                return x,0
            else:
                self.atomMatchStoreCache(
                    atom, caseSensitive, matchSlot,
                    multiMatch, matchBranches, matchTag,
                    matchUse, packagesFilter, matchRevision,
                    extendedResults, result = (x[0],0)
                )
                return x[0],0

        dbpkginfo = list(dbpkginfo)
        pkgdata = {}
        versions = set()
        for x in dbpkginfo:
            info_tuple = (x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0]))
            versions.add(info_tuple)
            pkgdata[info_tuple] = x[0]
        newer = self.entropyTools.getEntropyNewerVersion(list(versions))[0]
        x = pkgdata[newer]
        if extendedResults:
            x = (x,0,newer[0],newer[1],newer[2])
            self.atomMatchStoreCache(
                atom, caseSensitive, matchSlot,
                multiMatch, matchBranches, matchTag,
                matchUse, packagesFilter, matchRevision,
                extendedResults, result = (x,0)
            )
            return x,0
        else:
            self.atomMatchStoreCache(
                atom, caseSensitive, matchSlot,
                multiMatch, matchBranches, matchTag,
                matchUse, packagesFilter, matchRevision,
                extendedResults, result = (x,0)
            )
            return x,0

    def __generate_found_ids_match(self, branch_list, pkgkey, pkgname, pkgcat, caseSensitive, multiMatch):
        foundIDs = set()
        for idx in branch_list:

            if pkgcat == "null":
                results = self.searchPackagesByName(
                    pkgname, sensitive = caseSensitive,
                    branch = idx, justid = True
                )
            else:
                results = self.searchPackagesByNameAndCategory(
                    name = pkgname, category = pkgcat, branch = idx,
                    sensitive = caseSensitive, justid = True
                )

            mypkgcat = pkgcat
            mypkgname = pkgname
            virtual = False
            # if it's a PROVIDE, search with searchProvide
            # there's no package with that name
            if (not results) and (mypkgcat == "virtual"):
                virtuals = self.searchProvide(pkgkey, branch = idx, justid = True)
                if virtuals:
                    virtual = True
                    mypkgname = self.retrieveName(virtuals[0])
                    mypkgcat = self.retrieveCategory(virtuals[0])
                    results = virtuals

            # now validate
            if not results:
                continue # search into a stabler branch

            elif (len(results) > 1):

                # if it's because category differs, it's a problem
                foundCat = None
                cats = set()
                for idpackage in results:
                    cat = self.retrieveCategory(idpackage)
                    cats.add(cat)
                    if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)):
                        # in case of virtual packages only (that they're not stored as provide)
                        foundCat = cat

                # if we found something at least...
                if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual","null")):
                    foundCat = sorted(list(cats))[0]

                if not foundCat:
                    # got the issue
                    continue

                # we can use foundCat
                mypkgcat = foundCat

                # we need to search using the category
                if (not multiMatch) and (pkgcat == "null" or virtual):
                    # we searched by name, we need to search using category
                    results = self.searchPackagesByNameAndCategory(
                        name = mypkgname, category = mypkgcat,
                        branch = idx, sensitive = caseSensitive, justid = True
                    )

                # validate again
                if not results:
                    continue  # search into another branch

                # if we get here, we have found the needed IDs
                foundIDs |= set(results)
                break

            else:

                idpackage = results[0]
                # if mypkgcat is virtual, we can force
                if (mypkgcat == "virtual") and (not virtual):
                    # in case of virtual packages only (that they're not stored as provide)
                    mypkgcat = self.retrieveCategory(idpackage)

                # check if category matches
                if mypkgcat != "null":
                    foundCat = self.retrieveCategory(idpackage)
                    if mypkgcat == foundCat:
                        foundIDs.add(idpackage)
                    continue
                foundIDs.add(idpackage)
                break

        return foundIDs


    def __handle_found_ids_match(self, foundIDs, direction, matchTag, matchRevision, justname, strippedAtom, atom, pkgversion):

        dbpkginfo = set()
        # now we have to handle direction
        if ((direction) or ((not direction) and (not justname)) or ((not direction) and (not justname) and strippedAtom.endswith("*"))) and foundIDs:

            if (not justname) and \
                ((direction == "~") or (direction == "=") or \
                (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*"))):
                # any revision within the version specified OR the specified version

                if (direction == '' and not justname):
                    direction = "="

                # remove gentoo revision (-r0 if none)
                if (direction == "="):
                    if (pkgversion.split("-")[-1] == "r0"):
                        pkgversion = self.entropyTools.remove_revision(pkgversion)
                if (direction == "~"):
                    pkgrevision = self.entropyTools.dep_get_portage_revision(pkgversion)
                    pkgversion = self.entropyTools.remove_revision(pkgversion)

                for idpackage in foundIDs:

                    dbver = self.retrieveVersion(idpackage)
                    if (direction == "~"):
                        myrev = self.entropyTools.dep_get_portage_revision(dbver)
                        myver = self.entropyTools.remove_revision(dbver)
                        if myver == pkgversion and pkgrevision <= myrev:
                            # found
                            dbpkginfo.add((idpackage,dbver))
                    else:
                        # media-libs/test-1.2* support
                        if pkgversion[-1] == "*":
                            if dbver.startswith(pkgversion[:-1]):
                                dbpkginfo.add((idpackage,dbver))
                        elif (matchRevision != None) and (pkgversion == dbver):
                            dbrev = self.retrieveRevision(idpackage)
                            if dbrev == matchRevision:
                                dbpkginfo.add((idpackage,dbver))
                        elif (pkgversion == dbver) and (matchRevision == None):
                            dbpkginfo.add((idpackage,dbver))

            elif (direction.find(">") != -1) or (direction.find("<") != -1):

                if not justname:

                    # remove revision (-r0 if none)
                    if pkgversion.endswith("r0"):
                        # remove
                        self.entropyTools.remove_revision(pkgversion)

                    for idpackage in foundIDs:

                        revcmp = 0
                        tagcmp = 0
                        if matchRevision != None:
                            dbrev = self.retrieveRevision(idpackage)
                            revcmp = cmp(matchRevision,dbrev)
                        if matchTag != None:
                            dbtag = self.retrieveVersionTag(idpackage)
                            tagcmp = cmp(matchTag,dbtag)
                        dbver = self.retrieveVersion(idpackage)
                        pkgcmp = self.entropyTools.compareVersions(pkgversion,dbver)
                        if isinstance(pkgcmp,tuple):
                            failed = pkgcmp[1]
                            if failed == 0:
                                failed = pkgversion
                            else:
                                failed = dbver
                            # I am sorry, but either pkgversion or dbver are invalid
                            self.updateProgress(
                                bold("atomMatch: ")+red("%s %s %s %s %s. %s: %s") % (
                                    _("comparison between"),
                                    pkgversion,
                                    _("and"),
                                    dbver,
                                    _("failed"),
                                    _("Wrong syntax for"),
                                    failed,
                                ),
                                importance = 1,
                                type = "error",
                                header = darkred(" !!! ")
                            )
                            mytxt = "%s: %s, cmp(): %s, %s: %s" % (
                                _("from atom"),
                                atom,
                                pkgcmp,
                                _("failed"),
                                failed,
                            )
                            raise exceptionTools.InvalidVersionString("InvalidVersionString: %s" % (mytxt, ))
                        if direction == ">":
                            if pkgcmp < 0:
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchRevision != None) and pkgcmp <= 0 and revcmp < 0:
                                #print "found >",self.retrieveAtom(idpackage)
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchTag != None) and tagcmp < 0:
                                dbpkginfo.add((idpackage,dbver))
                        elif direction == "<":
                            if pkgcmp > 0:
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchRevision != None) and pkgcmp >= 0 and revcmp > 0:
                                #print "found <",self.retrieveAtom(idpackage)
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchTag != None) and tagcmp > 0:
                                dbpkginfo.add((idpackage,dbver))
                        elif direction == ">=":
                            if (matchRevision != None) and pkgcmp <= 0:
                                if pkgcmp == 0:
                                    if revcmp <= 0:
                                        dbpkginfo.add((idpackage,dbver))
                                        #print "found >=",self.retrieveAtom(idpackage)
                                else:
                                    dbpkginfo.add((idpackage,dbver))
                            elif pkgcmp <= 0 and matchRevision == None:
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchTag != None) and tagcmp <= 0:
                                dbpkginfo.add((idpackage,dbver))
                        elif direction == "<=":
                            if (matchRevision != None) and pkgcmp >= 0:
                                if pkgcmp == 0:
                                    if revcmp >= 0:
                                        dbpkginfo.add((idpackage,dbver))
                                        #print "found <=",self.retrieveAtom(idpackage)
                                else:
                                    dbpkginfo.add((idpackage,dbver))
                            elif pkgcmp >= 0 and matchRevision == None:
                                dbpkginfo.add((idpackage,dbver))
                            elif (matchTag != None) and tagcmp >= 0:
                                dbpkginfo.add((idpackage,dbver))

        else: # just the key

            dbpkginfo = set([(x,self.retrieveVersion(x)) for x in foundIDs])

        return dbpkginfo


class EmailSender:

    def __init__(self):
        import smtplib
        self.smtplib = smtplib
        from email.mime.audio import MIMEAudio
        from email.mime.image import MIMEImage
        from email.mime.text import MIMEText
        from email.mime.base import MIMEBase
        from email.mime.multipart import MIMEMultipart
        from email import encoders
        from email.message import Message
        import mimetypes
        self.smtpuser = None
        self.smtppassword = None
        self.smtphost = 'localhost'
        self.smtpport = 25
        self.text = MIMEText
        self.mimefile = MIMEBase
        self.audio = MIMEAudio
        self.image = MIMEImage
        self.multipart = MIMEMultipart
        self.default_sender = self.smtp_send
        self.mimetypes = mimetypes
        self.encoders = encoders
        self.Message = Message

    def smtp_send(self, sender, destinations, message):
        s = self.smtplib.SMTP(self.smtphost,self.smtpport)
        if self.smtpuser and self.smtppassword:
            s.login(self.smtpuser,self.smtppassword)
        s.sendmail(sender, destinations, message)
        s.quit()

    def send_text_email(self, sender_email, destination_emails, subject, content):

        # Create a text/plain message
        if isinstance(content,unicode):
            content = content.encode('utf-8')
        if isinstance(subject,unicode):
            subject = subject.encode('utf-8')

        msg = self.text(content)
        msg['Subject'] = subject
        msg['From'] = sender_email
        msg['To'] = ', '.join(destination_emails)
        return self.default_sender(sender_email, destination_emails, msg.as_string())

    def send_mime_email(self, sender_email, destination_emails, subject, content, files):

        outer = self.multipart()
        outer['Subject'] = subject
        outer['From'] = sender_email
        outer['To'] = ', '.join(destination_emails)
        outer.preamble = subject

        mymsg = self.text(content)
        outer.attach(mymsg)

        # attach files
        for myfile in files:
            if not (os.path.isfile(myfile) and os.access(myfile,os.R_OK)):
                continue

            ctype, encoding = self.mimetypes.guess_type(myfile)
            if ctype is None or encoding is not None:
                ctype = 'application/octet-stream'
            maintype, subtype = ctype.split('/', 1)

            if maintype == 'image':
                fp = open(myfile, 'rb')
                msg = self.image(fp.read(), _subtype = subtype)
                fp.close()
            elif maintype == 'audio':
                fp = open(myfile, 'rb')
                msg = self.audio(fp.read(), _subtype = subtype)
                fp.close()
            else:
                fp = open(myfile, 'rb')
                msg = self.mimefile(maintype, subtype)
                msg.set_payload(fp.read())
                fp.close()
                self.encoders.encode_base64(msg)

            msg.add_header('Content-Disposition', 'attachment', filename = os.path.basename(myfile))
            outer.attach(msg)

        composed = outer.as_string()
        return self.default_sender(sender_email, destination_emails, composed)