entropy/lib/entropy/server/interfaces/main.py

# -*- coding: utf-8 -*-
"""

    @author: Fabio Erculiani <lxnay@sabayon.org>
    @contact: lxnay@sabayon.org
    @copyright: Fabio Erculiani
    @license: GPL-2

    B{Entropy Package Manager Server Main Interfaces}.

"""
import sys
import os
import shutil
import copy
import tempfile
import time
import re
import errno
import hashlib
import subprocess
import stat
import codecs

from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
    SystemDatabaseError, RepositoryError
from entropy.const import etpConst, etpSys, const_setup_perms, \
    const_create_working_dirs, const_convert_to_unicode, \
    const_setup_file, const_get_stringtype, const_debug_write, \
    const_debug_enabled, const_convert_to_rawstring
from entropy.output import purple, red, darkgreen, \
    bold, brown, blue, darkred, teal
from entropy.cache import EntropyCacher
from entropy.server.interfaces.mirrors import Server as MirrorsServer
from entropy.i18n import _
from entropy.core.settings.base import SystemSettings
from entropy.core.settings.plugins.skel import SystemSettingsPlugin
from entropy.transceivers import EntropyTransceiver
from entropy.db import EntropyRepository
from entropy.db.skel import EntropyRepositoryPlugin
from entropy.server.interfaces.db import ServerRepositoryStatus, \
    ServerPackagesRepository
from entropy.spm.plugins.factory import get_default_instance as get_spm, \
    get_default_class as get_spm_class
from entropy.qa import QAInterfacePlugin
from entropy.security import Repository as RepositorySecurity
from entropy.db.exceptions import ProgrammingError
from entropy.client.interfaces import Client
from entropy.client.interfaces.db import InstalledPackagesRepository, \
    GenericRepository
from entropy.client.misc import ConfigurationUpdates, ConfigurationFiles

import entropy.dep
import entropy.tools
import entropy.dump

SERVER_QA_PLUGIN = "ServerQAInterfacePlugin"


class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin):

    PLUGIN_ID = "__server__"

    def __init__(self, server_interface, metadata = None):
        """
        Entropy server-side repository ServerPackagesRepository Plugin class.
        This class will be instantiated and automatically added to
        ServerPackagesRepository instances generated by Entropy Server.

        @param server_interface: Entropy Server interface instance
        @type server_interface: entropy.server.interfaces.Server class
        @param metadata: any dict form metadata map (key => value)
        @type metadata: dict
        """
        EntropyRepositoryPlugin.__init__(self)
        self._cacher = EntropyCacher()
        self._settings = SystemSettings()
        self.srv_sys_settings_plugin = \
            etpConst['system_settings_plugins_ids']['server_plugin']
        self._server = server_interface
        if metadata is None:
            self._metadata = {}
        else:
            self._metadata = metadata

    def get_id(self):
        return ServerEntropyRepositoryPlugin.PLUGIN_ID

    def get_metadata(self):
        """
        This method should always return a direct reference to the object and
        NOT a copy.
        """
        return self._metadata

    def add_plugin_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
                self,)
            )

        repo = entropy_repository_instance.repository_id()
        local_dbfile = self._metadata['local_dbfile']
        if local_dbfile is not None:
            taint_file = self._server._get_local_repository_taint_file(
                repo)
            if os.path.isfile(taint_file):
                dbs = ServerRepositoryStatus()
                dbs.set_tainted(local_dbfile)
                dbs.set_bumped(local_dbfile)

        if "__temporary__" in self._metadata: # in-memory db?
            local_dbfile_exists = True
        else:
            local_dbfile_exists = os.path.lexists(local_dbfile)

        if not local_dbfile_exists:
            # better than having a completely broken db
            self._metadata['read_only'] = False
            # force parameters, only ServerEntropyRepository exposes
            # the setReadonly method
            entropy_repository_instance.setReadonly(False)
            entropy_repository_instance.initializeRepository()
            entropy_repository_instance.commit()

        out_intf = self._metadata.get('output_interface')
        if out_intf is not None:
            entropy_repository_instance.output = out_intf.output
            entropy_repository_instance.ask_question = out_intf.ask_question

        return 0

    def close_repo_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling close_repo_hook => %s" % (
                self,)
            )

        # this happens because close_repositories() might be called
        # before _setup_services() and in general, at any time, so, in this
        # case, there is no need to print bullshit to dev.
        if self._server.Mirrors is None:
            return 0

        repo = entropy_repository_instance.repository_id()
        dbfile = self._metadata['local_dbfile']
        if dbfile is None:
            # fake repo, or temporary one
            return 0

        read_only = self._metadata['read_only']
        if not read_only:
            sts = ServerRepositoryStatus()
            if sts.is_tainted(dbfile) and not sts.is_unlock_msg(dbfile):
                u_msg = "[%s] %s" % (brown(repo),
                    darkgreen(_("mirrors have not been unlocked. Sync them.")),)
                self._server.output(
                    u_msg,
                    importance = 1,
                    level = "warning",
                    header = brown(" * ")
                )
                # avoid spamming
                sts.set_unlock_msg(dbfile)

        return 0

    def commit_hook(self, entropy_repository_instance):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling commit_hook => %s" % (
                self,)
            )

        dbs = ServerRepositoryStatus()
        dbfile = self._metadata['local_dbfile']
        if dbfile is None:
            # fake repo, or temporary one
            return 0
        repo = entropy_repository_instance.repository_id()
        read_only = self._metadata['read_only']
        if read_only:
            # do not taint database
            return 0

        # taint the database status
        taint_file = self._server._get_local_repository_taint_file(repo)
        enc = etpConst['conf_encoding']
        with codecs.open(taint_file, "w", encoding=enc) as f:
            f.write("repository tainted\n")
            f.flush()
        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        dbs.set_tainted(dbfile)

        if not dbs.is_bumped(dbfile):
            # bump revision, setting DatabaseBump causes
            # the session to just bump once
            dbs.set_bumped(dbfile)
            """
            Entropy repository revision bumping function.
            Every time it's called,
            revision is incremented by 1.
            """
            revision_file = self._server._get_local_repository_revision_file(
                repo)
            enc = etpConst['conf_encoding']
            if not os.path.isfile(revision_file):
                revision = 1
            else:
                with codecs.open(revision_file, "r", encoding=enc) as rev_f:
                    revision = int(rev_f.readline().strip())
                    revision += 1

            tmp_revision_file = revision_file + ".tmp"
            with codecs.open(tmp_revision_file, "w", encoding=enc) as rev_fw:
                rev_fw.write(str(revision)+"\n")
                rev_fw.flush()
            # atomic !
            os.rename(tmp_revision_file, revision_file)

        if not dbs.are_sets_synced(dbfile):
            # auto-update package sets
            self._server._sync_package_sets(entropy_repository_instance)
            dbs.set_synced_sets(dbfile)

        return 0

    def _get_category_description_from_disk(self, category):
        """
        Get category name description from Source Package Manager.

        @param category: category name
        @type category: string
        @return: category description
        @rtype: string
        """
        spm = self._server.Spm()
        return spm.get_package_category_description_metadata(category)

    def __save_rss(self, srv_repo, rss_name, srv_updates):
        # save to disk
        try:
            self._cacher.save(rss_name, srv_updates,
                cache_dir = Server.CACHE_DIR)
        except IOError as err:
            e_msg = "[%s] %s: %s" % (brown(srv_repo),
                purple(_("cannot store updates RSS cache")),
                repr(err),)
            self._server.output(
                e_msg,
                importance = 1,
                level = "warning",
                header = brown(" * ")
            )

    def _write_rss_for_removed_package(self, repo_db, package_id):

        # setup variables we're going to use
        srv_repo = repo_db.repository_id()
        rss_revision = repo_db.retrieveRevision(package_id)
        rss_atom = "%s~%s" % (repo_db.retrieveAtom(package_id), rss_revision,)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if pkgatom (rss_atom) is in the "added" metadata, drop it
        if rss_atom in srv_updates['added']:
            del srv_updates['added'][rss_atom]
        # same thing for light key
        if rss_atom in srv_updates['light']:
            del srv_updates['light'][rss_atom]

        # add metadata
        mydict = {}
        try:
            mydict['description'] = repo_db.retrieveDescription(package_id)
        except TypeError:
            mydict['description'] = "N/A"
        try:
            mydict['homepage'] = repo_db.retrieveHomepage(package_id)
        except TypeError:
            mydict['homepage'] = ""
        srv_updates['removed'][rss_atom] = mydict

        # save to disk
        self.__save_rss(srv_repo, rss_name, srv_updates)

    def _write_rss_for_added_package(self, repo_db, package_id, package_data):

        # setup variables we're going to use
        srv_repo = repo_db.repository_id()
        rss_atom = "%s~%s" % (package_data['atom'], package_data['revision'],)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if package_data['atom'] (rss_atom) is in the
        # "removed" metadata, drop it
        if rss_atom in srv_updates['removed']:
            del srv_updates['removed'][rss_atom]

        # add metadata
        srv_updates['added'][rss_atom] = {}
        srv_updates['added'][rss_atom]['description'] = \
            package_data['description']
        srv_updates['added'][rss_atom]['homepage'] = \
            package_data['homepage']

        srv_updates['light'][rss_atom] = {}
        srv_updates['light'][rss_atom]['description'] = \
            package_data['description']
        srv_updates['light'][rss_atom]['homepage'] = \
            package_data['homepage']
        srv_updates['light'][rss_atom]['package_id'] = package_id
        date_raw_str = const_convert_to_rawstring(package_data['datecreation'])
        srv_updates['light'][rss_atom]['time_hash'] = \
            hashlib.sha256(date_raw_str).hexdigest()

        # save to disk
        self.__save_rss(srv_repo, rss_name, srv_updates)

    def add_package_hook(self, entropy_repository_instance, package_id,
        package_data):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled']:
            self._write_rss_for_added_package(entropy_repository_instance,
                package_id, package_data)

        try:
            descdata = self._get_category_description_from_disk(
                package_data['category'])
            entropy_repository_instance.setCategoryDescription(
                package_data['category'], descdata)
        except (IOError, OSError, EOFError,):
            pass
        entropy_repository_instance.commit()

        return 0

    def remove_package_hook(self, entropy_repository_instance, package_id,
        from_add_package):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling remove_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled'] \
            and (not from_add_package):

            # store addPackage action
            self._write_rss_for_removed_package(entropy_repository_instance,
                package_id)

        return 0

    def treeupdates_move_action_hook(self, entropy_repository_instance,
        package_id):
        # check for injection and warn the developer
        injected = entropy_repository_instance.isInjected(package_id)
        new_atom = entropy_repository_instance.retrieveAtom(package_id)
        if injected:
            mytxt = "%s: %s %s. %s !!! %s." % (
                bold(_("INJECT")),
                blue(str(new_atom)),
                red(_("has been injected")),
                red(_("quickpkg manually to update embedded db")),
                red(_("Repository updated anyway")),
            )
            self._server.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = darkred(" * ")
            )
        return 0

    def treeupdates_slot_move_action_hook(self, entropy_repository_instance,
        package_id):
        return self.treeupdates_move_action_hook(entropy_repository_instance,
            package_id)


class ServerSystemSettingsPlugin(SystemSettingsPlugin):

    # List of static server-side repositories that must survive
    # a repositories metadata reload
    REPOSITORIES = {}



    def __init__(self, plugin_id, helper_interface):
        SystemSettingsPlugin.__init__(self, plugin_id, helper_interface)
        self._mtime_cache = {}

    @staticmethod
    def server_conf_path():
        """
        Return current server.conf path, this takes into account the current
        configuration files directory path (which is affected by "root" path
        changes [default: /])
        """
        # path to /etc/entropy/server.conf (usually, depends on systemroot)
        return os.path.join(etpConst['confdir'], "server.conf")

    @staticmethod
    def analyze_server_repo_string(repostring, product = None):
        """
        Analyze a server repository string (usually contained in server.conf),
        extracting all the parameters.

        @param repostring: repository string
        @type repostring: string
        @keyword product: system product which repository belongs to
        @rtype: None
        @return: None
        """

        if product is None:
            product = etpConst['product']

        mydata = {}
        repo_key, repostring = entropy.tools.extract_setting(repostring)
        if repo_key != "repository":
            raise AttributeError("invalid repostring passed")

        repo_split = repostring.split("|")
        if len(repo_split) < 3:
            raise AttributeError("invalid repostring passed (2)")

        repoid = repo_split[0].strip()
        repodesc = repo_split[1].strip()
        repouris = repo_split[2].strip()

        mydata = {}
        mydata['repoid'] = repoid
        mydata['description'] = repodesc
        mydata['pkg_mirrors'] = []
        mydata['repo_mirrors'] = []
        mydata['community'] = False
        uris = repouris.split()
        for uri in uris:
            do_pkg = False
            do_repo = False
            while True:
                if uri.startswith("<p>"):
                    do_pkg = True
                    uri = uri[3:]
                    continue
                if uri.startswith("<r>"):
                    do_repo = True
                    uri = uri[3:]
                    continue
                break
            if not (do_repo or do_pkg):
                do_repo = True
                do_pkg = True
            if do_repo:
                mydata['repo_mirrors'].append(uri)
            if do_pkg:
                mydata['pkg_mirrors'].append(uri)

        return repoid, mydata

    def __generic_parser(self, filepath):
        """
        Internal method. This is the generic file parser here.

        @param filepath: valid path
        @type filepath: string
        @return: raw text extracted from file
        @rtype: list
        """
        root = etpConst['systemroot']
        try:
            mtime = os.path.getmtime(filepath)
        except (OSError, IOError):
            mtime = 0.0

        cache_key = (root, filepath)
        cache_obj = self._mtime_cache.get(cache_key)
        if cache_obj is not None:
            if cache_obj['mtime'] == mtime:
                return cache_obj['data']

        cache_obj = {'mtime': mtime,}

        enc = etpConst['conf_encoding']
        data = entropy.tools.generic_file_content_parser(filepath,
            comment_tag = "##", encoding = enc)
        if SystemSettings.DISK_DATA_CACHE:
            cache_obj['data'] = data
            self._mtime_cache[cache_key] = cache_obj
        return data

    def get_updatable_configuration_files(self, repository_id):
        """
        Overridden from SystemSettings.
        """
        files = set()
        # hope that all the repos get synchronized with respect to
        # package names moves
        dep_rewrite_file = Server._get_conf_dep_rewrite_file()
        dep_blacklist_file = Server._get_conf_dep_blacklist_file()
        files.add(dep_rewrite_file)
        files.add(dep_blacklist_file)

        if (repository_id is not None) and \
            (repository_id in self._helper.repositories()):

            critical_file = self._helper._get_local_critical_updates_file(
                repository_id)
            files.add(critical_file)
            keywords_file = self._helper._get_local_repository_keywords_file(
                repository_id)
            files.add(keywords_file)
            mask_file = self._helper._get_local_repository_mask_file(
                repository_id)
            files.add(mask_file)
            bl_file = self._helper._get_missing_dependencies_blacklist_file(
                repository_id)
            files.add(bl_file)
            restricted_file = self._helper._get_local_restricted_file(
                repository_id)
            files.add(restricted_file)
            system_mask_file = \
                self._helper._get_local_repository_system_mask_file(
                    repository_id)
            files.add(system_mask_file)

        return files

    def dep_rewrite_parser(self, sys_set):

        cached = getattr(self, '_mod_rewrite_data', None)
        if cached is not None:
            return cached

        data = {}
        rewrite_file = Server._get_conf_dep_rewrite_file()
        if not os.path.isfile(rewrite_file):
            return data
        rewrite_content = self.__generic_parser(rewrite_file)

        for line in rewrite_content:
            params = line.split()
            if len(params) < 2:
                continue
            pkg_match, pattern, replaces = params[0], params[1], params[2:]
            if pattern.startswith("++"):
                compiled_pattern = None
                pattern = pattern[2:]
                if not pattern:
                    # malformed
                    continue
            else:
                try:
                    compiled_pattern = re.compile(pattern)
                except re.error:
                    # invalid pattern
                    continue
            # use this key to make sure to not overwrite similar entries
            data[(pkg_match, pattern)] = (compiled_pattern, replaces)

        self._mod_rewrite_data = data
        return data

    def dep_blacklist_parser(self, sys_set):

        data = {}
        blacklist_file = Server._get_conf_dep_blacklist_file()
        if not os.path.isfile(blacklist_file):
            return data
        blacklist_content = self.__generic_parser(blacklist_file)

        for line in blacklist_content:
            params = line.split()
            if len(params) < 2:
                continue
            pkg_match, blacklisted_deps = params[0], params[1:]
            # use this key to make sure to not overwrite similar entries
            obj = data.setdefault(pkg_match, [])
            obj.extend(blacklisted_deps)

        return data

    def qa_sets_parser(self, sys_set):

        data = {}
        sets_file = Server._get_conf_qa_sets_file()
        if not os.path.isfile(sets_file):
            return data
        sets_content = self.__generic_parser(sets_file)

        for line in sets_content:
            params = line.split()
            if len(params) < 2:
                continue
            repo_id, qa_sets = params[0], params[1:]
            obj = data.setdefault(repo_id, set())
            obj.update(qa_sets)

        return data

    def server_parser(self, sys_set):

        """
        Parses Entropy server system configuration file.

        @return dict data
        """
        server_conf = ServerSystemSettingsPlugin.server_conf_path()
        root = etpConst['systemroot']
        try:
            mtime = os.path.getmtime(server_conf)
        except (OSError, IOError):
            mtime = 0.0

        serverconf = None
        cache_key = (root, server_conf)
        cache_obj = self._mtime_cache.get(cache_key)
        if cache_obj is not None:
            if cache_obj['mtime'] == mtime:
                serverconf = cache_obj['data']
        else:
            cache_obj = {'mtime': mtime,}

        if serverconf is None:
            enc = etpConst['conf_encoding']
            try:
                with codecs.open(server_conf, "r", encoding=enc) \
                        as server_f:
                    serverconf = [x.strip() for x in server_f.readlines() \
                                      if x.strip()]
            except IOError as err:
                if err.errno != errno.ENOENT:
                    raise
                # if file doesn't exist, provide empty
                # serverconf list. In this way, we make sure that
                # any additional metadata gets added.
                # see the for loop iterating through the
                # repository identifiers
                serverconf = []

            if SystemSettings.DISK_DATA_CACHE:
                cache_obj['data'] = serverconf
                self._mtime_cache[cache_key] = cache_obj

        data = {
            'repositories': ServerSystemSettingsPlugin.REPOSITORIES.copy(),
            'community_mode': False,
            'qa_langs': [const_convert_to_unicode("en_US"),
                         const_convert_to_unicode("C")],
            'default_repository_id': const_convert_to_unicode(
                etpConst['defaultserverrepositoryid']),
            'base_repository_id': None,
            'packages_expiration_days': etpConst['packagesexpirationdays'],
            'database_file_format': const_convert_to_unicode(
                etpConst['etpdatabasefileformat']),
            'disabled_eapis': set(),
            'broken_revdeps_qa_check': True,
            'exp_based_scope': etpConst['expiration_based_scope'],
            # disabled by default for now
            'nonfree_packages_dir_support': False,
            'sync_speed_limit': None,
            'changelog': True,
            'rss': {
                'enabled': etpConst['rss-feed'],
                'name': const_convert_to_unicode(etpConst['rss-name']),
                'light_name': const_convert_to_unicode(
                    etpConst['rss-light-name']),
                'base_url': const_convert_to_unicode(etpConst['rss-base-url']),
                'website_url': const_convert_to_unicode(
                    etpConst['rss-website-url']),
                'editor': const_convert_to_unicode(
                    etpConst['rss-managing-editor']),
                'max_entries': etpConst['rss-max-entries'],
                'light_max_entries': etpConst['rss-light-max-entries'],
            },
        }

        fake_instance = self._helper.fake_default_repo
        default_repo_changed = False

        def _offservrepoid(line, setting):
            # NOTE: remove this in future, supported for backward compat.
            # NOTE: added for backward and mixed compat.
            if default_repo_changed:
                return
            if not fake_instance:
                data['default_repository_id'] = setting.strip()

        def _default_repo(line, setting):
            if not fake_instance:
                data['default_repository_id'] = setting.strip()
            default_repo_changed = True

        def _exp_days(line, setting):
            mydays = setting.strip()
            try:
                mydays = int(mydays)
                data['packages_expiration_days'] = mydays
            except ValueError:
                return

        def _exp_based_scope(line, setting):
            exp_opt = entropy.tools.setting_to_bool(setting)
            if exp_opt is not None:
                data['exp_based_scope'] = exp_opt

        def _nf_packages_dir_sup(line, setting):
            opt = entropy.tools.setting_to_bool(setting)
            if opt is not None:
                data['nonfree_packages_dir_support'] = opt

        def _disabled_eapis(line, setting):
            mydis = setting.strip().split(",")
            try:
                mydis = [int(x) for x in mydis]
                mydis = set([x for x in mydis if x in (1, 2, 3,)])
            except ValueError:
                return
            if (len(mydis) < 3) and mydis:
                data['disabled_eapis'] = mydis

        def _server_basic_lang(line, setting):
            data['qa_langs'] = setting.strip().split()

        def _broken_revdeps_qa(line, setting):
            opt = entropy.tools.setting_to_bool(setting)
            if opt is not None:
                data['broken_revdeps_qa_check'] = opt

        def _repository_func(line, setting):
            try:
                repoid, repodata = \
                    ServerSystemSettingsPlugin.analyze_server_repo_string(
                        line, product = sys_set['repositories']['product'])
            except AttributeError:
                # error parsing string
                return

            # validate repository id string
            if not entropy.tools.validate_repository_id(repoid):
                sys.stderr.write("!!! invalid repository id '%s' in '%s'\n" % (
                    repoid, ServerSystemSettingsPlugin.server_conf_path()))
                return

            if repoid in data['repositories']:
                # just update mirrors
                data['repositories'][repoid]['pkg_mirrors'].extend(
                    repodata['pkg_mirrors'])
                data['repositories'][repoid]['repo_mirrors'].extend(
                    repodata['repo_mirrors'])
            else:
                data['repositories'][repoid] = repodata.copy()

            # base_repository_id support
            if data['base_repository_id'] is None:
                data['base_repository_id'] = repoid

        def _database_format(line, setting):
            if setting in etpConst['etpdatabasesupportedcformats']:
                data['database_file_format'] = setting

        def _syncspeedlimit(line, setting):
            try:
                speed_limit = int(setting)
            except ValueError:
                speed_limit = None
            data['sync_speed_limit'] = speed_limit

        def _changelog(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['changelog'] = bool_setting

        def _community_mode(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['community_mode'] = bool_setting

        def _rss_feed(line, setting):
            bool_setting = entropy.tools.setting_to_bool(setting)
            if bool_setting is not None:
                data['rss']['enabled'] = bool_setting

        def _rss_name(line, setting):
            data['rss']['name'] = setting

        def _rss_light_name(line, setting):
            data['rss']['light_name'] = setting

        def _rss_base_url(line, setting):
            data['rss']['base_url'] = setting

        def _rss_website_url(line, setting):
            data['rss']['website_url'] = setting

        def _managing_editor(line, setting):
            data['rss']['editor'] = setting

        def _max_rss_entries(line, setting):
            try:
                entries = int(setting)
                data['rss']['max_entries'] = entries
            except (ValueError, IndexError,):
                return

        def _max_rss_light_entries(line, setting):
            try:
                entries = int(setting)
                data['rss']['light_max_entries'] = entries
            except (ValueError, IndexError,):
                return

        settings_map = {
            'officialserverrepositoryid': _offservrepoid,
            'default-repository': _default_repo,
            'expiration-days': _exp_days,
            'community-mode': _community_mode,
            'expiration-based-scope': _exp_based_scope,
            'nonfree-packages-directory-support': _nf_packages_dir_sup,
            'disabled-eapis': _disabled_eapis,
            'broken-reverse-deps': _broken_revdeps_qa,
            'server-basic-languages': _server_basic_lang,
            'repository': _repository_func,
            'database-format': _database_format,
            # backward compatibility
            'sync-speed-limit': _syncspeedlimit,
            'syncspeedlimit': _syncspeedlimit,
            'changelog': _changelog,
            'rss-feed': _rss_feed,
            'rss-name': _rss_name,
            'rss-light-name': _rss_light_name,
            'rss-base-url': _rss_base_url,
            'rss-website-url': _rss_website_url,
            'managing-editor': _managing_editor,
            'max-rss-entries': _max_rss_entries,
            'max-rss-light-entries': _max_rss_light_entries,
        }

        for line in serverconf:

            key, value = entropy.tools.extract_setting(line)
            if key is None:
                continue

            func = settings_map.get(key)
            if func is None:
                continue
            func(line, value)

        env_community_mode = os.getenv("ETP_COMMUNITY_MODE")
        if env_community_mode == "0":
            data['community_mode'] = False
        elif env_community_mode == "1":
            data['community_mode'] = True

        # add system database if community repository mode is enabled
        if data['community_mode']:
            client_repository_id = etpConst['clientserverrepoid']
            data['repositories'][client_repository_id] = {}
            mydata = {}
            mydata['description'] = const_convert_to_unicode(
                "Community Repositories System Repository")
            mydata['pkg_mirrors'] = []
            mydata['repo_mirrors'] = []
            mydata['community'] = False
            data['repositories'][client_repository_id].update(mydata)
            ServerSystemSettingsPlugin.REPOSITORIES[client_repository_id] = \
                mydata
            # installed packages repository is now the base repository
            data['base_repository_id'] = etpConst['clientserverrepoid']

        # expand paths
        for repoid in data['repositories']:
            ServerSystemSettingsPlugin.extend_repository_metadata(
                sys_set, repoid, data['repositories'][repoid])

        # Support for shell variables
        shell_repoid = os.getenv('ETP_REPO')
        if shell_repoid:
            data['default_repository_id'] = shell_repoid

        expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
        if expiration_days:
            try:
                expiration_days = int(expiration_days)
                data['packages_expiration_days'] = expiration_days
            except ValueError:
                pass

        return data

    @staticmethod
    def extend_repository_metadata(system_settings, repository_id, metadata):
        """
        Extend server-side Repository metadata dictionary
        with information required by Entropy Server.
        """
        metadata['repo_basedir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id)

        metadata['packages_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir'],
            etpConst['currentarch'])

        metadata['packages_dir_nonfree'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir_nonfree'],
            etpConst['currentarch'])

        metadata['packages_dir_restricted'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            etpConst['packagesrelativepath_basedir_restricted'],
            etpConst['currentarch'])

        metadata['store_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "store",
            etpConst['currentarch'])

        # consider this a base dir
        metadata['upload_basedir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "upload")

        metadata['database_dir'] = os.path.join(
            etpConst['entropyworkdir'],
            "server",
            repository_id,
            "database",
            etpConst['currentarch'])

        metadata['remote_repo_basedir'] = os.path.join(
            system_settings['repositories']['product'],
            repository_id)

        metadata['database_remote_path'] = \
            ServerSystemSettingsPlugin.get_repository_remote_path(
                system_settings, repository_id)
        metadata['override_database_remote_path'] = None

    @staticmethod
    def get_repository_remote_path(system_settings, repository_id):
        return system_settings['repositories']['product'] + "/" + \
            repository_id + "/database/" + etpConst['currentarch']

    @staticmethod
    def set_override_remote_repository(system_settings, repository_id,
        override_repository_id):
        """
        Used to set an overridden remote path where to push repository
        database. This can be used for quickly testing repository changes
        without directly overwriting the real repository.
        """
        repo_path = ServerSystemSettingsPlugin.get_repository_remote_path(
            system_settings, override_repository_id)

        sys_settings_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin']
        srv_data = system_settings[sys_settings_plugin_id]['server']
        repo_data = srv_data['repositories'][repository_id]
        repo_data['override_database_remote_path'] = repo_path


class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

    def repos_parser(self, sys_set):

        cached = getattr(self, '_repos_data', None)
        if cached is not None:
            return cached

        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']
        if not srv_parser_data['exp_based_scope']:
            return data

        # get expiration-based packages removal data from config files
        for repoid in srv_parser_data['repositories']:

            # filter out system repository if community repository
            # mode is enabled
            if repoid == etpConst['clientserverrepoid']:
                continue

            idpackages = set()
            exp_fp = self._helper._get_local_exp_based_pkgs_rm_whitelist_file(
                repoid)
            try:
                dbconn = self._helper.open_server_repository(
                    repoid, just_reading = True)
            except RepositoryError:
                # ignore
                continue

            if os.access(exp_fp, os.R_OK) and os.path.isfile(exp_fp):
                pkgs = entropy.tools.generic_file_content_parser(
                    exp_fp, encoding = etpConst['conf_encoding'])
                if '*' in pkgs: # wildcard support
                    idpackages.add(-1)
                else:
                    for pkg in pkgs:
                        idpackage, rc_match = dbconn.atomMatch(pkg)
                        if rc_match:
                            continue
                        idpackages.add(idpackage)

            data[repoid] = idpackages

        self._repos_data = data
        return data

class ServerFakeClientSystemSettingsPlugin(SystemSettingsPlugin):

    def fake_cli_parser(self, sys_set):
        """
        This is just fake, doesn't bring any new metadata but just tweak
        Entropy client ones.
        """
        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']

        # now setup fake Entropy Client repositories, so that Entropy Server
        # can use Entropy Client interfaces transparently
        srv_repodata = srv_parser_data['repositories']
        cli_repodata = sys_set['repositories']
        # remove unavailable server repos in client metadata first
        cli_repodata['available'].clear()

        for repoid, repo_data in srv_repodata.items():

            try:
                # we must skip the repository validation because
                # repositories have been already validated.
                # moreover, the system repository_id might not be
                # valid (__system__). But still, this is the wanted
                # behaviour.
                xxx, my_data = sys_set._analyze_client_repo_string(
                    "repository = %s|%s|http://--fake--|http://--fake--" \
                        % (repoid, repo_data['description'],),
                    _skip_repository_validation=True)
            except AttributeError as err:
                # yeah, at least let stderr know.
                sys.stderr.write(repr(err) + "\n")
                continue # sorry!

            my_data['repoid'] = repoid
            if '__temporary__' in repo_data:
                # fake repositories, temp ones
                # can't go into Entropy Client, they miss
                # 'database_dir' and other metadata
                my_data['dbpath'] = None
                my_data['__temporary__'] = repo_data['__temporary__']
                my_data['dbrevision'] = 0
            else:
                my_data['dbpath'] = self._helper._get_local_repository_dir(
                    repoid)
                my_data['dbrevision'] = \
                    self._helper.local_repository_revision(
                        repoid)
            cli_repodata['available'][repoid] = my_data

        cli_repodata['default_repository'] = \
            srv_parser_data['default_repository_id']

        del cli_repodata['order'][:]
        if srv_parser_data['base_repository_id'] is not None:
            cli_repodata['order'].append(srv_parser_data['base_repository_id'])
        for repoid in sorted(srv_repodata):
            if repoid not in cli_repodata['order']:
                cli_repodata['order'].append(repoid)

        return data

class ServerQAInterfacePlugin(QAInterfacePlugin):

    def __init__(self, entropy_server_instance):
        self._server = entropy_server_instance

    def __check_package_using_spm(self, package_path):

        spm_class = get_spm_class()
        spm_rc, spm_msg = spm_class.execute_qa_tests(package_path)

        if spm_rc == 0:
            return True
        sys.stderr.write("QA Error: " + spm_msg + "\n")
        sys.stderr.flush()
        return False

    def __extract_edb_analyze_metadata(self, package_path):

        def _is_supported(keywords):
            for arch in etpConst['keywords']:
                if arch in keywords:
                    return True
            return False

        tmp_fd, tmp_f = tempfile.mkstemp(prefix = 'entropy.server')
        dbc = None
        try:
            found_edb = entropy.tools.dump_entropy_metadata(package_path, tmp_f)
            if not found_edb:
                return False
            dbc = self._server._open_temp_repository("test", temp_file = tmp_f,
                initialize = False)
            for package_id in dbc.listAllPackageIds():
                # NOTE: content is tested in entropy.qa builtin package test
                # test content safety
                dbc.retrieveContentSafety(package_id)
                # test keywords
                keywords = dbc.retrieveKeywords(package_id)
                if not _is_supported(keywords):
                    atom = dbc.retrieveAtom(package_id)
                    # big PHAT warning !!
                    self._server.output(darkred("~"*40), level = "warning")
                    self._server.output("[%s, %s] %s" % (
                         brown(os.path.basename(package_path)), teal(atom),
                         purple(_("package has no keyword set, it will be masked !"))),
                        level = "warning", header = darkred(" !!! "))
                    self._server.output(darkred("~"*40), level = "warning")
                    time.sleep(10)
        finally:
            if dbc is not None:
                dbc.close()
            os.close(tmp_fd)

        return True

    def get_tests(self):
        return [self.__check_package_using_spm,
            self.__extract_edb_analyze_metadata]

    def get_id(self):
        return SERVER_QA_PLUGIN


class ServerConfigurationFiles(ConfigurationFiles):

    """
    Subclass Entropy Client version in order to return
    our repository identifiers
    """

    @property
    def _repository_ids(self):
        """
        Return a the list of repository identifiers the object
        is using.
        """
        return self._entropy.repositories()


class Server(Client):

    # Entropy Server cache directory, mainly used for storing commit changes
    CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "server_cache")

    # SystemSettings class variables
    SYSTEM_SETTINGS_PLG_ID = etpConst['system_settings_plugins_ids']['server_plugin']

    def init_singleton(self, default_repository = None, save_repository = False,
            fake_default_repo = False, fake_default_repo_id = None,
            fake_default_repo_desc = None, handle_uninitialized = True,
            **kwargs):

        if fake_default_repo_desc is None:
            fake_default_repo_desc = 'this is a fake repository'
        self.__instance_destroyed = False

        self._cacher = EntropyCacher()
        # settings
        self._memory_db_srv_instances = {}
        self._treeupdates_repos = set()
        self._server_dbcache = {}
        self._settings = SystemSettings()
        etpSys['serverside'] = True
        self.fake_default_repo = fake_default_repo
        self.fake_default_repo_id = fake_default_repo_id
        self._indexing = False
        self.xcache = False
        self.Mirrors = None
        self._settings_to_backup = []
        self._save_repository = save_repository
        self._sync_lock_cache = set()

        self.sys_settings_fake_cli_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fake_client']
        self.sys_settings_fatscope_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fatscope']

        # create our SystemSettings plugin
        with self._settings:
            self.sys_settings_plugin = ServerSystemSettingsPlugin(
                Server.SYSTEM_SETTINGS_PLG_ID, self)
            self._settings.add_plugin(self.sys_settings_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fatscope_plugin = \
                ServerFatscopeSystemSettingsPlugin(
                    self.sys_settings_fatscope_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fatscope_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fake_cli_plugin = \
                ServerFakeClientSystemSettingsPlugin(
                    self.sys_settings_fake_cli_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fake_cli_plugin)

        # setup fake repository
        if fake_default_repo:
            default_repository = fake_default_repo_id
            self._init_generic_memory_server_repository(
                fake_default_repo_id,
                fake_default_repo_desc, set_as_default = True)

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        self._repository = default_repository
        if self._repository is None:
            self._repository = srv_set['default_repository_id']

        if not fake_default_repo:
            if self._repository in srv_set['repositories']:
                try:
                    self._ensure_paths(self._repository)
                except OSError as err:
                    if err.errno != errno.EACCES:
                        raise
                    # sigh, ignore during init

        # if repository is still None, fallback to internal
        # fake repository. This way Entropy Server will work
        # out of the box without any server.conf tweak
        # (and eit bashcomp is happy)
        if self._repository is None:
            repository_id = "__builtin__"
            self._init_generic_memory_server_repository(
                repository_id, "Built-in fallback fake repository",
                set_as_default=True)
            self._repository = repository_id

        if self._repository not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        self._repository,
                        _("repository not configured"),
                    )
            )
        if etpConst['clientserverrepoid'] == self._repository:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                    etpConst['clientserverrepoid'],
                    _("protected repository id, can't use this, sorry dude..."),
                )
            )

        self.switch_default_repository(
            self._repository, handle_uninitialized=handle_uninitialized)
        # initialize Entropy Client superclass
        if "installed_repo" not in kwargs:
            kwargs["installed_repo"] = False
        if "repo_validation" not in kwargs:
            kwargs["repo_validation"] = False
        Client.init_singleton(self,
            indexing = self._indexing,
            xcache = self.xcache,
            **kwargs
        )

    def destroy(self, _from_shutdown = False):
        """
        Destroy this singleton instance.
        """
        self.__instance_destroyed = True
        Client.close_repositories(self, mask_clear = False)
        Client.destroy(self, _from_shutdown = _from_shutdown)

        if not _from_shutdown:
            plug_id2 = self.sys_settings_fake_cli_plugin_id
            plug_id1 = self.sys_settings_fatscope_plugin_id
            plug_id = Server.SYSTEM_SETTINGS_PLG_ID
            # reverse insert order
            plugs = [plug_id2, plug_id1, plug_id]
            for plug in plugs:
                if plug is None:
                    continue
                if not self._settings.has_plugin(plug):
                    continue
                self._settings.remove_plugin(plug)

        self.close_repositories()

    def is_destroyed(self):
        """
        Return whether the singleton instance is destroyed.
        """
        return self.__instance_destroyed

    def _get_branch_from_download_relative_uri(self, db_download_uri):
        return db_download_uri.split("/")[2]

    def _swap_branch_in_download_relative_uri(self, new_branch,
        db_download_uri):
        cur_branch = self._get_branch_from_download_relative_uri(
            db_download_uri)
        return db_download_uri.replace("/%s/" % (cur_branch,),
            "/%s/" % (new_branch,))

    def _get_basedir_pkg_listing(self, base_dir, extension, branch = None):

        pkgs_dir_types = set(self._get_pkg_dir_names())
        basedir_raw_content = []
        entropy.tools.recursive_directory_relative_listing(
            basedir_raw_content, base_dir)

        pkg_ext = extension
        pkg_list = [x for x in basedir_raw_content if x.endswith(pkg_ext)]
        pkg_list = [x for x in pkg_list if \
            x.split(os.path.sep)[0] in pkgs_dir_types]

        if branch is not None:
            branch_extractor = \
                self._get_branch_from_download_relative_uri
            pkg_list = [x for x in pkg_list if branch_extractor(x) == branch]

        return pkg_list

    def _get_pkg_dir_names(self):
        return [etpConst['packagesrelativepath_basedir'],
            etpConst['packagesrelativepath_basedir_nonfree'],
            etpConst['packagesrelativepath_basedir_restricted']]

    def _get_remote_repository_relative_path(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['database_remote_path']

    def _get_override_remote_repository_relative_path(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repo_data = srv_set['repositories'][repository_id]
        return repo_data['override_database_remote_path']

    def _get_local_repository_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasefile'])

    def _get_local_store_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['store_dir']

    def _get_local_upload_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['upload_basedir']

    def _get_local_repository_base_directory(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['repo_basedir']

    def _get_local_repository_taint_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasetaintfile'])

    def _get_local_repository_revision_file(self, repository_id, branch = None):
        return os.path.join(
            self._get_local_repository_dir(repository_id, branch = branch),
                etpConst['etpdatabaserevisionfile'])

    def _get_local_repository_timestamp_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasetimestampfile'])

    def _get_local_repository_mask_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemaskfile'])

    def _get_local_repository_system_mask_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasesytemmaskfile'])

    def _get_local_repository_licensewhitelist_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaselicwhitelistfile'])

    def _get_local_repository_mirrors_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemirrorsfile'])

    def _get_local_repository_fallback_mirrors_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasefallbackmirrorsfile'])

    def _get_local_repository_rss_file(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), srv_set['rss']['name'])

    def _get_local_repository_changelog_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['changelog_filename'])

    def _get_local_repository_compressed_changelog_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['changelog_filename_compressed'])

    def _get_local_repository_rsslight_file(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), srv_set['rss']['light_name'])

    def _get_local_repository_notice_board_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['rss-notice-board'])

    def _get_local_repository_treeupdates_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseupdatefile'])

    def _get_local_repository_compressed_metafiles_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemetafilesfile'])

    def _get_local_repository_metafiles_not_found_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasemetafilesnotfound'])

    def _get_local_repository_gpg_signature_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasegpgfile'])

    def _get_local_exp_based_pkgs_rm_whitelist_file(self, repository_id,
        branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseexpbasedpkgsrm'])

    def _get_local_pkglist_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasepkglist'])

    def _get_local_extra_pkglist_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaseextrapkglist'])

    def _get_local_database_sets_dir(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['confsetsdirname'])

    def _get_local_post_branch_mig_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_branch_hop_script'])

    def _get_local_post_branch_upg_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_branch_upgrade_script'])

    def _get_local_post_repo_update_script(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etp_post_repo_update_script'])

    def _get_local_critical_updates_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasecriticalfile'])

    def _get_local_restricted_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabaserestrictedfile'])

    def _get_local_repository_keywords_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasekeywordsfile'])

    def _get_local_repository_webserv_file(self, repository_id, branch = None):
        return os.path.join(self._get_local_repository_dir(repository_id,
            branch = branch), etpConst['etpdatabasewebservicesfile'])

    def _get_local_repository_dir(self, repository_id, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(
            srv_set['repositories'][repository_id]['database_dir'], branch)

    def _get_missing_dependencies_blacklist_file(self, repository_id,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(
            srv_set['repositories'][repository_id]['database_dir'],
                branch, etpConst['etpdatabasemissingdepsblfile'])

    def _get_repository_lockfile(self, repository_id):
        return os.path.join(self._get_local_repository_dir(repository_id),
            etpConst['etpdatabaselockfile'])

    def _get_repository_download_lockfile(self, repository_id):
        return os.path.join(self._get_local_repository_dir(repository_id),
            etpConst['etpdatabasedownloadlockfile'])

    def _create_local_repository_download_lockfile(self, repository_id):
        lock_file = self._get_repository_download_lockfile(repository_id)
        enc = etpConst['conf_encoding']
        with codecs.open(lock_file, "w", encoding=enc) as f_lock:
            f_lock.write("download locked")
            f_lock.flush()

    def _create_local_repository_lockfile(self, repository_id):
        lock_file = self._get_repository_lockfile(repository_id)
        enc = etpConst['conf_encoding']
        with codecs.open(lock_file, "w", encoding=enc) as f_lock:
            f_lock.write("database locked")
            f_lock.flush()

    def _remove_local_repository_lockfile(self, repository_id):
        lock_file = self._get_repository_lockfile(repository_id)
        try:
            os.remove(lock_file)
        except OSError:
            pass

    def _remove_local_repository_download_lockfile(self, repository_id):
        lock_file = self._get_repository_download_lockfile(repository_id)
        try:
            os.remove(lock_file)
        except OSError:
            pass

    @staticmethod
    def _get_conf_dep_rewrite_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.dep_rewrite")

    @staticmethod
    def _get_conf_dep_blacklist_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.dep_blacklist")

    @staticmethod
    def _get_conf_qa_sets_file():
        packages_dir = SystemSettings.packages_config_directory()
        return os.path.join(packages_dir, "packages.server.sets")

    def complete_remote_package_relative_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['remote_repo_basedir'],
                pkg_rel_url)

    def complete_local_upload_package_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['upload_basedir'],
                pkg_rel_url)

    def complete_local_package_path(self, pkg_rel_url, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return os.path.join(
            srv_set['repositories'][repository_id]['repo_basedir'],
                pkg_rel_url)

    def remote_repository_mirrors(self, repository_id):
        """
        Return a list of remote repository mirrors (database) for given
        repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: list of available repository mirrors
        @rtype: list
        @raise KeyError: if repository_id is invalid
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['repo_mirrors'][:]

    def remote_packages_mirrors(self, repository_id):
        """
        Return a list of remote packages mirrors (packages) for given
        repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: list of available packages mirrors
        @rtype: list
        @raise KeyError: if repository_id is invalid
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]['pkg_mirrors'][:]

    def local_repository_revision(self, repository_id):
        """
        Return local repository revision.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: the actual repository revision
        @rtype: int
        """
        dbrev_file = self._get_local_repository_revision_file(repository_id)
        if not os.path.isfile(dbrev_file):
            return 0

        enc = etpConst['conf_encoding']
        with codecs.open(dbrev_file, "r", encoding=enc) as f_rev:
            rev = f_rev.readline().strip()
        try:
            rev = int(rev)
        except ValueError:
            self.output(
                "[%s] %s: %s - %s" % (
                        darkgreen(repository_id),
                        blue(_("invalid repository revision")),
                        bold(rev),
                        blue(_("defaulting to 0")),
                    ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            rev = 0
        return rev

    def remote_repository_revision(self, repository_id):
        """
        Return the highest repository revision available on mirrors for
        given repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: remote repository revision
        @rtype: int
        """
        repo_status = self.Mirrors.remote_repository_status(repository_id)
        remote_status =  list(repo_status.items())
        if not [x for x in remote_status if x[1]]:
            return 0
        return max([x[1] for x in remote_status])

    def repositories(self):
        """
        Return a list of available Entropy Server repositories.

        @return: list of available Entropy Server repositories
        @rtype: list
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return sorted(srv_set['repositories'])

    def repository(self):
        """
        Return the current repository marked as default.
        """
        return self._repository


    def QA(self):
        """
        Get Entropy QA Interface instance.

        @return: Entropy QA Interface instance
        @rtype: entropy.qa.QAInterface instance
        """
        qa_plugin = ServerQAInterfacePlugin(self)
        qa = Client.QA(self)
        qa.add_plugin(qa_plugin)
        return qa

    def Spm(self):
        """
        Get Source Package Manager interface instance.

        #@return: Source Package Manager interface instance
        @rtype: entropy.spm.plugins.skel.SpmPlugin based instance
        """
        return get_spm(self)

    def Spm_class(self):
        """
        Get Source Package Manager interface class.
        """
        return get_spm_class()

    def ConfigurationUpdates(self):
        """
        Return Entropy Configuration File Updates management object.
        """
        return ConfigurationUpdates(
            self, _config_class=ServerConfigurationFiles)

    def Transceiver(self, uri):
        """
        Get EntropyTransceiver interface instance.

        @param uri: EntropyTransceiver URI
        @type uri: string
        @return: EntropyTransceiver instance
        @rtype: entropy.transceivers.EntropyTransceiver
        """
        txc = EntropyTransceiver(uri)
        txc.set_output_interface(self)
        return txc

    def _sync_package_sets(self, entropy_repository):
        """
        Synchronize repository package sets.

        @param entropy_repository: EntropyRepositoryBase object
        @type entropy_repository: entropy.db.skel.EntropyRepositoryBase
        """
        repository_id = entropy_repository.repository_id()
        self.output(
            "[%s|%s] %s" % (
                    blue(repository_id),
                    red(_("repository")),
                    blue(_("syncing package sets")),
                ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )
        cur_sets = entropy_repository.retrievePackageSets()
        sys_sets = self._get_configured_package_sets(repository_id)
        if cur_sets != sys_sets:
            self._update_package_sets(repository_id, entropy_repository)
        # NOTE: this is called by the commit hook plugin, keep no_plugins=True!
        entropy_repository.commit(no_plugins = True)

    def sets_available(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.available(*args, **kwargs)

    def sets_search(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.search(*args, **kwargs)

    def sets_match(self, *args, **kwargs):
        sets = Client.Sets(self)
        return sets.match(*args, **kwargs)

    def atom_match(self, *args, **kwargs):
        # disable masked packages for server-side repos
        kwargs['mask_filter'] = False
        return Client.atom_match(self, *args, **kwargs)

    def _match_packages(self, repository_id, packages):

        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        if ("world" in packages) or not packages:
            return dbconn.listAllPackageIds(), True
        else:
            idpackages = set()
            for package in packages:
                matches = dbconn.atomMatch(package, multiMatch = True)
                if matches[1] == 0:
                    idpackages |= matches[0]
                else:
                    mytxt = "%s: %s: %s" % (
                        red(_("Attention")),
                        blue(_("cannot match")),
                        bold(package),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "warning",
                        header = darkred(" !!! ")
                    )
            return idpackages, False

    def mask_packages(self, repository_id, packages):
        """
        Mask given package dependencies for given repository, if any (otherwise
        use default one).

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages: list of package dependency strings
        @type packages: list
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_repository_mask_file(repository_id)
        current_packages = []

        if os.path.isfile(mask_file) and os.access(mask_file, os.R_OK):
            current_packages = entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False,
                encoding = etpConst['conf_encoding'])
        # this is untrusted input, it's fine because that config file is
        # untrusted too
        current_packages.extend(packages)

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        enc = etpConst['conf_encoding']
        with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")
            mask_f.flush()
        os.rename(mask_file_tmp, mask_file)

        return True

    def unmask_packages(self, repository_id, packages):
        """
        Unmask given package dependencies for given repository, if any
        (otherwise use default one).

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages: list of package dependency strings
        @type packages: list
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_repository_mask_file(repository_id)
        current_packages = []

        if os.path.isfile(mask_file) and os.access(mask_file, os.R_OK):
            current_packages = entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False,
                encoding = etpConst['conf_encoding'])

        def mask_filter(package):
            if package.startswith("#"):
                # comment, always valid
                return True
            in_file_pkg_match = self.atom_match(package)
            for req_package in packages:
                if package == req_package:
                    # of course remove if it's equal
                    return False
                req_package_match = self.atom_match(req_package)
                if req_package_match == in_file_pkg_match:
                    # drop it, they point to the same package match
                    return False
            return True

        current_packages = list(filter(mask_filter, current_packages))

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        enc = etpConst['conf_encoding']
        with codecs.open(mask_file_tmp, "w", encoding=enc) as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")
            mask_f.flush()
        os.rename(mask_file_tmp, mask_file)

        return True

    def initialize_repository(self, repository_id, ask = True):
        """
        Initialize (and wipe all data!) given repository to empty status.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword ask: ask before making any change?
        @type ask: bool
        @return: execution status (0 = fine)
        @rtype: int
        """
        self.output(
            "[%s] %s..." % (
                purple(repository_id), darkgreen(_("initializing repository")),
            ),
            importance = 1,
            level = "info", header = darkgreen(" * ")
        )
        self.close_repositories()

        rc_question = self.ask_question(
            "[%s] %s" % (
                purple(repository_id),
                teal(_("do you really want to initialize this repository ?"))
            )
        )
        if rc_question == _("No"):
            return 1

        try:
            os.remove(self._get_local_repository_file(repository_id))
        except OSError as err:
            if err.errno != errno.ENOENT:
                raise

        # initialize
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True, is_new = True)
        dbconn.initializeRepository()
        dbconn.commit()

        # create the store directory
        store_dir = self._get_local_store_directory(repository_id)
        if not os.path.isdir(store_dir):
            try:
                os.makedirs(store_dir)
            except (IOError, OSError) as err:
                self.output(
                    "%s: %s" % (_("Cannot create store directory"), err),
                    header=brown(" !!! "),
                    importance=1,
                    level="error")
                return 1

        # create the upload directory
        upload_dir = self._get_local_upload_directory(repository_id)
        if not os.path.isdir(upload_dir):
            try:
                os.makedirs(upload_dir)
            except (IOError, OSError) as err:
                self.output(
                    "%s: %s" % (_("Cannot create upload directory"), err),
                    header=brown(" !!! "),
                    importance=1,
                    level="error")
                return 1

        return 0

    def tag_packages(self, package_matches, package_tag, ask = True):
        """
        Change version tag for given package matches.

        @param package_matches: list of Entropy package matches
        @type package_matches: list
        @param package_tag: new Entropy package tag string
        @type package_tag: string
        @return: execution status (0 = fine)
        @rtype: int
        """
        # repo
        # idpackages

        try:
            package_tag = str(package_tag)
            if " " in package_tag:
                raise ValueError
        except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
            self.output(
                "%s: %s" % (
                    blue(_("Invalid tag specified")),
                    package_tag,
                ),
                importance = 1, level = "error", header = darkred(" !! ")
            )
            return 1

        pkg_map = {}
        for pkg_id, pkg_repo in package_matches:
            obj = pkg_map.setdefault(pkg_repo, [])
            obj.append(pkg_id)

        for pkg_repo in sorted(pkg_map.keys()):
            switched = self._move_packages(pkg_map[pkg_repo], pkg_repo,
                pkg_repo, ask = ask, do_copy = True, new_tag = package_tag)
            if not switched:
                return 1
        return 0

    def flushback_packages(self, repository_id, from_branches, ask = True):
        """
        When creating a new branch, for space reasons, packages are not
        moved to a new location. This works fine until old branch is removed.
        To avoid inconsistences, before deciding to do that, all the packages
        in the old branch should be flushed back to the the currently configured
        branch.

        @param repository_id: repository identifier
        @type repository_id: string
        @param from_branches: list of branches to move packages from
        @type from_branches: list
        @keyword ask: ask before making any change?
        @type ask: bool
        @return execution status (0 = fine)
        @rtype: int
        """
        branch = self._settings['repositories']['branch']

        if branch in from_branches:
            from_branches = [x for x in from_branches if x != branch]

        self.output(
            "[%s=>%s|%s] %s" % (
                darkgreen(', '.join(from_branches)),
                darkred(branch),
                brown(repository_id),
                blue(_("flushing back selected packages from branches")),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)

        idpackage_map = dict(((x, [],) for x in from_branches))
        idpackages = dbconn.listAllPackageIds(order_by = 'atom')
        for idpackage in idpackages:
            download_url = dbconn.retrieveDownloadURL(idpackage)
            url_br = self._get_branch_from_download_relative_uri(
                download_url)
            if url_br in from_branches:
                idpackage_map[url_br].append(idpackage)

        mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
        if not mapped_branches:
            self.output(
                "[%s=>%s|%s] %s !" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("nothing to do")),
                ),
                importance = 0,
                level = "warning",
                header = blue(" @@ ")
            )
            return 0


        all_fine = True
        tmp_down_dir = tempfile.mkdtemp(prefix = "entropy.server")

        download_queue = {}
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)

        def generate_queue(branch, repository_id, from_branch, down_q,
            idpackage_map):

            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repository_id),
                    brown(_("these are the packages that will be flushed")),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )


            for idpackage in idpackage_map[from_branch]:
                atom = dbconn.retrieveAtom(idpackage)
                self.output(
                    "[%s=>%s|%s] %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        purple(atom),
                    ),
                    importance = 0,
                    level = "info",
                    header = blue("  # ")
                )
                pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
                pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
                down_q.append((pkg_fp, idpackage,))


        for from_branch in sorted(mapped_branches):

            download_queue[from_branch] = []
            all_fine = False
            generate_queue(branch, repository_id, from_branch,
                download_queue[from_branch], idpackage_map)

            if ask:
                rc_question = self.ask_question(
                    _("Would you like to continue ?"))
                if rc_question == _("No"):
                    continue

            for uri in self.remote_packages_mirrors(repository_id):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                queue_map = {}

                for pkg_fp, idpackage in download_queue[from_branch]:
                    down_url = dbconn.retrieveDownloadURL(idpackage)
                    down_rel = self.complete_remote_package_relative_path(
                        down_url, repository_id)
                    down_rel_dir = os.path.dirname(down_rel)
                    obj = queue_map.setdefault(down_rel_dir, [])
                    obj.append(pkg_fp)

                errors = False
                m_fine_uris = set()
                m_broken_uris = set()

                for down_rel_dir, downloader_queue in queue_map.items():

                    downloader = self.Mirrors.TransceiverServerHandler(
                        self,
                        [uri],
                        downloader_queue,
                        critical_files = downloader_queue,
                        txc_basedir = down_rel_dir,
                        local_basedir = tmp_down_dir,
                        download = True,
                        repo = repository_id
                    )
                    xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
                    if xerrors:
                        errors = True
                    m_fine_uris.update(xm_fine_uris)
                    m_broken_uris.update(xm_broken_uris)

                if not errors:
                    for downloaded_path, idpackage in \
                        download_queue[from_branch]:

                        self.output(
                            "[%s=>%s|%s|%s] %s: %s" % (
                                darkgreen(from_branch),
                                darkred(branch),
                                brown(repository_id),
                                dbconn.retrieveAtom(idpackage),
                                blue(_("checking package hash")),
                                darkgreen(os.path.basename(downloaded_path)),
                            ),
                            importance = 0,
                            level = "info",
                            header = brown("   "),
                            back = True
                        )

                        md5hash = entropy.tools.md5sum(downloaded_path)
                        db_md5hash = dbconn.retrieveDigest(idpackage)
                        if md5hash != db_md5hash:
                            errors = True
                            self.output(
                                "[%s=>%s|%s|%s] %s: %s" % (
                                    darkgreen(from_branch),
                                    darkred(branch),
                                    brown(repository_id),
                                    dbconn.retrieveAtom(idpackage),
                                    blue(_("hash does not match for")),
                                    darkgreen(os.path.basename(downloaded_path)),
                                ),
                                importance = 0,
                                level = "error",
                                header = brown("   ")
                            )
                            continue

                if errors:
                    reason = _("wrong md5")
                    if m_broken_uris:
                        my_broken_uris = [
                        (EntropyTransceiver.get_uri_name(x), y,) \
                            for x, y in m_broken_uris]
                        reason = my_broken_uris[0][1]

                    self.output(
                        "[%s=>%s|%s] %s, %s: %s" % (
                            darkgreen(from_branch),
                            darkred(branch),
                            brown(repository_id),
                            blue(_("download errors")),
                            blue(_("reason")),
                            reason,
                        ),
                        importance = 1,
                        level = "error",
                        header = darkred(" !!! ")
                    )
                    # continuing if possible
                    continue

                all_fine = True

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("download completed successfully")),
                        darkgreen(crippled_uri),
                    ),
                    importance = 1,
                    level = "info",
                    header = darkgreen(" * ")
                )

        if not all_fine:
            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("error downloading packages from mirrors")),
                ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            return 1

        for from_branch in sorted(mapped_branches):

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repository_id),
                    blue(_("working on branch")),
                    darkgreen(from_branch),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )

            down_queue = download_queue[from_branch]
            for package_path, idpackage in down_queue:

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("updating package")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   "),
                    back = True
                )

                # build new download url
                download_url = dbconn.retrieveDownloadURL(idpackage)
                download_url = \
                    self._swap_branch_in_download_relative_uri(
                        branch, download_url)

                # move files to upload
                new_package_path = self.complete_local_upload_package_path(
                    download_url, repository_id)
                self._ensure_dir_path(os.path.dirname(new_package_path))

                try:
                    os.rename(package_path, new_package_path)
                except OSError as err:
                    if err.errno != errno.EXDEV:
                        raise
                    shutil.move(package_path, new_package_path)

                # update database
                dbconn.setDownloadURL(idpackage, download_url)
                dbconn.commit()
                dbconn.switchBranch(idpackage, branch)
                dbconn.commit()

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repository_id),
                        blue(_("package flushed")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   ")
                )

        try:
            os.rmdir(tmp_down_dir)
        except OSError:
            pass

        return 0

    def move_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, pull_dependencies = False):
        """
        Move packages from a repository to another.

        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        @param from_repository_id: source repository identifier
        @type from_repository_id: string
        @param to_repository_id: destination repository identifier
        @type to_repository_id: string
        @keyword ask: execute in interactive mode
        @type ask: bool
        @keyword pull_dependencies: also include reverse dependencies in
            move
        @type pull_dependencies: bool
        @return: list (set) of moved package identifiers
        @rtype: set
        """
        return self._move_packages(package_ids, from_repository_id,
            to_repository_id, ask = ask, pull_deps = pull_dependencies,
            do_copy = False)

    def copy_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, pull_dependencies = False):
        """
        Copy packages from a repository to another.

        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        @param from_repository_id: source repository identifier
        @type from_repository_id: string
        @param to_repository_id: destination repository identifier
        @type to_repository_id: string
        @keyword ask: execute in interactive mode
        @type ask: bool
        @keyword pull_dependencies: also include reverse dependencies in
            copy
        @type pull_dependencies: bool
        @return: list (set) of copied package identifiers
        @rtype: set
        """
        return self._move_packages(package_ids, from_repository_id,
            to_repository_id, ask = ask, pull_deps = pull_dependencies,
            do_copy = True)

    def _move_packages(self, package_ids, from_repository_id, to_repository_id,
        ask = True, do_copy = False, new_tag = None, pull_deps = False):
        """
        Move, copy or re-tag packages in repositories.
        If do_copy is True, a copy is executed of package_ids from
        from_repository_id to to_repository_id. If new_tag is not None,
        and from_repository_id == to_repository_id a re-tag will be executed.
        """

        switched = set()
        my_matches = [(x, from_repository_id) for x in package_ids]

        # avoid setting __default__ as default server repo
        if etpConst['clientserverrepoid'] in (to_repository_id, 
            from_repository_id):
            self.output(
                "%s: %s" % (
                    blue(_("Cannot touch system repository")),
                    red(etpConst['clientserverrepoid']),
                ),
                importance = 2, level = "warning", header = darkred(" @@ ")
            )
            return switched

        if not my_matches and from_repository_id:
            dbconn = self.open_server_repository(from_repository_id,
                read_only = True, no_upload = True)
            my_matches = set( \
                [(x, from_repository_id) for x in \
                    dbconn.listAllPackageIds()]
            )

        mytxt = _("Preparing to move selected packages to")
        if do_copy:
            mytxt = _("Preparing to copy selected packages to")
        self.output(
            "%s %s:" % (
                blue(mytxt),
                red(to_repository_id),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        self.output(
            "%s: %s" % (
                bold(_("Note")),
                red(_("all old packages with conflicting scope will be " \
                    "removed from destination repo unless injected")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        new_tag_string = ''
        if new_tag != None:
            new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
                brown(new_tag),)

        # open both repos here to make sure it's all fine with them
        dbconn = self.open_server_repository(from_repository_id,
            read_only = True, no_upload = True)
        todbconn = self.open_server_repository(to_repository_id,
            read_only = False, no_upload = True)

        my_qa = self.QA()
        branch = self._settings['repositories']['branch']
        pull_deps_matches = []
        for idpackage, repo in my_matches:
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True)
            self.output(
                "[%s=>%s|%s] %s " % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(dbconn.retrieveAtom(idpackage)),
                ) + new_tag_string,
                importance = 0,
                level = "info",
                header = brown("    # ")
            )

            # check if there are pkgs that are going to be overwritten
            # and warn user about that.
            from_name, from_category, from_slot, from_injected = \
                dbconn.retrieveName(idpackage), \
                dbconn.retrieveCategory(idpackage), \
                dbconn.retrieveSlot(idpackage), \
                dbconn.isInjected(idpackage)
            to_rm_idpackages = todbconn.getPackagesToRemove(from_name,
                from_category, from_slot, from_injected)
            for to_rm_idpackage in to_rm_idpackages:
                self.output(
                    "    [=>%s|%s] %s" % (
                            darkred(to_repository_id),
                            bold(_("remove")),
                            blue(todbconn.retrieveAtom(to_rm_idpackage)),
                    ),
                    importance = 0,
                    level = "info",
                    header = purple("    # ")
                )

            # do we want to pull in also package dependencies?
            if pull_deps:
                dep_matches = my_qa.get_deep_dependency_list(self,
                    (idpackage, repo), match_repo = (repo,))
                revdep_matches = self.get_reverse_queue(dep_matches,
                    system_packages = False)
                dep_matches.update(revdep_matches)

                for dep_idpackage, dep_repo in dep_matches:

                    my_dep_match = (dep_idpackage, dep_repo,)
                    if my_dep_match in pull_deps_matches:
                        continue
                    if my_dep_match in my_matches:
                        continue

                    pull_deps_matches.append(my_dep_match)
                    dep_dbconn = self.open_server_repository(dep_repo,
                        read_only = True, no_upload = True)
                    dep_atom = dep_dbconn.retrieveAtom(dep_idpackage)
                    if my_dep_match in revdep_matches:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("reverse dependency")),
                                teal(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )
                    else:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("dependency")),
                                purple(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )

        if pull_deps:
            # put deps first!
            my_matches = pull_deps_matches + [x for x in my_matches if x not \
                in pull_deps_matches]

        if ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                return switched

        package_ids_added = set()
        for idpackage, repo in my_matches:

            dbconn = self.open_server_repository(repo, read_only = False,
                no_upload = True)

            match_atom = dbconn.retrieveAtom(idpackage)
            package_rel_path = dbconn.retrieveDownloadURL(idpackage)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("switching")),
                    darkgreen(match_atom),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            # move binary file
            from_file = self.complete_local_package_path(package_rel_path,
                repo)
            if not os.path.isfile(from_file):
                from_file = self.complete_local_upload_package_path(
                    package_rel_path, repo)
            if not os.path.isfile(from_file):
                self.output(
                    "[%s=>%s|%s] %s: %s -> %s" % (
                        darkgreen(repo),
                        darkred(to_repository_id),
                        brown(branch),
                        bold(_("cannot switch, package not found, skipping")),
                        darkgreen(match_atom),
                        red(from_file),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                continue

            # we need to ask SpmPlugin to re-extract metadata from pkg file
            # and grab the new "download" metadatum value using our
            # license check callback. It has to be done here because
            # we need the new path.

            def _package_injector_check_license(pkg_data):
                licenses = pkg_data['license'].split()
                return self._is_pkg_free(to_repository_id, licenses)

            def _package_injector_check_restricted(pkg_data):
                pkgatom = entropy.dep.create_package_atom_string(
                    pkg_data['category'], pkg_data['name'], pkg_data['version'],
                    pkg_data['versiontag'])
                return self._is_pkg_restricted(to_repository_id, pkgatom,
                    pkg_data['slot'])

            # check if pkg is restricted
            # and check if pkg is free, we must do this step in any case
            # NOTE: it sucks!
            tmp_data = self.Spm().extract_package_metadata(from_file,
                license_callback = _package_injector_check_license,
                restricted_callback = _package_injector_check_restricted)
            # NOTE: since ~0.tbz2 << revision is lost, we need to trick
            # the logic.
            updated_package_rel_path = os.path.join(
                os.path.dirname(tmp_data['download']),
                os.path.basename(package_rel_path))
            del tmp_data

            to_file = self.complete_local_upload_package_path(
                updated_package_rel_path, to_repository_id)

            if new_tag != None:

                match_category = dbconn.retrieveCategory(idpackage)
                match_name = dbconn.retrieveName(idpackage)
                match_version = dbconn.retrieveVersion(idpackage)
                tagged_package_filename = \
                    entropy.dep.create_package_filename(
                        match_category, match_name, match_version, new_tag)

                to_file = self.complete_local_upload_package_path(
                    updated_package_rel_path, to_repository_id)
                # directly move to correct place, tag changed, so file name
                to_file = os.path.join(os.path.dirname(to_file),
                    tagged_package_filename)

            self._ensure_dir_path(os.path.dirname(to_file))

            copy_data = [
                (from_file, to_file,),
                (from_file + etpConst['packagesexpirationfileext'],
                    to_file + etpConst['packagesexpirationfileext'],)
            ]
            extra_downloads = dbconn.retrieveExtraDownload(idpackage)
            for extra_download in extra_downloads:
                # just need the first entry, "download"
                extra_rel = extra_download['download']

                from_extra = self.complete_local_package_path(extra_rel,
                    repo)
                if not os.path.isfile(from_extra):
                    from_extra = self.complete_local_upload_package_path(
                        extra_rel, repo)

                to_extra = self.complete_local_upload_package_path(
                    extra_rel, to_repository_id)

                copy_data.append((from_extra, to_extra))

            for from_item, to_item in copy_data:
                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repository_id),
                        brown(branch),
                        blue(_("moving file")),
                        darkgreen(os.path.basename(from_item)),
                    ),
                    importance = 0,
                    level = "info",
                    header = red(" @@ "),
                    back = True
                )
                if os.path.isfile(from_item):
                    shutil.copy2(from_item, to_item)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("loading data from source repository")),
                    darkgreen(repo),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            # install package into destination db
            data = dbconn.getPackageData(idpackage)
            if new_tag != None:
                data['versiontag'] = new_tag

            # need to set back data['download'], because pkg path might got
            # changed, due to license re-validation
            data['download'] = updated_package_rel_path

            # GPG
            # before inserting new pkg, drop GPG signature and re-sign
            old_gpg = copy.copy(data['signatures']['gpg'])
            data['signatures']['gpg'] = None
            for extra_download in data['extra_download']:
                extra_download['gpg'] = None
            try:
                repo_sec = RepositorySecurity()
            except RepositorySecurity.GPGError as err:
                if old_gpg:
                    self.output(
                        "[%s] %s %s: %s." % (
                            darkgreen(to_repository_id),
                            darkred(_("GPG key was available in")),
                            bold(from_repository_id),
                            err,
                        ),
                        importance = 1,
                        level = "warning",
                        header = bold(" !!! ")
                    )
                repo_sec = None

            if repo_sec is not None:
                data['signatures']['gpg'] = self._get_gpg_signature(repo_sec,
                    to_repository_id, to_file)

                for extra_download in data['extra_download']:
                    to_extra = self.complete_local_upload_package_path(
                        extra_download['download'], to_repository_id)
                    extra_download['gpg'] = self._get_gpg_signature(repo_sec,
                        to_repository_id, to_extra)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("injecting data to destination repository")),
                    darkgreen(to_repository_id),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            data['original_repository'] = to_repository_id
            new_idpackage = todbconn.handlePackage(data)
            del data
            todbconn.commit()
            package_ids_added.add(new_idpackage)

            if not do_copy:
                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repository_id),
                        brown(branch),
                        blue(_("removing entry from source repository")),
                        darkgreen(repo),
                    ),
                    importance = 0,
                    level = "info",
                    header = red(" @@ "),
                    back = True
                )

                # remove package from old db
                dbconn.removePackage(idpackage)
                dbconn.clean()
                dbconn.commit()

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repository_id),
                    brown(branch),
                    blue(_("successfully handled atom")),
                    darkgreen(match_atom),
                ),
                importance = 0,
                level = "info",
                header = blue(" @@ ")
            )
            switched.add(idpackage)

        todbconn = self.open_server_repository(to_repository_id,
            read_only = False, no_upload = True)
        todbconn.clean()

        if package_ids_added:
            self._add_packages_qa_tests(
                [(x, to_repository_id) for x in package_ids_added], ask = ask)
            # just run this to make dev aware
            self.extended_dependencies_test([to_repository_id])

        return switched

    def _inject_database_into_packages(self, repository_id, injection_data):

        # now inject metadata into tbz2 packages
        self.output(
            "[%s] %s:" % (
                darkgreen(repository_id),
                blue(_("Injecting entropy metadata into built packages")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("JFYI, GPG infrastructure failed to load")),
                    err,
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            repo_sec = None # gnupg not found, perhaps report it

        treeupdates_actions = dbconn.listAllTreeUpdatesActions()

        # generate empty repository file and re-use it every time
        # this improves the execution a lot
        orig_fd = None
        tmp_repo_orig_path = None
        try:
            orig_fd, tmp_repo_orig_path = tempfile.mkstemp(
                prefix="entropy.server._inject")

            empty_repo = GenericRepository(
                readOnly = False,
                dbFile = tmp_repo_orig_path,
                name = None,
                xcache = False,
                indexing = False,
                skipChecks = True)
            empty_repo.initializeRepository()
            empty_repo.bumpTreeUpdatesActions(treeupdates_actions)
            empty_repo.commit()
        except Exception:
            os.remove(tmp_repo_orig_path)
            raise
        finally:
            if empty_repo is not None:
                empty_repo.close()
            if orig_fd is not None:
                os.close(orig_fd)

        try:
            for idpackage, package_path in injection_data:

                tmp_repo_file = None
                tmp_fd = None
                try:
                    tmp_fd, tmp_repo_file = tempfile.mkstemp(
                        prefix="entropy.server._inject_for")
                    with os.fdopen(tmp_fd, "wb") as tmp_f:
                        with open(tmp_repo_orig_path, "rb") as empty_f:
                            shutil.copyfileobj(empty_f, tmp_f)

                    self.output(
                        "[%s|%s] %s: %s" % (
                            darkgreen(repository_id),
                            brown(str(idpackage)),
                            blue(_("injecting entropy metadata")),
                            darkgreen(os.path.basename(package_path)),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        back = True
                    )
                    data = dbconn.getPackageData(idpackage)
                    self._inject_entropy_database_into_package(
                        package_path, data,
                        treeupdates_actions = treeupdates_actions,
                        initialized_repository_path = tmp_repo_file)
                finally:
                    if tmp_fd is not None:
                        try:
                            os.close(tmp_fd)
                        except OSError as err:
                            if err.errno != errno.EBADF:
                                raise
                    if tmp_repo_file is not None:
                        os.remove(tmp_repo_file)

                # GPG-sign package if GPG signature is set
                gpg_sign = None
                if repo_sec is not None:
                    gpg_sign = self._get_gpg_signature(repo_sec, repository_id,
                        package_path)

                digest = entropy.tools.md5sum(package_path)
                # update digest
                dbconn.setDigest(idpackage, digest)
                # update signatures
                signatures = data['signatures'].copy()
                for hash_key in sorted(signatures):
                    if hash_key == "gpg": # gpg already created
                        continue
                    hash_func = getattr(entropy.tools, hash_key)
                    signatures[hash_key] = hash_func(package_path)
                dbconn.setSignatures(idpackage, signatures['sha1'],
                    signatures['sha256'], signatures['sha512'],
                    gpg_sign)
                dbconn.commit()
                const_setup_file(package_path, etpConst['entropygid'], 0o664)
                self.output(
                    "[%s|%s] %s: %s" % (
                        darkgreen(repository_id),
                        brown(str(idpackage)),
                        blue(_("injection complete")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = red(" @@ ")
                )
        finally:
            os.remove(tmp_repo_orig_path)

    def remove_packages(self, repository_id, package_ids):
        """
        Remove packages from given repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param package_ids: list of package identifiers contained in
            from_repository_id repository
        @type package_ids: list
        """
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        for idpackage in package_ids:
            atom = dbconn.retrieveAtom(idpackage)
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("removing package")),
                    darkgreen(atom),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            dbconn.removePackage(idpackage)
        dbconn.clean()
        dbconn.commit()
        self.close_repository(dbconn)
        self.output(
            "[%s] %s" % (
                darkgreen(repository_id),
                blue(_("removal complete")),
            ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )

    def _verify_remote_packages(self, repository_id, packages, ask = True):

        self.output(
            "[%s] %s:" % (
                red("remote"),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )

        idpackages, world = self._match_packages(repository_id, packages)
        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        branch = self._settings['repositories']['branch']

        if world:
            self.output(
                blue(
                    _("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )
        else:
            mytxt = red("%s:") % (
                _("This is the list of the packages that would be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "    "
            )
            for idpackage in idpackages:
                pkgatom = dbconn.retrieveAtom(idpackage)
                down_url = dbconn.retrieveDownloadURL(idpackage)
                pkgfile = os.path.basename(down_url)
                self.output(
                    red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
                    importance = 1,
                    level = "info",
                    header = darkgreen("   - ")
                )

        if ask:
            rc_question = self.ask_question(
                _("Would you like to continue ?"))
            if rc_question == _("No"):
                return set(), set(), {}

        match = set()
        not_match = set()
        broken_packages = {}

        for uri in self.remote_packages_mirrors(repository_id):

            crippled_uri = EntropyTransceiver.get_uri_name(uri)
            self.output(
                "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("Working on mirror")),
                    brown(crippled_uri),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )


            totalcounter = len(idpackages)
            currentcounter = 0

            txc = self.Transceiver(uri)
            txc.set_verbosity(False)
            with txc as handler:

                for idpackage in idpackages:

                    currentcounter += 1
                    pkgfile = dbconn.retrieveDownloadURL(idpackage)
                    pkgfile = self.complete_remote_package_relative_path(
                        pkgfile, repository_id)
                    pkghash = dbconn.retrieveDigest(idpackage)

                    self.output(
                        "[%s] %s: %s" % (
                            brown(crippled_uri),
                            blue(_("checking hash")),
                            darkgreen(pkgfile),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        back = True,
                        count = (currentcounter, totalcounter,)
                    )

                    ck_remote = handler.get_md5(pkgfile)
                    if ck_remote is None:
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("digest verification of")),
                                bold(pkgfile),
                                blue(_("not supported")),
                            ),
                            importance = 1,
                            level = "info",
                            header = blue(" @@ "),
                            count = (currentcounter, totalcounter,)
                        )
                        continue

                    if ck_remote == pkghash:
                        match.add(idpackage)
                    else:
                        not_match.add(idpackage)
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("package")),
                                bold(pkgfile),
                                red(_("NOT healthy")),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred(" !!! "),
                            count = (currentcounter, totalcounter,)
                        )
                        if crippled_uri not in broken_packages:
                            broken_packages[crippled_uri] = []
                        broken_packages[crippled_uri].append(pkgfile)

            if broken_packages:
                mytxt = blue("%s:") % (
                    _("This is the list of broken packages"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = red(" * ")
                )
                for mirror in list(broken_packages.keys()):
                    mytxt = "%s: %s" % (
                        brown(_("Mirror")),
                        bold(mirror),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "info",
                        header = red("   <> ")
                    )
                    for broken_package in broken_packages[mirror]:
                        self.output(
                            blue(broken_package),
                            importance = 1,
                            level = "info",
                            header = red("      - ")
                        )

            self.output(
                "%s:" % (
                    blue(_("Statistics")),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    brown(_("Number of checked packages")),
                    brown(str(len(match) + len(not_match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkgreen(_("Number of healthy packages")),
                    darkgreen(str(len(match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkred(_("Number of broken packages")),
                    darkred(str(len(not_match))),
                ),
                importance = 1,
                level = "info",
                header = brown("   # ")
            )

        return match, not_match, broken_packages

    def _verify_local_packages(self, repository_id, packages, ask = True):

        self.output(
            "[%s] %s:" % (
                red(_("local")),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        idpackages, world = self._match_packages(repository_id, packages)
        dbconn = self.open_server_repository(repository_id, read_only = True)
        if world:
            self.output(
                blue(_("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )

        fine = set()
        failed = set()

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(idpackages, repository_id,
                ask = ask)
        if not rc_status:
            return fine, failed, downloaded_fine, downloaded_errors

        my_qa = self.QA()

        totalcounter = str(len(available))
        currentcounter = 0
        for idpackage in available:
            currentcounter += 1
            pkg_path = dbconn.retrieveDownloadURL(idpackage)

            self.output(
                "%s: %s" % (
                    blue(_("checking status of")),
                    darkgreen(pkg_path),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            storedmd5 = dbconn.retrieveDigest(idpackage)
            pkgpath = self._get_package_path(repository_id, dbconn, idpackage)
            result = entropy.tools.compare_md5(pkgpath, storedmd5)
            qa_fine = my_qa.entropy_package_checks(pkgpath)
            if result and qa_fine:
                fine.add(idpackage)
            else:
                failed.add(idpackage)
                self.output(
                    "%s: %s -- %s: %s" % (
                            blue(_("package")),
                            darkgreen(pkg_path),
                            blue(_("is corrupted, stored checksum")),
                            brown(storedmd5),
                    ),
                    importance = 1,
                    level = "info",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )

        if failed:
            mytxt = blue("%s:") % (_("This is the list of broken packages"),)
            self.output(
                    mytxt,
                    importance = 1,
                    level = "warning",
                    header =  darkred("  # ")
            )
            for idpackage in failed:
                atom = dbconn.retrieveAtom(idpackage)
                down_p = dbconn.retrieveDownloadURL(idpackage)
                self.output(
                        blue("[atom:%s] %s" % (atom, down_p,)),
                        importance = 0,
                        level = "warning",
                        header =  brown("    # ")
                )

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    len(fine) + len(failed),
                    _("checked packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkgreen("%s => %s" % (
                    len(fine),
                    _("healthy packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    len(failed),
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return fine, failed, downloaded_fine, downloaded_errors

    def sign_local_packages(self, repository_id, ask = True):
        """
        Sign local packages in given repository using GPG key hopefully set
        for it.

        @raises OnlineMirrorError: if package path is not available after
            having tried to download it, this should never happen btw.
        """
        self.output(
            "[%s] %s: %s" % (
                red(_("local")),
                blue(_("GPG signing packages for repository")),
                repository_id,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False)
        idpackages = dbconn.listAllPackageIds()

        self.output(
            blue(_("All the missing packages in repository will be downloaded.")),
            importance = 1,
            level = "info",
            header = "    "
        )

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(idpackages, repository_id,
                ask = ask)
        if not rc_status:
            return False, 0, 0

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output("%s: %s" % (
                    darkgreen(_("GnuPG not available")),
                    err,
                ),
                level = "error"
            )
            return False, 0, 0

        kp_expired = False
        try:
            kp_avail = repo_sec.is_keypair_available(repository_id)
        except RepositorySecurity.KeyExpired:
            kp_avail = False
            kp_expired = True

        if kp_expired:
            for x in (1, 2, 3):
                # SPAM!
                self.output("%s: %s" % (
                        darkred(_("Keys for repository are expired")),
                        bold(repository_id),
                    ),
                    level = "warning",
                    header = bold(" !!! ")
                )
        elif not kp_avail:
            self.output("%s: %s" % (
                    darkgreen(_("Keys not available for")),
                    bold(repository_id),
                ),
                level = "error"
            )
            return False, 0, 0

        fine = 0
        failed = 0
        totalcounter = len(available)
        currentcounter = 0

        # clear all GPG signatures?

        # we can eventually sign!
        for idpackage in available:

            currentcounter += 1

            pkg_path = self._get_package_path(repository_id, dbconn, idpackage)
            if not os.path.isfile(pkg_path):
                pkg_path = self._get_upload_package_path(repository_id, dbconn,
                    idpackage)
            if not os.path.isfile(pkg_path):
                # wtf!?
                pkg_atom = dbconn.retrieveAtom(idpackage)
                raise OnlineMirrorError("WTF!?!?! => %s, %s" % (
                    pkg_path, pkg_atom,))

            self.output(
                "%s: %s" % (
                    blue(_("signing package")),
                    darkgreen(os.path.basename(pkg_path)),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            gpg_sign = self._get_gpg_signature(repo_sec, repository_id,
                pkg_path)
            if gpg_sign is None:
                self.output(
                    "%s: %s" % (
                        darkred(_("Unknown error signing package")),
                        darkgreen(os.path.basename(pkg_path)),
                    ),
                    importance = 1,
                    level = "error",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )
                failed += 1
                continue

            # now inject gpg signature into repo
            sha1, sha256, sha512, old_gpg = dbconn.retrieveSignatures(idpackage)
            dbconn.setSignatures(idpackage, sha1, sha256, sha512,
                gpg = gpg_sign)

            fine += 1

        dbconn.commit()

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    fine,
                    _("signed packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    failed,
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return True, fine, failed

    def _download_locally_missing_files(self, package_ids, repository_id,
        ask = True):

        dbconn = self.open_server_repository(repository_id, read_only = True)
        to_download = set()
        available = set()
        for idpackage in package_ids:

            bindir_path = self._get_package_path(repository_id, dbconn,
                idpackage)
            uploaddir_path = self._get_upload_package_path(repository_id,
                dbconn, idpackage)
            pkg_path = dbconn.retrieveDownloadURL(idpackage)

            pkgatom = dbconn.retrieveAtom(idpackage)
            if os.path.isfile(bindir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkgreen(_("available")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                available.add(idpackage)
            elif os.path.isfile(uploaddir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkred(_("upload/ignored")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
            else:
                self.output(
                    "[%s] %s :: %s" % (
                        brown(_("download")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                to_download.add((idpackage, pkg_path,))

        if to_download and ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                # = downloaded fine, downloaded error
                return False, available, set(), set()

        if not to_download:
            # = downloaded fine, downloaded error
            return True, available, set(), set()

        downloaded_fine = set()
        downloaded_errors = set()

        not_downloaded = set()
        mytxt = blue("%s ...") % (_("Starting to download missing files"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "   "
        )
        for uri in self.remote_packages_mirrors(repository_id):

            if not_downloaded:
                mytxt = blue("%s ...") % (
                    _("Searching missing/broken files on another mirror"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                to_download = not_downloaded.copy()
                not_downloaded = set()

            for idpackage, pkg_path in to_download:
                rc_down = self.Mirrors.download_package(repository_id, uri,
                    pkg_path)
                if rc_down:
                    downloaded_fine.add(idpackage)
                    available.add(idpackage)
                else:
                    not_downloaded.add(pkg_path)

            if not not_downloaded:
                self.output(
                    red(_("Binary packages downloaded successfully.")),
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                break

        if not_downloaded:
            mytxt = blue("%s:") % (
                _("These are the packages that cannot be found online"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "   "
            )
            for pkg_path in not_downloaded:
                downloaded_errors.add(pkg_path)
                self.output(
                        brown(pkg_path),
                        importance = 1,
                        level = "warning",
                        header = red("    * ")
                )
            downloaded_errors |= not_downloaded
            mytxt = "%s." % (_("They won't be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "   "
            )

        return True, available, downloaded_fine, downloaded_errors

    def _switch_packages_branch(self, repository_id, from_branch, to_branch):

        if to_branch != self._settings['repositories']['branch']:
            mytxt = "%s: %s %s" % (
                blue(_("Please setup your branch to")),
                bold(to_branch),
                blue(_("and retry")),
            )
            self.output(
                mytxt,
                importance = 1,
                level = "error",
                header = darkred(" !! ")
            )
            return None

        mytxt = red("%s ...") % (_("Copying repository (if not exists)"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )
        branch_dbdir = self._get_local_repository_dir(repository_id)
        old_branch_dbdir = self._get_local_repository_dir(repository_id,
            branch = from_branch)

        # close all our databases
        self.close_repositories()

        # if database file did not exist got created as an empty file
        # we can just rm -rf it
        branch_dbfile = self._get_local_repository_file(repository_id)
        if os.path.isfile(branch_dbfile):
            if entropy.tools.get_file_size(branch_dbfile) == 0:
                shutil.rmtree(branch_dbdir, True)

        if os.path.isdir(branch_dbdir):

            while True:
                rnd_num = entropy.tools.get_random_number()
                backup_dbdir = branch_dbdir + str(rnd_num)
                if not os.path.isdir(backup_dbdir):
                    break
            os.rename(branch_dbdir, backup_dbdir)

        if os.path.isdir(old_branch_dbdir):
            shutil.copytree(old_branch_dbdir, branch_dbdir)

        mytxt = red("%s ...") % (_("Switching packages"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True, lock_remote = False)
        try:
            dbconn.validate()
        except SystemDatabaseError:
            self._handle_uninitialized_repository(repository_id)
            dbconn = self.open_server_repository(repository_id,
                read_only = False, no_upload = True, lock_remote = False)

        idpackages = dbconn.listAllPackageIds()
        already_switched = set()
        not_found = set()
        switched = set()
        ignored = set()
        no_checksum = set()

        maxcount = len(idpackages)
        count = 0
        for idpackage in idpackages:
            count += 1

            cur_branch = dbconn.retrieveBranch(idpackage)
            atom = dbconn.retrieveAtom(idpackage)
            if cur_branch == to_branch:
                already_switched.add(idpackage)
                self.output(
                    red("%s %s, %s %s" % (
                            _("Ignoring"),
                            bold(atom),
                            _("already in branch"),
                            cur_branch,
                        )
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen(" @@ "),
                    count = (count, maxcount,)
                )
                ignored.add(idpackage)
                continue

            self.output(
                "[%s=>%s] %s" % (
                    brown(cur_branch),
                    bold(to_branch),
                    darkgreen(atom),
                ),
                importance = 0,
                level = "info",
                header = darkgreen(" @@ "),
                back = True,
                count = (count, maxcount,)
            )
            dbconn.switchBranch(idpackage, to_branch)
            dbconn.commit()
            switched.add(idpackage)

        dbconn.commit()

        # now migrate counters
        dbconn.moveSpmUidsToBranch(to_branch)

        self.close_repository(dbconn)
        mytxt = blue("%s.") % (_("migration loop completed"),)
        self.output(
            "[%s=>%s] %s" % (
                    brown(from_branch),
                    bold(to_branch),
                    mytxt,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        return switched, already_switched, ignored, not_found, no_checksum


    def orphaned_spm_packages_test(self):

        mytxt = "%s %s" % (
            blue(_("Running orphaned SPM packages test")), red("..."),)
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        installed_packages = self.Spm().get_installed_packages()
        length = len(installed_packages)
        installed_packages.sort()
        not_found = {}
        count = 0
        for installed_package in installed_packages:
            count += 1
            self.output(
                "%s: %s" % (
                    darkgreen(_("Scanning package")),
                    brown(installed_package),),
                importance = 0,
                level = "info",
                back = True,
                count = (count, length),
                header = darkred(" @@ ")
            )
            key = entropy.dep.dep_getkey(installed_package)
            try:
                slot = self.Spm().get_installed_package_metadata(
                    installed_package, "SLOT")
            except KeyError:
                # ignore, race condition, pkg got removed in the meantime
                continue
            pkg_atom = "%s%s%s" % (key, ":", slot,)
            try:
                tree_atoms = self.Spm().match_package(pkg_atom,
                    match_type = "match-all")
            except (KeyError,):
                tree_atoms = None # ouch!
            if not tree_atoms:
                not_found[installed_package] = pkg_atom
                self.output(
                    "%s: %s" % (
                        blue(pkg_atom),
                        darkred(_("not found anymore")),
                    ),
                    importance = 0,
                    level = "warning",
                    count = (count, length),
                    header = darkred(" @@ ")
                )

        if not_found:
            not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
            self.output(
                "%s: %s" % (
                        blue(_("Packages string")),
                        not_found_list,
                    ),
                importance = 0,
                level = "warning",
                count = (count, length),
                header = darkred(" @@ ")
            )

        return not_found

    def _deps_tester(self, default_repository_id, match_repo = None):

        sys_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(sys_set['repositories'].keys())
        if match_repo is None:
            match_repo = server_repos
        installed_packages = set()
        # if a default repository is passed, we will just test against it
        if default_repository_id:
            server_repos = [default_repository_id]

        for repo in server_repos:
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True, do_treeupdates = False)
            installed_packages |= set([(x, repo) for x in \
                dbconn.listAllPackageIds()])

        deps_not_satisfied = set()
        length = str((len(installed_packages)))
        count = 0
        mytxt = _("Checking")
        _deps_cache = set()

        for idpackage, repo in installed_packages:
            count += 1
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True, do_treeupdates = False)

            if (count%150 == 0) or (count == length) or (count == 1):
                atom = dbconn.retrieveAtom(idpackage)
                self.output(
                    darkgreen(mytxt)+" "+bold(atom),
                    importance = 0,
                    level = "info",
                    back = True,
                    count = (count, length),
                    header = darkred(" @@  ")
                )

            # NOTE: this must also test build dependencies to make sure
            # that every packages comes out with all of them.
            xdeps = dbconn.retrieveDependencies(idpackage)
            xdeps = [x for x in xdeps if x not in _deps_cache]
            _deps_cache.update(xdeps)
            for xdep in xdeps:
                xid, xuseless = self.atom_match(xdep,
                    match_repo = match_repo)
                if xid == -1:
                    deps_not_satisfied.add(xdep)

        return deps_not_satisfied

    def extended_dependencies_test(self, repository_ids):
        """
        Test repository against missing dependencies.
        Moreover, the base repository (the first listed in server.conf)
        is tested by itself, since it must always be self-contained.

        @param repository_ids: list of repository identifiers to test
        @type repository_ids: list
        @return: list (set) of unsatisfied dependencies
        @rtype: set
        """
        # just run this to make dev aware
        unsatisfied_deps = set()
        for repository_id in repository_ids:
            unsatisfied_deps |= self.dependencies_test(repository_id)

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        base_repository_id = srv_set['base_repository_id']
        if base_repository_id is not None:
            # dependency test base repository, which must always be
            # self-contained
            unsatisfied_deps |= self.dependencies_test(base_repository_id,
                match_repo = [base_repository_id])
        return unsatisfied_deps

    def dependencies_test(self, repository_id, match_repo = None):
        """
        Test repository against missing dependencies.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword match_repo: list of repositories to look for missing deps
        @type match_repo: list
        @return: list (set) of unsatisfied dependencies
        @rtype: set
        """
        mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(srv_set['repositories'].keys())
        if match_repo is not None:
            server_repo = match_repo
        deps_not_matched = self._deps_tester(repository_id,
            match_repo = match_repo)

        if deps_not_matched:

            crying_atoms = {}
            for atom in deps_not_matched:
                for repo in server_repos:
                    dbconn = self.open_server_repository(repo,
                        just_reading = True, do_treeupdates = False)
                    riddep = dbconn.searchDependency(atom)
                    if riddep == -1:
                        continue
                    ridpackages = dbconn.searchPackageIdFromDependencyId(riddep)
                    for i in ridpackages:
                        iatom = dbconn.retrieveAtom(i)
                        if atom not in crying_atoms:
                            crying_atoms[atom] = set()
                        crying_atoms[atom].add((iatom, repo))

            mytxt = blue("%s:") % (_("These are the dependencies not found"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            mytxt = "%s:" % (_("Needed by"),)
            for atom in deps_not_matched:
                self.output(
                    red(atom),
                    importance = 1,
                    level = "info",
                    header = blue("   # ")
                )
                if atom in crying_atoms:
                    self.output(
                        red(mytxt),
                        importance = 0,
                        level = "info",
                        header = blue("      # ")
                    )
                    for my_dep, myrepo in crying_atoms[atom]:
                        self.output(
                            "[%s:%s] %s" % (
                                blue(_("by repo")),
                                darkred(myrepo),
                                darkgreen(my_dep),
                            ),
                            importance = 0,
                            level = "info",
                            header = blue("      # ")
                        )
        else:

            mytxt = blue(_("Every dependency is satisfied. It's all fine."))
            self.output(
                mytxt,
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )

        return deps_not_matched

    def test_shared_objects(self, repository_id, dump_results_to_file = False):
        """
        Test packages in repository, against missing shared objects.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword dump_results_to_file: dump results to file
        @type dump_results_to_file: bool
        @return: execution status (0 = fine)
        @rtype: int
        """
        pkg_list_path = None
        if dump_results_to_file:
            tmp_dir = tempfile.mkdtemp(prefix = "entropy.server")
            pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt")
            dmp_data = [
                (_("Broken and matched packages list"), pkg_list_path,),
            ]
            mytxt = "%s:" % (purple(_("Dumping results into these files")),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = blue(" @@ ")
            )
            for txt, path in dmp_data:
                mytxt = "%s: %s" % (blue(txt), path,)
                self.output(
                    mytxt,
                    importance = 0,
                    level = "info",
                    header = darkgreen("   ## ")
                )

        # load db
        dbconn = self.open_server_repository(repository_id, read_only = True,
            no_upload = True)
        QA = self.QA()
        packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn,
            broken_symbols = True, dump_results_to_file = dump_results_to_file)
        if status != 0:
            return 1

        if (not brokenexecs) and (not packages_matched):
            mytxt = "%s." % (_("System is healthy"),)
            self.output(
                blue(mytxt),
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )
            return 0

        mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
        self.output(
            blue(mytxt),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        real_brokenexecs = [os.path.realpath(x) for x in brokenexecs if \
            x != os.path.realpath(x)]
        brokenexecs.update(real_brokenexecs)
        packages = self.Spm().search_paths_owners(brokenexecs)

        if packages:
            mytxt = "%s:" % (_("These are the matched packages"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            for my_atom, my_elf_id in packages:
                package_slot = my_atom, my_elf_id
                self.output(
                    "%s [elf:%s]" % (
                        purple(my_atom),
                        darkgreen(str(my_elf_id)),
                    ),
                    importance = 0,
                    level = "info",
                    header = red("     # ")
                )
                for filename in sorted(packages[package_slot]):
                    self.output(
                        darkgreen(filename),
                        importance = 0,
                        level = "info",
                        header = brown("       => ")
                    )

            pkgstring_list = sorted(["%s%s%s" % (
                entropy.dep.dep_getkey(x[0]), etpConst['entropyslotprefix'],
                    x[1],) for x in sorted(packages)])
            if pkg_list_path is not None:
                enc = etpConst['conf_encoding']
                with codecs.open(pkg_list_path, "w", encoding=enc) as pkg_f:
                    for pkgstr in pkgstring_list:
                        pkg_f.write(pkgstr + "\n")
                    pkg_f.flush()
            pkgstring = ' '.join(pkgstring_list)
            mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
        else:
            self.output(
                red(_("No matched packages")),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )

        return 0

    def close_repositories(self, mask_clear = False):

        srv_dbcache = self._server_dbcache
        if srv_dbcache is not None:
            for item in srv_dbcache.keys():
                try:
                    srv_dbcache[item].close()
                except ProgrammingError: # already closed?
                    pass
            srv_dbcache.clear()
        if mask_clear:
            self._settings.clear()

    def commit_repositories(self):
        """
        Execute commit on all the open (in rw) repositories.
        """
        srv_dbcache = self._server_dbcache
        if srv_dbcache is not None:
            for repo in srv_dbcache.values():
                # commit on readonly repos has no effect
                repo.commit()

    def close_repository(self, entropy_repository):
        """
        Close single EntropyRepositoryBase instance, given its class object.

        @param entropy_repository: EntropyRepositoryBase instance
        @type entropy_repository: entropy.db.skel.EntropyRepositoryBase
        """
        found = None
        for item in self._server_dbcache:
            if entropy_repository is self._server_dbcache[item]:
                found = item
                break
        if found is not None:
            instance = self._server_dbcache.pop(found)
            instance.close()

    def repository_metadata(self, repository_id):
        """
        Return Entropy Server repository configuration metadata.

        @return: repository configuration metadata
        @rtype: dict
        @raise KeyError: if repository is not configured or available
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'][repository_id]

    def available_repositories(self):
        """
        Return a dictionary representing available repositories metadata.

        @return: available repository metadata
        @rtype: dict
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'].copy()

    def switch_default_repository(self, repository_id, save = None,
        handle_uninitialized = True):
        """
        Change Entropy Server default (current) repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword save: if True, save the default repository in server
            configuration
        @type save: bool
        @keyword handle_uninitialized: if repository is uninitialized, handle
            the case automatically
        @type handle_uninitialized: bool
        """
        # avoid setting __default__ as default server repo
        if repository_id == etpConst['clientserverrepoid']:
            return
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        if save is None:
            save = self._save_repository
        if repository_id not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        repository_id,
                        _("repository not configured"),
                    )
            )
        self.close_repositories()
        srv_set['default_repository_id'] = repository_id
        self._repository = repository_id
        self._setup_services()
        if save:
            self._save_default_repository(repository_id)

        self._setup_community_repositories_settings()
        if handle_uninitialized:
            self._handle_uninitialized_repository(repository_id)

    def _setup_community_repositories_settings(self):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if srv_set['community_mode']:
            for repoid in srv_set['repositories']:
                srv_set['repositories'][repoid]['community'] = True

    def _handle_uninitialized_repository(self, repoid):

        if self._is_repository_initialized(repoid):
            return

        mytxt = blue("%s.") % (
            _("Your default repository is not initialized"),)
        self.output(
            "[%s:%s] %s" % (
                brown("repo"),
                purple(repoid),
                mytxt,
            ),
            importance = 1,
            level = "warning",
            header = darkred(" !!! ")
        )
        answer = self.ask_question(
            _("Do you want to initialize your default repository ?"))
        if answer == _("No"):
            mytxt = red("%s.") % (
                _("Continuing with an uninitialized repository"),)
            self.output(
                "[%s:%s] %s" % (
                    brown("repo"),
                    purple(repoid),
                    mytxt,
                ),
                importance = 1,
                level = "warning",
                header = darkred(" !!! ")
            )
        else:
            # move empty database for security sake
            dbfile = self._get_local_repository_file(repoid)
            if os.path.isfile(dbfile):
                shutil.move(dbfile, dbfile+".backup")
            self.initialize_repository(repoid)

    def _save_default_repository(self, repoid):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return

        enc = etpConst['conf_encoding']
        server_conf = ServerSystemSettingsPlugin.server_conf_path()
        try:
            with codecs.open(server_conf, "r", encoding=enc) as f_srv:
                content = f_srv.readlines()
        except IOError as err:
            if err.errno == errno.ENOENT:
                content = None
            else:
                raise

        if content is not None:
            content = [x.strip() for x in content]
            found = False
            new_content = []
            for line in content:
                key, value = entropy.tools.extract_setting(line)
                if key == "default-repository":
                    line = "default-repository = %s" % (repoid,)
                    found = True
                new_content.append(line)
            if not found:
                new_content.append("default-repository = %s" % (repoid,))

            tmp_fd, tmp_path = tempfile.mkstemp()
            with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as f_srv_t:
                for line in new_content:
                    f_srv_t.write(line+"\n")
                f_srv_t.flush()

            entropy.tools.rename_keep_permissions(
                tmp_path, server_conf)

        else:
            with codecs.open(server_conf, "w", encoding=enc) as f_srv:
                f_srv.write("default-repository = %s\n" % (repoid,))
                f_srv.flush()

    def enable_repository(self, repository_id):
        """
        Enable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param enable: True for enable, False for disable
        @type enable: bool
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """
        return self._toggle_repository(repository_id, True)

    def disable_repository(self, repository_id):
        """
        Disable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """
        return self._toggle_repository(repository_id, False)

    def _toggle_repository(self, repository_id, enable):
        """
        Enable or disable a repository.

        @param repository_id: repository identifier
        @type repository_id: string
        @param enable: True for enable, False for disable
        @type enable: bool
        @return: True, if switch went fine, False otherwise
        @rtype: bool
        """

        # avoid setting __default__ as default server repo
        if repository_id == etpConst['clientserverrepoid']:
            return False

        server_conf = ServerSystemSettingsPlugin.server_conf_path()
        enc = etpConst['conf_encoding']
        try:
            with codecs.open(server_conf, "r", encoding=enc) as f_srv:
                content = [x.strip() for x in f_srv.readlines()]
        except IOError as err:
            if err.errno != errno.ENOENT:
                raise
            return None

        tmp_fd, tmp_path = tempfile.mkstemp()
        with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as f_tmp:
            status = False
            for line in content:
                key, value = entropy.tools.extract_setting(line)
                if key is not None:
                    key = key.replace(" ", "")
                    key = key.replace("\t", "")
                    if key in ("repository", "#repository"):
                        if enable and (key == "#repository"):
                            line = "repository = %s" % (value,)
                            status = True
                        elif not enable and (key == "repository"):
                            line = "# repository = %s" % (value,)
                            status = True
                f_tmp.write(line+"\n")
            f_tmp.flush()

        if status:
            entropy.tools.rename_keep_permissions(
                tmp_path, server_conf)
            self.close_repositories()
            self._settings.clear()
            self._setup_services()
        return status

    def _is_repository_initialized(self, repo):

        def do_validate(dbc):
            try:
                dbc.validate()
                return True
            except SystemDatabaseError:
                return False

        try:
            dbc = self.open_server_repository(repo, just_reading = True)
        except RepositoryError:
            return False
        valid = do_validate(dbc)
        if not valid:
            dbc = self.open_server_repository(repo, read_only = False,
                no_upload = True, is_new = True)
            valid = do_validate(dbc)

        return valid

    def _server_repository_sync_lock(self, repo, no_upload):

        if repo is None:
            repo = self._repository

        # check if the database is locked locally
        lock_file = self._get_repository_lockfile(repo)
        if os.path.isfile(lock_file):
            self.output(
                red(_("Entropy repository is already locked by you :-)")),
                importance = 1,
                level = "info",
                header = red(" * ")
            )
        else:
            # check if the database is locked REMOTELY
            mytxt = "%s ..." % (_("Locking and Syncing Entropy repository"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" * "),
                back = True
            )
            for uri in self.remote_repository_mirrors(repo):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                given_up = self.Mirrors.mirror_locked(repo, uri)
                if given_up:
                    mytxt = "%s:" % (_("Mirrors status table"),)
                    self.output(
                        darkgreen(mytxt),
                        importance = 1,
                        level = "info",
                        header = brown(" * ")
                    )
                    dbstatus = self.Mirrors.mirrors_status(repo) 
                    for db_uri, db_st1, db_st2 in dbstatus:
                        db_st1_info = darkgreen(_("Unlocked"))
                        if db_st1:
                            db_st1_info = red(_("Locked"))
                        db_st2_info = darkgreen(_("Unlocked"))
                        if db_st2:
                            db_st2_info = red(_("Locked"))

                        crippled_uri = EntropyTransceiver.get_uri_name(db_uri)
                        self.output(
                            "%s: [%s: %s] [%s: %s]" % (
                                bold(crippled_uri),
                                brown(_("repository")),
                                db_st1_info,
                                brown(_("download")),
                                db_st2_info,
                            ),
                            importance = 1,
                            level = "info",
                            header = "\t"
                        )

                    raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
                            _("cannot lock mirror"),
                            crippled_uri,
                        )
                    )

            # if we arrive here, it is because all the mirrors are unlocked
            self.Mirrors.lock_mirrors(repo, True)
            self.Mirrors.sync_repository(repo, enable_upload = not no_upload)


    def _init_generic_memory_server_repository(self, repository_id, description,
        pkg_mirrors = None, repo_mirrors = None, community_repo = False,
        service_url = None, set_as_default = False):

        if pkg_mirrors is None:
            pkg_mirrors = []
        if repo_mirrors is None:
            repo_mirrors = []

        repo = self._open_temp_repository(repository_id, temp_file = ":memory:")
        self._memory_db_srv_instances[repository_id] = repo

        # add to settings
        repodata = {
            'repoid': repository_id,
            'description': description,
            'pkg_mirrors': pkg_mirrors,
            'repo_mirrors': repo_mirrors,
            'community': community_repo,
            'handler': '', # not supported
            '__temporary__': True,
        }
        ServerSystemSettingsPlugin.extend_repository_metadata(
            self._settings, repository_id, repodata)

        # this will make the change permanent until remove_repository()
        ServerSystemSettingsPlugin.REPOSITORIES[repository_id] = repodata
        # this will add our repo to the metadata stuff
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        srv_set['repositories'][repository_id] = repodata
        if set_as_default:
            # this will make the change permanent
            etpConst['defaultserverrepositoryid'] = repository_id
            # this will affect the current settings
            srv_set['default_repository_id'] = repository_id
        # not calling SystemSettings.clear()
        # because it's not strictly needed, but match metadata
        # could be available if this method gets called after
        # init_singleton(). In that case, it's up the user to call
        # clear()

        etp_repo_meta = {
            'lock_remote': False,
            'no_upload': True,
            'output_interface': self,
            'read_only': False,
            'local_dbfile': None,
            '__temporary__': True,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        repo.add_plugin(srv_plug)
        return repo

    def _open_temp_repository(self, repo, temp_file = None, initialize = True):
        """
        Open temporary ServerPackagesRepository interface.

        @keyword dbname: database name
        @type dbname: string
        @keyword output_interface: entropy.output.TextInterface based instance
        @type output_interface: entropy.output.TextInterface based instance
        """
        if temp_file is None:
            tmp_fd, temp_file = tempfile.mkstemp(prefix = 'entropy.server')
            os.close(tmp_fd)

        conn = ServerPackagesRepository(
            readOnly = False,
            dbFile = temp_file,
            name = repo,
            xcache = False,
            indexing = False,
            skipChecks = True,
            temporary = True
        )
        if initialize:
            conn.initializeRepository()
        return conn

    @staticmethod
    def get_repository(repoid):
        """
        Reimplemented from entropy.client.interfaces.client.Client class
        """
        if repoid == etpConst['clientserverrepoid']:
            return InstalledPackagesRepository
        return ServerPackagesRepository

    def open_repository(self, repository_id):
        """
        This method aims to improve class usability by providing an easier
        method to open Entropy Server-side repositories like it happens
        with Entropy Client-side ones. If you just want open a read-only
        repository, feel free to use this wrapping method.

        @param repository_id: repository identifier
        @type repository_id: string
        @return: ServerPackagesRepository instance
        @rtype: entropy.server.interfaces.ServerPackagesRepository
        """
        return self.open_server_repository(repository_id,
            just_reading = True, do_treeupdates = False)

    def remove_repository(self, repository_id, disable = False):
        """
        Remove Repository from those available, server-side override.
        """
        try:
            repo_data = ServerSystemSettingsPlugin.REPOSITORIES
            del repo_data[repository_id]
        except KeyError:
            pass

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        try:
            del srv_set['repositories'][repository_id]
        except KeyError as err:
            pass

        return super(Server, self).remove_repository(
            repository_id, disable = disable)

    def open_server_repository(self, repository_id, read_only = True,
            no_upload = True, just_reading = False, indexing = True,
            warnings = True, do_cache = True, use_branch = None,
            lock_remote = True, is_new = False, do_treeupdates = True):
        """
        Open Server-side Entropy repository given its repository identifier.

        @param repository_id: repository identifier
        @type repository_id: string
        @keyword read_only: open repository in read-only
        @type read_only: bool
        @param no_upload: allow to force the upload of the repository if
            required
        @type no_upload: bool
        @keyword just_reading: open the repository just for the purpose of
            getting data from it, without any fancy action. This avoids
            triggering some forced behaviours. If just_reading is True,
            read_only is forced to True.
        @type just_reading: bool
        @keyword indexing: allow repository indexing
        @type indexing: bool
        @keyword warnings: show warnings to user if True
        @type warnings: bool
        @keyword do_cache: use cache to retrieve a fresh repository object
        @type do_cache: bool
        @keyword use_branch: override default package branch when opening a
            new repository
        @type use_branch: bool
        @keyword lock_remote: lock remote mirrors when opening
        @type lock_remote: bool
        @keyword is_new: set this to True if the repository has been just
            created
        @type is_new: bool
        @type do_treeupdates: allow the execution of package names and slot
            updates
        @type do_treeupdates: bool
        @raise RepositoryError: if repository doesn't exist
        @return: EntropyRepositoryBase instance
        @rtype: EntropyRepositoryBase
        """
        # in-memory server repos
        if repository_id in self._memory_db_srv_instances:
            return self._memory_db_srv_instances[repository_id]

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repository_id == etpConst['clientserverrepoid'] and \
                srv_set['community_mode']:
            return self.installed_repository()

        if just_reading:
            read_only = True
            no_upload = True

        try:
            local_dbfile = self._get_local_repository_file(
                repository_id, branch = use_branch)
        except KeyError:
            # repository not available
            raise RepositoryError(repository_id)
        if do_cache:
            cached = self._server_dbcache.get(
                (repository_id, etpConst['systemroot'], local_dbfile, read_only,
                    no_upload, just_reading, use_branch, lock_remote,)
            )
            if cached != None:
                return cached

        local_dbfile_dir = os.path.dirname(local_dbfile)
        try:
            self._ensure_dir_path(local_dbfile_dir)
        except OSError as err:
            if err.errno != errno.EACCES:
                raise
            raise RepositoryError(repository_id)

        if (not read_only) and (lock_remote) and \
            (repository_id not in self._sync_lock_cache):
            self._server_repository_sync_lock(repository_id, no_upload)
            self._sync_lock_cache.add(repository_id)

        conn = ServerPackagesRepository(
            readOnly = read_only,
            dbFile = local_dbfile,
            name = repository_id,
            xcache = False # always set to False, if you want to enable
            # you need to make sure that client-side and server-side caches
            # don't collide due to sharing ServerPackagesRepository.repository_id()
        )
        etp_repo_meta = {
            'lock_remote': lock_remote,
            'no_upload': no_upload,
            'output_interface': self,
            'read_only': read_only,
            'local_dbfile': local_dbfile,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        conn.add_plugin(srv_plug)

        valid = True
        try:
            conn.validate()
        except SystemDatabaseError:
            valid = False

        # verify if we need to update the database to sync
        # with portage updates, we just ignore being readonly in the case
        if (repository_id not in self._treeupdates_repos) and \
            (not just_reading):
            # sometimes, when filling a new server db
            # we need to avoid tree updates
            if valid:
                if do_treeupdates and not conn.readonly():
                    # readonly() always returns the effective
                    # write access to repository (despire what is
                    # really set during instantiation)
                    self._repository_packages_spm_sync(repository_id, conn,
                        branch = use_branch)
            elif warnings and not is_new:
                mytxt = _("Repository is corrupted!")
                self.output(
                    darkred(mytxt),
                    importance = 1,
                    level = "warning",
                    header = bold(" !!! ")
                )

        if not read_only and valid and indexing:

            self.output(
                "[%s|%s] %s" % (
                        blue(repository_id),
                        red(_("repository")),
                        blue(_("indexing repository")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            conn.createAllIndexes()

        if do_cache:
            # !!! also cache just_reading otherwise there will be
            # real issues if the connection is opened several times
            self._server_dbcache[
                (repository_id, etpConst['systemroot'], local_dbfile, read_only,
                no_upload, just_reading, use_branch, lock_remote,)] = conn

        # this ensures that side effect outcomes are committed
        # to db (treeupdates for example)
        conn.commit(force = True, no_plugins = True)
        return conn

    def _repository_packages_spm_sync(self, repository_id, repo_db,
        branch = None):
        """
        Service method used to sync package names with Source Package Manager.
        Source Package Manager can change package names, categories or slot
        and Entropy repositories must be kept in sync.
        """
        if branch is None:
            branch = self._settings['repositories']['branch']
        self._treeupdates_repos.add(repository_id)
        self.Spm().package_names_update(repo_db, repository_id, self, branch)

    def _setup_empty_repository(self, repository_path):

        dbdir = os.path.dirname(repository_path)
        if not os.path.isdir(dbdir):
            os.makedirs(dbdir)

        mytxt = red("%s ...") % (_("Initializing an empty repository"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * "),
            back = True
        )
        dbconn = self.open_generic_repository(repository_path)
        dbconn.initializeRepository()
        dbconn.commit()
        dbconn.close()
        mytxt = "%s %s %s." % (
            red(_("Entropy repository file")),
            bold(repository_path),
            red(_("successfully initialized")),
        )
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

    def _get_whitelisted_licenses(self, repository_id):
        wl_file = self._get_local_repository_licensewhitelist_file(
            repository_id)
        if not os.path.isfile(wl_file):
            return []
        return entropy.tools.generic_file_content_parser(
            wl_file, encoding = etpConst['conf_encoding'])

    def _get_restricted_packages(self, repository_id):
        rl_file = self._get_local_restricted_file(repository_id)
        if not os.path.isfile(rl_file):
            return []
        return entropy.tools.generic_file_content_parser(rl_file,
            comment_tag = "##", encoding = etpConst['conf_encoding'])

    def _is_pkg_restricted(self, repository_id, pkg_atom, pkg_slot):

        restricted_pkgs = self._get_restricted_packages(repository_id)
        if not restricted_pkgs:
            return False

        pkg_key = entropy.dep.dep_getkey(pkg_atom)
        for r_dep in restricted_pkgs:
            r_key, r_slot = entropy.dep.dep_getkey(r_dep), \
                entropy.dep.dep_getslot(r_dep)
            if r_slot is None:
                r_slot = pkg_slot

            if (r_key == pkg_key) and (r_slot == pkg_slot):
                return True

        return False

    def _is_pkg_free(self, repository_id, pkg_licenses):

        # check if nonfree directory support is enabled, if not,
        # always return True.
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if not srv_set['nonfree_packages_dir_support']:
            return True

        wl_licenses = self._get_whitelisted_licenses(repository_id)

        if not pkg_licenses:
            return True # free if no licenses provided
        if not wl_licenses:
            return True # no whitelist !

        for lic in pkg_licenses:
            if lic not in wl_licenses:
                return False
        return True

    def _package_injector(self, repository_id, package_files, inject = False):

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        def _package_injector_check_license(pkg_data):
            licenses = pkg_data['license'].split()
            return self._is_pkg_free(repository_id, licenses)

        def _package_injector_check_restricted(pkg_data):
            pkgatom = entropy.dep.create_package_atom_string(
                pkg_data['category'], pkg_data['name'], pkg_data['version'],
                pkg_data['versiontag'])
            return self._is_pkg_restricted(repository_id,
                pkgatom, pkg_data['slot'])

        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        package_file = package_files[0]
        self.output(
            "[%s] %s: %s" % (
                    darkgreen(repository_id),
                    _("adding package"),
                    bold(os.path.basename(package_file)),
                ),
            importance = 1,
            level = "info",
            header = brown(" * "),
            back = True
        )
        mydata = self.Spm().extract_package_metadata(package_file,
            license_callback = _package_injector_check_license,
            restricted_callback = _package_injector_check_restricted)
        is_licensed_ugly = not _package_injector_check_license(mydata)
        is_restricted = _package_injector_check_restricted(mydata)

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            # GPG not available
            repo_sec = None

        def _generate_extra_download(path, down_type):
            extra_url = entropy.tools.create_package_dirpath(mydata['branch'],
                nonfree = is_licensed_ugly, restricted = is_restricted) + \
                    "/" + os.path.basename(path)
            size = entropy.tools.get_file_size(path)
            disksize = entropy.tools.get_uncompressed_size(path)
            md5 = entropy.tools.md5sum(path)
            sha1 = entropy.tools.sha1(path)
            sha256 = entropy.tools.sha256(path)
            sha512 = entropy.tools.sha512(path)
            gpg = None
            if repo_sec is not None:
                gpg = self._get_gpg_signature(repo_sec, repository_id, path)
            edw = {
                'download': extra_url,
                'type': down_type,
                'size': size,
                'disksize': disksize,
                'md5': md5,
                'sha1': sha1,
                'sha256': sha256,
                'sha512': sha512,
                'gpg': gpg,
            }
            return edw

        # ~~~
        # Support for separate debug package files
        extra_files = package_files[1:]
        debuginfo_files = []
        for extra_filepath in extra_files:
            if extra_filepath.endswith(etpConst['packagesdebugext']):
                debuginfo_files.append(extra_filepath)
        for debuginfo_filepath in debuginfo_files:
            extra_files.remove(debuginfo_filepath)

        extra_download = []
        for debuginfo_filepath in debuginfo_files:
            extra_download.append(
                _generate_extra_download(debuginfo_filepath, "debug"))

        for extra_filepath in extra_files:
            extra_download.append(
                _generate_extra_download(extra_filepath, "data"))

        self._pump_extracted_package_metadata(mydata, repository_id,
            {
                'injected': inject,
                'original_repository': repository_id,
                'extra_download': extra_download,
            }
        )
        idpackage = dbconn.handlePackage(mydata)
        revision = dbconn.retrieveRevision(idpackage)
        # make sure that info have been written to disk
        dbconn.commit()

        myserver_repos = list(srv_set['repositories'].keys())

        ### since we are handling more repositories, we need to make sure
        ### that there are no packages in other repositories with same atom
        ### and greater revision
        rev_test_atom = mydata['atom']
        max_rev = -1
        for myrepo in myserver_repos:

            # not myself
            if myrepo == repository_id:
                continue

            mydbconn = self.open_server_repository(myrepo, read_only = True,
                no_upload = True)

            myrepo_idpackages = mydbconn.getPackageIds(rev_test_atom)
            for myrepo_idpackage in myrepo_idpackages:
                myrev = mydbconn.retrieveRevision(myrepo_idpackage)
                if myrev > max_rev:
                    max_rev = myrev

        if max_rev >= revision:
            max_rev += 1
            revision = max_rev
            mydata['revision'] = revision
            # update revision for pkg now
            dbconn.setRevision(idpackage, revision)

        # make sure that info have been written to disk, again
        dbconn.commit()

        # set trashed counters
        trashing_counters = set()

        for myrepo in myserver_repos:
            mydbconn = self.open_server_repository(myrepo, read_only = True,
                no_upload = True)
            mylist = mydbconn.getPackagesToRemove(
                    mydata['name'],
                    mydata['category'],
                    mydata['slot'],
                    mydata['injected']
            )
            for myitem in mylist:
                trashing_counters.add(mydbconn.retrieveSpmUid(myitem))

        for mycounter in trashing_counters:
            dbconn.setTrashedUid(mycounter)

        # make sure that info have been written to disk, again
        dbconn.commit()

        atom = dbconn.retrieveAtom(idpackage)
        self.output(
            "[%s] %s: %s %s: %s" % (
                    darkgreen(repository_id),
                    blue(_("added package")),
                    darkgreen(atom),
                    blue(_("rev")), # as in revision
                    bold(str(revision)),
                ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage,
            resolve_conditional_deps = False))
        if manual_deps:
            self.output(
                "[%s] %s: %s" % (
                        darkgreen(repository_id),
                        blue(_("manual dependencies for")),
                        darkgreen(atom),
                    ),
                importance = 1,
                level = "warning",
                header = darkgreen("   ## ")
            )
            for m_dep in manual_deps:
                self.output(
                    brown(m_dep),
                    importance = 1,
                    level = "warning",
                    header = darkred("    # ")
                )

        download_url = self._setup_repository_package_filename(dbconn,
            idpackage)
        destination_path = self.complete_local_upload_package_path(
            download_url, repository_id)
        destination_dir = os.path.dirname(destination_path)
        self._ensure_dir_path(destination_dir)

        # since destination_path determines the final path name
        # make sure it corresponds to the one in package_files
        # but copy the object first (shallow)
        move_files = package_files[:]
        package_file = move_files[0]
        # rename to final name first
        final_path = os.path.join(os.path.dirname(package_file),
            os.path.basename(destination_path))
        if package_file != final_path:
            os.rename(package_file, final_path)
            move_files[0] = final_path

        # run in reverse order, this way, the base package file will
        # be moved at the end, resulting in improved reliability.
        destination_paths = []
        for package_file in reversed(move_files):
            final_path = os.path.join(destination_dir,
                os.path.basename(package_file))
            destination_paths.append(final_path)
            try:
                os.rename(package_file, final_path)
            except OSError as err:
                if err.errno != errno.EXDEV:
                    raise
                shutil.move(package_file, final_path)

        # make sure that info have been written to disk, again
        dbconn.commit()

        # reverse it again, since we wrote it reversed already
        destination_paths.reverse()
        return idpackage, destination_paths

    # this function changes the final repository package filename
    def _setup_repository_package_filename(self, dbconn, idpackage):

        downloadurl = dbconn.retrieveDownloadURL(idpackage)
        packagerev = dbconn.retrieveRevision(idpackage)
        downloaddir = os.path.dirname(downloadurl)
        downloadfile = os.path.basename(downloadurl)
        # add revision
        pkg_ext = etpConst['packagesext']
        downloadfile = downloadfile[:-len(pkg_ext)]+"~%s%s" % (packagerev,
            pkg_ext,)
        downloadurl = os.path.join(downloaddir, downloadfile)

        # update url
        dbconn.setDownloadURL(idpackage, downloadurl)
        dbconn.commit()

        return downloadurl

    def __user_filter_out_missing_deps(self, pkg_repo, entropy_repository,
        missing_map, ask):

        def _show_missing_deps(missing_deps):
            self.output(
                "[%s] %s:" % (
                    darkgreen(pkg_repo),
                    teal(_("these are the missing dependencies")),
                ),
                importance = 1,
                level = "info",
                header = purple(" @@ ")
            )
            for pkg_match, deps in missing_deps.items():
                pkg_id, repo = pkg_match
                atom = entropy_repository.retrieveAtom(pkg_id)
                slot = entropy_repository.retrieveSlot(pkg_id)
                self.output(
                    "%s:%s" % (
                        teal(atom),
                        purple(slot),
                    ),
                    level = "info",
                    header = " :: "
                )
                for dep in sorted(deps):
                    self.output(
                        brown(dep),
                        level = "info",
                        header = red("    # ")
                    )

        missing_deps = {}
        if not ask:
            # not interactive, add everything
            for pkg_match, missing_extended in missing_map.items():
                if not missing_extended:
                    continue
                obj = missing_deps.setdefault(pkg_match, set())
                for dep_list in missing_extended.values():
                    obj.update(dep_list)

            _show_missing_deps(missing_deps)
            return missing_deps

        header_txt = """\
# Some missing dependencies have been found.
# Please use this text file to commit the ones you want
# to get added to their respective packages.
#
# HOWTO: just remove the lines containing unwanted dependencies and
# keep those you think are sane.
#
# IMPORTANT:
# Lines starting with "#" are comments and will be ignored.
# Lines starting with "##" are reserved to this application for
# parsing purposes.


"""

        editor_lines = []
        for pkg_match, missing_extended in missing_map.items():

            if not missing_extended:
                # wtf
                continue

            pkg_id, missing_pkg_repo = pkg_match
            if pkg_repo != missing_pkg_repo:
                raise AttributeError("this cannot happen")

            atom = entropy_repository.retrieveAtom(pkg_id)
            slot = entropy_repository.retrieveSlot(pkg_id)
            line = "## %s %s => %s:%s" % (pkg_id, pkg_repo, atom, slot)
            editor_lines.append(const_convert_to_unicode(line))

            for lib_match, dep_list in missing_extended.items():
                library, elf = lib_match
                line = "# %s, %s" % (library, elf)
                editor_lines.append(const_convert_to_unicode(line))
                for dep in sorted(dep_list):
                    editor_lines.append(dep)
            editor_lines.append(const_convert_to_unicode(""))
            editor_lines.append(const_convert_to_unicode(""))

        if not editor_lines:
            # wtf!?
            return {}

        enc = etpConst['conf_encoding']
        tmp_path = None
        while True:

            if tmp_path is None:
                tmp_fd, tmp_path = tempfile.mkstemp(prefix = 'entropy.server',
                    suffix = ".conf")
                with entropy.tools.codecs_fdopen(tmp_fd, "w", enc) as tmp_f:
                    tmp_f.write(header_txt)
                    for editor_line in editor_lines:
                        tmp_f.write(editor_line)
                        tmp_f.write("\n")
                    tmp_f.flush()

            success = self.edit_file(tmp_path)
            if not success:
                # retry ?
                os.remove(tmp_path)
                tmp_path = None
                continue

            # parse the file back, build missing_deps
            all_good = True
            missing_deps = {}
            with codecs.open(tmp_path, "r", encoding=enc) as tmp_f:
                pkg_match = None
                for line in tmp_f.readlines():
                    line = line.strip()

                    if not line:
                        # empty line
                        continue

                    elif line.startswith("##"):
                        # new package match, hopefully
                        line_data = line.lstrip("## ").split()
                        if len(line_data) < 2:
                            # something is really bad
                            all_good = False
                            break
                        try:
                            pkg_id = int(line_data[0])
                        except ValueError:
                            # something is really bad
                            all_good = False
                            break

                        repo = line_data[1]
                        if repo != pkg_repo:
                            # bad here too
                            all_good = False
                            break

                        pkg_match = pkg_id, repo
                        continue

                    elif line.startswith("#"):
                        # ignore comment line
                        continue

                    elif pkg_match:
                        # real line, should contain a valid dependency string
                        # if pkg_match is set
                        obj = missing_deps.setdefault(pkg_match, set())
                        obj.add(line)

            if not all_good:
                os.remove(tmp_path)
                tmp_path = None
                continue

            if not missing_deps:
                self.output(
                    "[%s] %s:" % (
                        darkgreen(pkg_repo),
                        teal(_("no missing dependencies !")),
                    ),
                    importance = 1,
                    level = "warning",
                    header = red(" @@ ")
                )
            else:
                _show_missing_deps(missing_deps)

            # ask confirmation
            while True:
                try:
                    rc_question = self.ask_question(
                        "[%s] %s" % (
                            purple(pkg_repo),
                            teal(_("Do you agree?"))
                        ),
                        responses = (_("Yes"), _("Repeat"),)
                    )
                except KeyboardInterrupt:
                    # do not allow, we're in a critical region
                    continue
                break
            if rc_question == _("Yes"):
                break
            # otherwise repeat everything again
            # keep tmp_path


        if tmp_path is not None:
            try:
                os.remove(tmp_path)
            except (OSError) as err:
                if err.errno != errno.ENOENT:
                    raise
        return missing_deps

    def missing_runtime_dependencies_test(self, package_matches, ask = True,
        bump_packages = False):
        """
        Use Entropy QA interface to check package matches against missing
        runtime dependencies, adding them.

        @param package_matches: list of Entropy package matches
        @type package_matches: list
        @keyword ask: if missing runtime dependencies should be validated
            interactively
        @type ask: bool
        @keyword bump_packages: True, if packages should be bumped (revision
            bumped)
        @type bump_packages: bool
        """
        blacklisted_deps = \
            self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_blacklist']
        my_qa = self.QA()

        def get_blacklisted_deps(package_ids, repo_db):
            my_blacklist = set()
            for pkg_dep, bl_pkg_deps in blacklisted_deps.items():
                pkg_ids, rc = repo_db.atomMatch(pkg_dep, multiMatch = True)
                for package_id in package_ids:
                    if package_id in pkg_ids:
                        my_blacklist.update(bl_pkg_deps)
                        break
            return my_blacklist

        pkg_map = {}
        for pkg_id, pkg_repo in package_matches:
            obj = pkg_map.setdefault(pkg_repo, [])
            obj.append(pkg_id)

        for pkg_repo, package_ids in pkg_map.items():

            repo_blacklist = self._get_missing_dependencies_blacklist(pkg_repo)

            dbconn = self.open_server_repository(pkg_repo, read_only = False,
                no_upload = True)
            pkg_blacklisted_deps = get_blacklisted_deps(package_ids, dbconn)
            pkg_blacklisted_deps |= repo_blacklist

            # missing dependencies check
            missing_map = my_qa.test_missing_dependencies(
                self, [(x, pkg_repo) for x in package_ids],
                self_check = True, black_list = pkg_blacklisted_deps)

            missing_deps = self.__user_filter_out_missing_deps(pkg_repo,
                dbconn, missing_map, ask)

            for (pkg_id, missing_pkg_repo), missing in missing_deps.items():
                if pkg_repo != missing_pkg_repo:
                    # with current API, this never happens!
                    # but since this is a critical region, better being
                    # safe than sorry.
                    # pkg_repo is always the same...
                    raise AssertionError(
                        "pkg_repo and missing_pkg_repo must be equal")

                if bump_packages:
                    # in this case, a new package should be generated, with
                    # bumped revision
                    pkg_data = dbconn.getPackageData(pkg_id)
                    # also bump injected packages properly
                    original_injected = pkg_data['injected']
                    pkg_data['injected'] = False
                    pkg_id = dbconn.handlePackage(pkg_data)
                    if original_injected:
                        dbconn.setInjected(pkg_id)
                    # make sure that info have been written to disk
                    dbconn.commit()

                # NOTE that missing is a list here, so no fancy
                # dependency type is set.
                dbconn.insertDependencies(pkg_id, missing)

            if missing_deps:
                # save changes here again
                dbconn.commit()

    def _external_metadata_qa_hook_test(self, package_matches):
        """
        Execute external metadata QA check, if executable exists.
        Returns True for success, False for error. Warnings are not considered
        blocking.
        """
        qa_exec = os.path.join(SystemSettings.packages_config_directory(),
                               "packages.server.qa.exec")
        if not os.path.isfile(qa_exec):
            return True
        if not os.access(qa_exec, os.X_OK | os.R_OK | os.F_OK):
            return True
        # avoid privs escalation
        st = os.stat(qa_exec)
        file_uid = st[stat.ST_UID]
        file_gid = st[stat.ST_GID]
        if not ((file_uid == 0) and (file_gid == 0)):
            self.output(
                "[%s] %s: %s, %s" % (
                    purple("qa"),
                    brown(_("metadata QA hook")),
                    purple(qa_exec),
                    brown(_("not owned by uid and gid = 0")),
                ),
                importance = 1,
                level = "error",
                header = darkred(" !!! "),
            )
            return False

        count = 0
        maxcount = len(package_matches)
        self.output(
            "[%s] %s: %s" % (
                purple("qa"),
                teal(_("using metadata QA hook")),
                darkgreen(qa_exec),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )
        qa_success = True
        for package_id, repository_id in package_matches:
            count += 1
            repo_db = self.open_server_repository(repository_id,
                read_only = False, no_upload = True)

            pkg_atom, pkg_name, pkg_version, pkg_tag, \
            pkg_description, pkg_category, pkg_chost, \
            pkg_cflags, pkg_cxxflags, pkg_homepage, \
            pkg_license, pkg_branch, pkg_uri, \
            pkg_md5, pkg_slot, pkg_etpapi, \
            pkg_date, pkg_size, pkg_rev = repo_db.getBaseData(package_id)
            pkg_deps = repo_db.retrieveDependenciesList(package_id)
            pkg_needed = repo_db.retrieveNeeded(package_id, extended = True)
            pkg_provided_libs = repo_db.retrieveProvidedLibraries(package_id)
            pkg_keywords = repo_db.retrieveKeywords(package_id)

            self.output(
                "[%s] %s: %s" % (
                    purple("qa"),
                    teal(_("metadata QA check for")),
                    darkgreen(pkg_atom),
                ),
                importance = 1,
                level = "info",
                header = blue(" @@ "),
                count = (count, maxcount),
                back = True
            )
            env = os.environ.copy()
            env['REPOSITORY_ID'] = str(repository_id)
            env['PKG_ID'] = str(package_id)
            env['PKG_ATOM'] = str(pkg_atom)
            env['PKG_NAME'] = str(pkg_name)
            env['PKG_VERSION'] = str(pkg_version)
            env['PKG_TAG'] = str(pkg_tag or "")
            env['PKG_DESCRIPTION'] = const_convert_to_rawstring(pkg_description)
            env['PKG_CATEGORY'] = str(pkg_category)
            env['PKG_CHOST'] = str(pkg_chost)
            env['PKG_CFLAGS'] = str(pkg_cflags)
            env['PKG_CXXFLAGS'] = str(pkg_cxxflags)
            env['PKG_HOMEPAGE'] = const_convert_to_rawstring(pkg_homepage)
            env['PKG_LICENSE'] = str(pkg_license)
            env['PKG_BRANCH'] = str(pkg_branch)
            env['PKG_KEYWORDS'] = str(" ".join(pkg_keywords))
            env['PKG_DOWNLOAD'] = str(pkg_uri)
            env['PKG_MD5'] = str(pkg_md5)
            env['PKG_SLOT'] = str(pkg_slot)
            env['PKG_ETPAPI'] = str(pkg_etpapi)
            env['PKG_DATE'] = str(pkg_date)
            env['PKG_SIZE'] = str(pkg_size)
            env['PKG_REVISION'] = str(pkg_rev)
            env['PKG_DEPS'] = str("\n".join(pkg_deps))
            env['PKG_NEEDED_LIBS'] = str("\n".join(["%s|%s" % (x, y) \
                    for x, y in pkg_needed]))
            env['PKG_PROVIDED_LIBS'] = str("\n".join(
                ["%s|%s|%s" % (x, y, z) for x, y, z in \
                    pkg_provided_libs]))

            # now call the script
            try:
                proc = subprocess.Popen(
                    [qa_exec], stdout = sys.stdout, stderr = sys.stderr,
                    stdin = sys.stdin, env = env)
                rc = proc.wait()
            except OSError as err:
                self.output(
                    "[%s] %s: %s, %s" % (
                        purple("qa"),
                        bold(_("cannot execute metadata QA hook for")),
                        darkgreen(pkg_atom),
                        repr(err),
                    ),
                    importance = 1,
                    level = "error",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                rc = 2
                if const_debug_enabled():
                    import pdb
                    pdb.set_trace()

            if rc == 0:
                # all good
                continue
            elif rc == 1:
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                self.output(
                    "[%s] %s !" % (
                        darkred("qa"),
                        brown(_("attention, QA hook returned a warning")),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
            else:
                # anything != 0 and 1 is considered error
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                self.output(
                    "[%s] %s !" % (
                        darkred("qa"),
                        brown(_("attention, QA hook returned an error")),
                    ),
                    importance = 1,
                    level = "error",
                    header = darkred(" !!! "),
                    count = (count, maxcount)
                )
                self.output("",
                    importance = 0,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                qa_success = False

        self.output(
            "[%s] %s" % (
                purple("qa"),
                teal(_("metadata QA check complete")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )

        return qa_success

    def _add_packages_qa_tests(self, package_matches, ask = True):
        """
        Execute some generic QA checks for broken libraries on packages added
        to repository.
        """
        self.missing_runtime_dependencies_test(package_matches, ask = ask)

        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        my_qa = self.QA()
        my_qa.test_missing_runtime_libraries(self, package_matches,
            base_repository_id = my_settings['base_repository_id'])

        qa_success = self._external_metadata_qa_hook_test(package_matches)
        if my_settings['broken_revdeps_qa_check']:
            my_qa.test_reverse_dependencies_linking(self, package_matches)
        return qa_success

    def add_packages_to_repository(self, repository_id, packages_data,
        ask = True):
        """
        Add package files to given repository. packages_data contains a list
        of tuples composed by (path to package files, execute_injection boolean).
        Injection is a way to avoid a package being removed from the repository
        automatically when an updated package is added.

        @param repository_id: repository identifier
        @type repository_id: string
        @param packages_data: list of tuples composed by ([path, path1], bool)
            The first path object represents the main package file.
            While the others represent further package files.
            If one ends with eptConst['packagesdebugext'], it will be considered
            as debuginfo package file.
        @type packages_data: list
        @return: list (set) of package identifiers added
        @rtype: set
        """
        mycount = 0
        maxcount = len(packages_data)
        idpackages_added = set()
        to_be_injected = set()

        for package_filepaths, inject in packages_data:

            mycount += 1
            for package_filepath in package_filepaths:
                header = blue(" @@ ")
                count = (mycount, maxcount,)
                if package_filepaths[0] != package_filepath:
                    self.output(
                        "%s" % (
                            brown(os.path.basename(package_filepath)),
                        ),
                        importance = 1,
                        level = "info",
                        header = teal("     # ")
                    )
                else:
                    self.output(
                        "[%s] %s: %s" % (
                            darkgreen(repository_id),
                            blue(_("adding package")),
                            darkgreen(os.path.basename(package_filepath)),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        count = (mycount, maxcount,)
                    )

            if inject and len(package_filepaths) == 1:
                # just make sure user is aware of the fact that no separate
                # debug packages will be made.
                self.output(
                    "%s" % (
                        brown(_("injected package, no separate debug package")),
                    ),
                    importance = 1,
                    level = "info",
                    header = teal("     !! ")
                )

            try:
                # add to database
                idpackage, destination_paths = self._package_injector(
                    repository_id, package_filepaths, inject = inject)
                idpackages_added.add(idpackage)
                to_be_injected.add((idpackage, destination_paths[0]))
            except Exception as err:
                entropy.tools.print_traceback()
                self.output(
                    "[%s] %s: %s" % (
                        darkgreen(repository_id),
                        darkred(_("Exception caught, closing tasks")),
                        darkgreen(str(err)),
                    ),
                    importance = 1,
                    level = "error",
                    header = bold(" !!! "),
                    count = (mycount, maxcount,)
                )
                # reinit librarypathsidpackage table
                if idpackages_added:
                    self._add_packages_qa_tests(
                        [(x, repository_id) for x in idpackages_added],
                        ask = ask)
                if to_be_injected:
                    self._inject_database_into_packages(repository_id,
                        to_be_injected)
                self.close_repositories()
                raise

        # make sure packages are really available, it can happen
        # after a previous failure to have garbage here
        dbconn = self.open_server_repository(repository_id, just_reading = True)
        idpackages_added = set((x for x in idpackages_added if \
            dbconn.isPackageIdAvailable(x)))

        if idpackages_added:
            self._add_packages_qa_tests(
                [(x, repository_id) for x in idpackages_added], ask = ask)

        # inject database into packages
        self._inject_database_into_packages(repository_id, to_be_injected)

        return idpackages_added

    def _taint_database(self, repository_id):

        # taint the database status
        db_file = self._get_local_repository_file(repository_id)
        taint_file = self._get_local_repository_taint_file(repository_id)
        with codecs.open(taint_file, "w") as f:
            f.write("repository tainted\n")
            f.flush()
        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        ServerRepositoryStatus().set_tainted(db_file)

    def _bump_database(self, repository_id):
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        self._taint_database(repository_id)
        dbconn.commit()
        self.close_repository(dbconn)


    def _ensure_paths(self, repo):
        upload_dir = self._get_local_upload_directory(repo)
        db_dir = self._get_local_repository_dir(repo)
        for mydir in [upload_dir, db_dir]:
            if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
                os.makedirs(mydir, 0o775)
                const_setup_perms(mydir, etpConst['entropygid'],
                    recursion = False, uid = etpConst['uid'])

    def _ensure_dir_path(self, dir_path):
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path, 0o775)
            const_setup_perms(dir_path, etpConst['entropygid'],
                recursion = False, uid = etpConst['uid'])

    def _setup_services(self):
        self._setup_entropy_settings(self._repository)
        self._backup_entropy_settings()
        self.Mirrors = MirrorsServer(self, self._repository)

    def _setup_entropy_settings(self, repository_id):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        backup_list = [
            'etpdatabaseclientfilepath',
            'clientdbid',
            {'server': srv_set.copy()},
        ]
        for setting in backup_list:
            if setting not in self._settings_to_backup:
                self._settings_to_backup.append(setting)
        # setup entropy client repo
        if not srv_set['community_mode']:
            repo_data = srv_set['repositories'][repository_id]
            if "__temporary__" not in repo_data:
                etpConst['etpdatabaseclientfilepath'] = \
                    self._get_local_repository_file(repository_id)
            etpConst['clientdbid'] = etpConst['serverdbid']
        const_create_working_dirs()

    def _show_interface_status(self):
        type_txt = _("server-side repository")
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if srv_set['community_mode']:
            type_txt = _("community repository")
        # ..on repository: <repository_name>
        mytxt = _("Entropy Server Interface Instance on repository")
        self.output(
            "%s: %s" % (blue(mytxt),
                red(self._repository),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        self.output(
            "%s: %s (%s: %s)" % (
                brown(_("current branch")),
                darkgreen(self._settings['repositories']['branch']),
                purple(_("type")),
                bold(type_txt),
            ),
            importance = 1,
            level = "info",
            header = red("    ")
        )
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repos = list(srv_set['repositories'].keys())
        mytxt = blue("%s:") % (_("Currently configured repositories"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )
        for repo in repos:
            self.output(
                darkgreen(repo),
                importance = 0,
                level = "info",
                header = brown("   # ")
            )

    def _backup_constant(self, constant_name):
        if constant_name not in etpConst:
            raise KeyError("%s not available" % (constant_name,))
        etpConst['backed_up'].update(
            {constant_name: copy.copy(etpConst[constant_name])})

    def _backup_entropy_settings(self):
        for setting in self._settings_to_backup:
            if isinstance(setting, const_get_stringtype()):
                self._backup_constant(setting)
            elif isinstance(setting, dict):
                self._settings.set_persistent_setting(setting)

    def _get_gpg_signature(self, repo_sec, repo, pkg_path):
        try:
            if not repo_sec.is_keypair_available(repo):
                return None # GPG is not enabled
        except RepositorySecurity.KeyExpired as err:
            self.output(
                "[%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG key expired")),
                    err,
                    darkred(_("please frigging fix")),
                ),
                importance = 1,
                level = "warning",
                header = bold(" !!! ")
            )
            return None
        except RepositorySecurity.GPGError as err:
            self.output(
                "[%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG got unexpected error")),
                    err,
                    darkred(_("skipping")),
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            return None
        gpg_sign_path = repo_sec.sign_file(repo, pkg_path)
        # read file content and add to 'gpg' signature
        with open(gpg_sign_path, "rb") as gpg_f:
            gpg_signature = gpg_f.read()
        os.remove(gpg_sign_path)
        return gpg_signature

    def _check_config_file_updates(self):
        self.output(
            "[%s] %s" % (
                red(_("config files")), # something short please
                blue(_("checking system")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ "),
            back = True
        )
        # scanning for config files not updated
        updates = self.ConfigurationUpdates()
        file_updates = updates.get()
        if file_updates:
            self.output(
                "[%s] %s" % (
                    red(_("config files")), # something short please
                    blue(_("there are configuration files not updated yet")),
                ),
                importance = 1,
                level = "error",
                header = darkred(" @@ ")
            )
            for val in file_updates.values():
                self.output(
                    val['destination'],
                    importance = 1,
                    level = "info",
                    header = "\t"
                )
            return True
        return False

    def _get_missing_dependencies_blacklist(self, repository_id, branch = None):
        if branch is None:
            branch = self._settings['repositories']['branch']
        wl_file = self._get_missing_dependencies_blacklist_file(repository_id,
            branch = branch)
        wl_data = []
        if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
            f_wl = codecs.open(wl_file, "r", encoding=etpConst['conf_encoding'])
            wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
                not x.strip().startswith("#")]
            f_wl.close()
        return set(wl_data)

    def _get_package_path(self, repo, dbconn, idpackage):
        """
        Given EntropyRepository instance and package identifier, return local
        path of package. This method does not check for path validity though.
        """
        pkg_rel_url = dbconn.retrieveDownloadURL(idpackage)
        complete_path = self.complete_local_package_path(pkg_rel_url, repo)
        return complete_path

    def _get_upload_package_path(self, repo, dbconn, idpackage):
        """
        Given ServerPackagesRepository instance and package identifier,
        return local path of package.
        This method does not check for path validity though.
        """
        pkg_path = dbconn.retrieveDownloadURL(idpackage)
        return os.path.join(self._get_local_upload_directory(repo), pkg_path)

    def scan_package_changes(self):
        """
        Scan, using Source Package Manager, for added/removed/updated packages.
        Please note that in order to trigger SPM level package moves, it is better
        to also call Spm().package_names_update() before this, or opening all the repos
        in read/write (read_only=False).

        @return: tuple composed of (1) list of spm package name and spm package
        id, (2) list of entropy package matches for packages to be removed (3)
        list of entropy package matches for packages to be injected.
        """
        spm = self.Spm()

        spm_packages = spm.get_installed_packages()
        installed_packages = []
        for spm_package in spm_packages:
            try:
                pkg_counter = spm.resolve_spm_package_uid(spm_package)
            except KeyError:
                # not found
                continue
            installed_packages.append((spm_package, pkg_counter,))

        installed_counters = set()
        to_be_added = set()
        to_be_removed = set()
        to_be_injected = set()
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        exp_based_scope = my_settings['exp_based_scope']

        server_repos = list(my_settings['repositories'].keys())

        # packages to be added
        for spm_atom, spm_counter in installed_packages:
            found = False
            for server_repo in server_repos:
                installed_counters.add(spm_counter)
                server_dbconn = self.open_server_repository(server_repo,
                    read_only = True, no_upload = True)
                counter = server_dbconn.isSpmUidAvailable(spm_counter)
                if counter:
                    found = True
                    break
            if not found:
                to_be_added.add((spm_atom, spm_counter,))

        # packages to be removed from the database
        database_counters = {}
        for server_repo in server_repos:
            server_dbconn = self.open_server_repository(server_repo,
                read_only = True, no_upload = True)
            database_counters[server_repo] = server_dbconn.listAllSpmUids()

        ordered_counters = set()
        for server_repo in database_counters:
            for data in database_counters[server_repo]:
                ordered_counters.add((data, server_repo))
        database_counters = ordered_counters

        # do some memoization to speed up the scanning
        _spm_key_slot_map = {}
        for _spm_pkg, _spm_pkg_id in to_be_added:
            key = entropy.dep.dep_getkey(_spm_pkg)
            obj = _spm_key_slot_map.setdefault(key, set())
            try:
                slot = spm.get_installed_package_metadata(
                    _spm_pkg, "SLOT")
                # workaround for ebuilds without SLOT
                if slot is None:
                    slot = "0"
                obj.add(slot)
            except KeyError:
                continue

        for (counter, idpackage), xrepo in database_counters:

            if counter < 0:
                continue # skip packages without valid counter

            if counter in installed_counters:
                continue

            dbconn = self.open_server_repository(xrepo, read_only = True,
                no_upload = True)

            dorm = True
            # check if the package is in to_be_added
            if to_be_added:

                dorm = False
                atom = dbconn.retrieveAtom(idpackage)
                atomkey = entropy.dep.dep_getkey(atom)
                atomtag = entropy.dep.dep_gettag(atom)
                atomslot = dbconn.retrieveSlot(idpackage)
                add = True

                spm_slots = _spm_key_slot_map.get(atomkey, [])
                # atomtag != None is for handling tagged pkgs correctly
                if (atomslot in spm_slots) or (atomtag is not None):
                    # do not add to to_be_removed
                    add = False

                if not add:
                    continue
                dorm = True

            # checking if we are allowed to remove stuff on this repo
            # if xrepo is not the default one, we MUST skip this to
            # avoid touching what developer doesn't expect
            if dorm and (xrepo == self._repository):
                trashed = self._is_spm_uid_trashed(counter)
                if trashed:
                    # search into portage then
                    try:
                        key, slot = dbconn.retrieveKeySlot(idpackage)
                        slot = slot.split(",")[0]
                        try:
                            trashed = spm.match_installed_package(
                                key + ":" + slot)
                        except KeyError:
                            trashed = True
                    except TypeError: # referred to retrieveKeySlot
                        trashed = True
                if not trashed:

                    dbtag = dbconn.retrieveTag(idpackage)
                    if dbtag:
                        is_injected = dbconn.isInjected(idpackage)
                        if not is_injected:
                            to_be_injected.add((idpackage, xrepo))

                    elif exp_based_scope:

                        # check if support for this is set
                        plg_id = self.sys_settings_fatscope_plugin_id
                        exp_data = self._settings[plg_id]['repos'].get(
                            xrepo, set())

                        # only some packages are set, check if our is
                        # in the list
                        if (idpackage not in exp_data) and (-1 not in exp_data):
                            to_be_removed.add((idpackage, xrepo))
                            continue

                        idpackage_expired = self._is_match_expired((idpackage,
                            xrepo,))

                        if idpackage_expired:
                            # expired !!!
                            # add this and its depends (reverse deps)

                            rm_match = (idpackage, xrepo)
                            #to_be_removed.add(rm_match)
                            revdep_matches = self.get_reverse_queue([rm_match],
                                system_packages = False)
                            to_be_removed.update(revdep_matches)

                    else:
                        to_be_removed.add((idpackage, xrepo))

        return to_be_added, to_be_removed, to_be_injected

    def _is_match_expired(self, match):

        idpackage, repoid = match
        dbconn = self.open_server_repository(repoid, just_reading = True)
        # 3600 * 24 = 86400
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
        cur_unix_time = time.time()
        # if packages removal is triggered by expiration
        # we will have to check if our package is really
        # expired and remove its reverse deps too
        mydate = dbconn.retrieveCreationDate(idpackage)
        # cross fingers hoping that time is set correctly
        mydelta = cur_unix_time - float(mydate)
        if mydelta > pkg_exp_secs:
            return True
        return False

    def _is_spm_uid_trashed(self, counter):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(srv_set['repositories'].keys())
        for repo in server_repos:
            dbconn = self.open_server_repository(repo, read_only = True,
                no_upload = True)
            if dbconn.isSpmUidTrashed(counter):
                return True
        return False

    def _transform_package_into_injected(self, package_id, repository_id):
        dbconn = self.open_server_repository(repository_id, read_only = False,
            no_upload = True)
        counter = dbconn.getFakeSpmUid()
        dbconn.setSpmUid(package_id, counter)
        dbconn.setInjected(package_id)
        dbconn.commit()

    def _pump_extracted_package_metadata(self, pkg_meta, repo, extra_metadata):
        """
        Add to pkg_meta dict, server-side package metadata information before
        injecting it into repository database.
        """
        # add extra metadata
        pkg_meta.update(extra_metadata)
        # do not set GPG signature here for performance, just provide an
        # empty default. Real GPG signature will be written inside
        # _inject_database_into_packages()
        pkg_meta['signatures']['gpg'] = None

        # rewrite dependency strings using dep_rewrite metadata
        self.__handle_dep_rewrite(pkg_meta, repo)

    def __handle_dep_rewrite(self, pkg_meta, repo):

        dep_rewrite = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_rewrite']

        # NOTE: to be able to match the package, we need to add it
        # to a temp repo
        tmp_repo = self._open_temp_repository("dep_rewrite_temp",
            temp_file = ":memory:")
        new_idpackage = tmp_repo.handlePackage(pkg_meta)
        pkg_atom = tmp_repo.retrieveAtom(new_idpackage)

        rewrites_enabled = []
        wildcard_rewrite = False
        for dep_string_rewrite, dep_pattern in dep_rewrite:
            # magic catch-all support
            if dep_string_rewrite == "*":
                pkg_id, rc = None, 0
                wildcard_rewrite = True
            else:
                wildcard_rewrite = False
                pkg_id, rc = tmp_repo.atomMatch(dep_string_rewrite)
            if rc == 0:
                rewrites_enabled.append((dep_string_rewrite, dep_pattern))
        tmp_repo.close()

        if not rewrites_enabled:
            return

        if not wildcard_rewrite:
            self.output(
                "[%s|%s] %s:" % (
                        blue(repo),
                        brown(pkg_atom),
                        teal(
                            _("found available dep_rewrites for this package")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            for dep_string_rewrite, dep_pattern in rewrites_enabled:
                compiled_pattern, replaces = \
                    dep_rewrite[(dep_string_rewrite, dep_pattern)]
                if compiled_pattern is None:
                    # this means that user is asking to add dep_pattern
                    # as a dependency to package
                    replaces_str = _("added")
                elif not replaces:
                    # this means that user is asking to remove dep_pattern
                    replaces_str = _("removed")
                else:
                    replaces_str = "=> " + ', '.join(replaces)
                self.output(
                    "%s %s" % (
                        purple(dep_pattern),
                        replaces_str,
                    ),
                    importance = 1,
                    level = "info",
                    header = teal("   # ")
                )

        def _extract_dep_add_from_dep_pattern(_dep_pattern):
            """
            when action is to add a dependency, extracts the dependency type
            from dependency string, if found.
            """
            _dep_type = etpConst['dependency_type_ids']['mdepend_id']
            bracket_start_idx = _dep_pattern.rfind("<")
            _is_conflict = False

            if _dep_pattern.endswith(">") and (bracket_start_idx != -1):
                _c_dep_type = _dep_pattern[bracket_start_idx+1:-1]
                try:
                    _c_dep_type = int(_c_dep_type)
                    if _c_dep_type not in (1, 2, 3, 4):
                        raise ValueError()
                except ValueError:
                    # cannot correctly evaluate, giving up silently
                    return _dep_pattern, _dep_type

                _dep_pattern = _dep_pattern[:bracket_start_idx]
                # given packages.server.dep_rewrite.example specifications
                # this is the mapping:
                if _c_dep_type == 1:
                    _dep_type = etpConst['dependency_type_ids']['bdepend_id']
                elif _c_dep_type == 2:
                    _dep_type = etpConst['dependency_type_ids']['rdepend_id']
                elif _c_dep_type == 3:
                    _dep_type = etpConst['dependency_type_ids']['pdepend_id']
                elif _c_dep_type == 4:
                    # manual dependency
                    _dep_type = etpConst['dependency_type_ids']['mdepend_id']

            if _dep_pattern.startswith("!"):
                _is_conflict = True
                _dep_pattern = _dep_pattern[1:]

            return _dep_pattern, _dep_type, _is_conflict

        # pkg_meta['conflicts'] is a frozenset
        conflicts = set(pkg_meta['conflicts'])

        for dep_string, dep_value in pkg_meta['dependencies'].items():

            dep_string_matched = False
            matched_pattern = False

            for key in rewrites_enabled:

                compiled_pattern, replaces = dep_rewrite[key]
                if compiled_pattern is None:
                    # user is asking to add dep_pattern to dependency list
                    dep_pattern_string, dep_pattern_type, conflict = \
                        _extract_dep_add_from_dep_pattern(dep_pattern)
                    if conflict:
                        conflicts.add(dep_pattern_string)
                    else:
                        pkg_meta['dependencies'][dep_pattern_string] = \
                            dep_pattern_type
                    continue

                if not compiled_pattern.match(dep_string):
                    # dep_string not matched, skipping
                    continue
                matched_pattern = True

                if not replaces:
                    # then it's a removal
                    dep_string_matched = True

                for replace in replaces:
                    new_dep_string, number_of_subs_made = \
                        compiled_pattern.subn(replace, dep_string)
                    if number_of_subs_made:
                        dep_string_matched = True
                        if new_dep_string and (new_dep_string != "-"):
                            pkg_meta['dependencies'][new_dep_string] = dep_value
                            self.output(
                                "%s: %s => %s" % (
                                    teal(_("replaced")),
                                    brown(dep_string),
                                    purple(new_dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )
                        else:
                            self.output(
                                "%s: %s => X" % (
                                    teal(_("removed")),
                                    brown(dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )

                    else:
                        self.output(
                            "%s: %s + %s" % (
                                darkred(_("No dependency rewrite made for")),
                                brown(dep_string),
                                purple(replace),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred("   !!! ")
                        )

            if dep_string_matched:
                del pkg_meta['dependencies'][dep_string]
            elif (not dep_string_matched) and matched_pattern:
                self.output(
                    "%s: %s :: %s" % (
                        darkred(_("No dependency rewrite made for")),
                        brown(pkg_atom),
                        purple(dep_string),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred("   !x!x!x! ")
                )

        # save conflicts metadata back in place
        pkg_meta['conflicts'] = frozenset(conflicts)

    def _get_entropy_sets(self, repository_id, branch = None):
        if branch is None:
            branch = self._settings['repositories']['branch']
        sets_dir = self._get_local_database_sets_dir(repository_id,
            branch = branch)
        if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
            return {}

        mydata = {}
        items = os.listdir(sets_dir)
        for item in items:

            try:
                item_clean = str(item)
            except (UnicodeEncodeError, UnicodeDecodeError,):
                continue
            item_path = os.path.join(sets_dir, item)
            if not (os.path.isfile(item_path) and \
                os.access(item_path, os.R_OK)):
                continue
            item_elements = self._settings._extract_packages_from_set_file(
                item_path)
            if item_elements:
                mydata[item_clean] = item_elements.copy()

        return mydata

    def _get_configured_package_sets(self, repository_id):

        branch = self._settings['repositories']['branch']
        # portage sets
        sets_data = self.Spm().get_package_sets(False)
        sets_data.update(self._get_entropy_sets(repository_id, branch = branch))

        invalid_sets = set()
        # validate
        for setname in sets_data:
            good = True
            for atom in sets_data[setname]:
                if atom.startswith(etpConst['packagesetprefix']):
                    # ignore nested package sets
                    continue
                dbconn = self.open_server_repository(repository_id,
                    just_reading = True)
                match = dbconn.atomMatch(atom)
                if match[0] == -1:
                    good = False
                    break
            if not good:
                invalid_sets.add(setname)
        for invalid_set in invalid_sets:
            del sets_data[invalid_set]

        return sets_data

    def _update_package_sets(self, repoitory_id, entropy_repository):

        self.output(
            "[%s|%s] %s..." % (
                darkgreen(repoitory_id),
                purple(_("sets")),
                blue(_("updating package sets")),
            ),
            importance = 0,
            level = "info",
            header = blue(" @@ "),
            back = True
        )

        package_sets = self._get_configured_package_sets(repoitory_id)

        # tell what package sets got added, and what got removed
        current_sets = set(entropy_repository.retrievePackageSets())
        configured_sets = set(package_sets)
        new_sets = sorted(configured_sets - current_sets)
        removed_sets = sorted(current_sets - configured_sets)
        for new_set in new_sets:
            self.output(
                "[%s|%s] %s: %s" % (
                    darkgreen(repoitory_id),
                    purple(_("sets")),
                    blue(_("adding package set")),
                    brown(new_set),
                ),
                importance = 0,
                level = "info",
                header = darkgreen(" @@ ")
            )
        for removed_set in removed_sets:
            self.output(
                "[%s|%s] %s: %s" % (
                    teal(repoitory_id),
                    brown(_("sets")),
                    purple(_("removing package set")),
                    bold(removed_set),
                ),
                importance = 1,
                level = "warning",
                header = darkred(" @@ ")
            )

        entropy_repository.clearPackageSets()
        if package_sets:
            entropy_repository.insertPackageSets(package_sets)