entropy/libraries/entropy/server/interfaces/main.py

# -*- coding: utf-8 -*-
"""

    @author: Fabio Erculiani <lxnay@sabayon.org>
    @contact: lxnay@sabayon.org
    @copyright: Fabio Erculiani
    @license: GPL-2

    B{Entropy Package Manager Server Main Interfaces}.

"""
import sys
import os
import shutil
import copy
import tempfile
import time
import re

from entropy.exceptions import OnlineMirrorError, PermissionDenied, \
    SystemDatabaseError
from entropy.const import etpConst, etpSys, const_setup_perms, \
    const_create_working_dirs, etpUi, \
    const_setup_file, const_get_stringtype, const_debug_write
from entropy.output import purple, red, darkgreen, \
    bold, brown, blue, darkred, teal
from entropy.cache import EntropyCacher
from entropy.server.interfaces.mirrors import Server as MirrorsServer
from entropy.i18n import _
from entropy.core.settings.base import SystemSettings
from entropy.core.settings.plugins.skel import SystemSettingsPlugin
from entropy.transceivers import EntropyTransceiver
from entropy.db import EntropyRepository
from entropy.db.skel import EntropyRepositoryPlugin
from entropy.server.interfaces.db import ServerRepositoryStatus, \
    ServerPackagesRepository
from entropy.spm.plugins.factory import get_default_instance as get_spm, \
    get_default_class as get_spm_class
from entropy.qa import QAInterfacePlugin
from entropy.security import Repository as RepositorySecurity
from entropy.db.exceptions import ProgrammingError
from entropy.client.interfaces import Client as _Client

import entropy.tools
import entropy.dump

SERVER_QA_PLUGIN = "ServerQAInterfacePlugin"


class ServerEntropyRepositoryPlugin(EntropyRepositoryPlugin):

    PLUGIN_ID = "__server__"

    def __init__(self, server_interface, metadata = None):
        """
        Entropy server-side repository ServerPackagesRepository Plugin class.
        This class will be instantiated and automatically added to
        ServerPackagesRepository instances generated by Entropy Server.

        @param server_interface: Entropy Server interface instance
        @type server_interface: entropy.server.interfaces.Server class
        @param metadata: any dict form metadata map (key => value)
        @type metadata: dict
        """
        EntropyRepositoryPlugin.__init__(self)
        self._cacher = EntropyCacher()
        self._settings = SystemSettings()
        self.srv_sys_settings_plugin = \
            etpConst['system_settings_plugins_ids']['server_plugin']
        self._server = server_interface
        if metadata is None:
            self._metadata = {}
        else:
            self._metadata = metadata

    def get_id(self):
        return ServerEntropyRepositoryPlugin.PLUGIN_ID

    def get_metadata(self):
        """
        This method should always return a direct reference to the object and
        NOT a copy.
        """
        return self._metadata

    def add_plugin_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
                self,)
            )

        repo = self._metadata['repo_name']
        local_dbfile = self._metadata['local_dbfile']
        taint_file = self._server._get_local_database_taint_file(repo)
        if os.path.isfile(taint_file):
            dbs = ServerRepositoryStatus()
            dbs.set_tainted(local_dbfile)
            dbs.set_bumped(local_dbfile)


        if "__temporary__" in self._metadata: # in-memory db?
            local_dbfile_exists = True
        else:
            local_dbfile_exists = os.path.lexists(local_dbfile)

        if not local_dbfile_exists:
            # better than having a completely broken db
            self._metadata['read_only'] = False
            entropy_repository_instance.readonly = False
            entropy_repository_instance.initializeRepository()
            entropy_repository_instance.commitChanges()

        out_intf = self._metadata.get('output_interface')
        if out_intf is not None:
            entropy_repository_instance.output = out_intf.output
            entropy_repository_instance.ask_question = out_intf.ask_question

        return 0

    def close_repo_hook(self, entropy_repository_instance):
        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling close_repo_hook => %s" % (
                self,)
            )

        # this happens because close_repositories() might be called
        # before _setup_services() and in general, at any time, so, in this
        # case, there is no need to print bullshit to dev.
        if self._server.Mirrors is None:
            return 0

        repo = self._metadata['repo_name']
        dbfile = self._metadata['local_dbfile']
        read_only = self._metadata['read_only']
        if not read_only:
            sts = ServerRepositoryStatus()
            if sts.is_tainted(dbfile) and not sts.is_unlock_msg(dbfile):
                u_msg = "[%s] %s" % (brown(repo),
                    darkgreen(_("mirrors have not been unlocked. Sync them.")),)
                self._server.output(
                    u_msg,
                    importance = 1,
                    level = "warning",
                    header = brown(" * ")
                )
                # avoid spamming
                sts.set_unlock_msg(dbfile)

        return 0

    def commit_hook(self, entropy_repository_instance):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling commit_hook => %s" % (
                self,)
            )

        dbs = ServerRepositoryStatus()
        dbfile = self._metadata['local_dbfile']
        repo = self._metadata['repo_name']
        read_only = self._metadata['read_only']
        if read_only:
            # do not taint database
            return 0

        # taint the database status
        taint_file = self._server._get_local_database_taint_file(repo = repo)
        f = open(taint_file, "w")
        f.write(etpConst['currentarch']+" database tainted\n")
        f.flush()
        f.close()
        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        dbs.set_tainted(dbfile)

        if not dbs.is_bumped(dbfile):
            # bump revision, setting DatabaseBump causes
            # the session to just bump once
            dbs.set_bumped(dbfile)
            """
            Entropy repository revision bumping function. Every time it's called,
            revision is incremented by 1.
            """
            revision_file = self._server._get_local_database_revision_file(
                repo = repo)
            if not os.path.isfile(revision_file):
                revision = 1
            else:
                with open(revision_file, "r") as rev_f:
                    revision = int(rev_f.readline().strip())
                    revision += 1

            tmp_revision_file = revision_file + ".tmp"
            with open(tmp_revision_file, "w") as rev_fw:
                rev_fw.write(str(revision)+"\n")
                rev_fw.flush()
            # atomic !
            os.rename(tmp_revision_file, revision_file)

            # auto-update package sets
            self._server.output(
                "[repo:%s|%s] %s" % (
                        blue(repo),
                        red(_("database")),
                        blue(_("syncing package sets")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            cur_sets = entropy_repository_instance.retrievePackageSets()
            sys_sets = self._server._get_configured_package_sets(repo)
            if cur_sets != sys_sets:
                self._server._update_database_package_sets(repo,
                    dbconn = entropy_repository_instance)
            entropy_repository_instance.commitChanges(no_plugins = True)

        return 0

    def _get_category_description_from_disk(self, category):
        """
        Get category name description from Source Package Manager.

        @param category: category name
        @type category: string
        @return: category description
        @rtype: string
        """
        spm = self._server.Spm()
        return spm.get_package_category_description_metadata(category)

    def _write_rss_for_removed_package(self, repo_db, package_id):

        # setup variables we're going to use
        srv_repo = self._metadata['repo_name']
        rss_revision = repo_db.retrieveRevision(package_id)
        rss_atom = "%s~%s" % (repo_db.retrieveAtom(package_id), rss_revision,)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if pkgatom (rss_atom) is in the "added" metadata, drop it
        if rss_atom in srv_updates['added']:
            del srv_updates['added'][rss_atom]
        # same thing for light key
        if rss_atom in srv_updates['light']:
            del srv_updates['light'][rss_atom]

        # add metadata
        mydict = {}
        try:
            mydict['description'] = repo_db.retrieveDescription(package_id)
        except TypeError:
            mydict['description'] = "N/A"
        try:
            mydict['homepage'] = repo_db.retrieveHomepage(package_id)
        except TypeError:
            mydict['homepage'] = ""
        srv_updates['removed'][rss_atom] = mydict

        # save to disk
        self._cacher.push(rss_name, srv_updates, async = False,
            cache_dir = Server.CACHE_DIR)

    def _write_rss_for_added_package(self, repo_db, package_data):

        # setup variables we're going to use
        srv_repo = self._metadata['repo_name']
        rss_atom = "%s~%s" % (package_data['atom'], package_data['revision'],)
        status = ServerRepositoryStatus()
        srv_updates = status.get_updates_log(srv_repo)
        rss_name = srv_repo + etpConst['rss-dump-name']

        # load metadata from on disk cache, if available
        rss_obj = self._cacher.pop(rss_name, cache_dir = Server.CACHE_DIR)
        if rss_obj:
            srv_updates.update(rss_obj)

        # setup metadata keys, if not available
        if 'added' not in srv_updates:
            srv_updates['added'] = {}
        if 'removed' not in srv_updates:
            srv_updates['removed'] = {}
        if 'light' not in srv_updates:
            srv_updates['light'] = {}

        # if package_data['atom'] (rss_atom) is in the
        # "removed" metadata, drop it
        if rss_atom in srv_updates['removed']:
            del srv_updates['removed'][rss_atom]

        # add metadata
        srv_updates['added'][rss_atom] = {}
        srv_updates['added'][rss_atom]['description'] = \
            package_data['description']
        srv_updates['added'][rss_atom]['homepage'] = \
            package_data['homepage']
        srv_updates['light'][rss_atom] = {}
        srv_updates['light'][rss_atom]['description'] = \
            package_data['description']

        # save to disk
        self._cacher.push(rss_name, srv_updates, async = False,
            cache_dir = Server.CACHE_DIR)

    def add_package_hook(self, entropy_repository_instance, package_id,
        package_data):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling add_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled']:
            self._write_rss_for_added_package(entropy_repository_instance,
                package_data)

        try:
            descdata = self._get_category_description_from_disk(
                package_data['category'])
            entropy_repository_instance.setCategoryDescription(
                package_data['category'], descdata)
        except (IOError, OSError, EOFError,):
            pass

        return 0

    def remove_package_hook(self, entropy_repository_instance, package_id,
        from_add_package):

        const_debug_write(__name__,
            "ServerEntropyRepositoryPlugin: calling remove_package_hook => %s" % (
                self,)
            )

        # handle server-side repo RSS support
        sys_set_plug = self.srv_sys_settings_plugin
        if self._settings[sys_set_plug]['server']['rss']['enabled'] \
            and (not from_add_package):

            # store addPackage action
            self._write_rss_for_removed_package(entropy_repository_instance,
                package_id)

        return 0

    def treeupdates_move_action_hook(self, entropy_repository_instance,
        package_id):
        # check for injection and warn the developer
        injected = entropy_repository_instance.isInjected(package_id)
        new_atom = entropy_repository_instance.retrieveAtom(package_id)
        if injected:
            mytxt = "%s: %s %s. %s !!! %s." % (
                bold(_("INJECT")),
                blue(str(new_atom)),
                red(_("has been injected")),
                red(_("quickpkg manually to update embedded db")),
                red(_("Repository database updated anyway")),
            )
            self._server.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = darkred(" * ")
            )
        return 0

    def treeupdates_slot_move_action_hook(self, entropy_repository_instance,
        package_id):
        return self.treeupdates_move_action_hook(entropy_repository_instance,
            package_id)


class ServerSystemSettingsPlugin(SystemSettingsPlugin):

    @staticmethod
    def analyze_server_repo_string(repostring, product = None):
        """
        Analyze a server repository string (usually contained in server.conf),
        extracting all the parameters.

        @param repostring: repository string
        @type repostring: string
        @keyword product: system product which repository belongs to
        @rtype: None
        @return: None
        """

        if product == None:
            product = etpConst['product']

        mydata = {}
        repoid = repostring.split("|")[1].strip()
        repodesc = repostring.split("|")[2].strip()
        repouris = repostring.split("|")[3].strip()

        service_url = None
        eapi3_port = etpConst['socket_service']['port']
        eapi3_ssl_port = etpConst['socket_service']['ssl_port']
        if len(repostring.split("|")) > 4:
            service_url = repostring.split("|")[4].strip()

            eapi3_formatcolon = service_url.rfind("#")
            if eapi3_formatcolon != -1:
                try:
                    ports = service_url[eapi3_formatcolon+1:].split(",")
                    eapi3_port = int(ports[0])
                    if len(ports) > 1:
                        eapi3_ssl_port = int(ports[1])
                    service_url = service_url[:eapi3_formatcolon]
                except (ValueError, IndexError,):
                    eapi3_port = etpConst['socket_service']['port']
                    eapi3_ssl_port = etpConst['socket_service']['ssl_port']

        mydata = {}
        mydata['repoid'] = repoid
        mydata['description'] = repodesc
        mydata['pkg_mirrors'] = []
        mydata['repo_mirrors'] = []
        mydata['community'] = False
        mydata['service_url'] = service_url
        mydata['service_port'] = eapi3_port
        mydata['ssl_service_port'] = eapi3_ssl_port
        uris = repouris.split()
        for uri in uris:
            do_pkg = False
            do_repo = False
            while True:
                if uri.startswith("<p>"):
                    do_pkg = True
                    uri = uri[3:]
                    continue
                if uri.startswith("<r>"):
                    do_repo = True
                    uri = uri[3:]
                    continue
                break
            if not (do_repo or do_pkg):
                do_repo = True
                do_pkg = True
            if do_repo:
                mydata['repo_mirrors'].append(uri)
            if do_pkg:
                mydata['pkg_mirrors'].append(uri)

        return repoid, mydata

    def __generic_parser(self, filepath):
        """
        Internal method. This is the generic file parser here.

        @param filepath: valid path
        @type filepath: string
        @return: raw text extracted from file
        @rtype: list
        """
        lines = entropy.tools.generic_file_content_parser(filepath,
            comment_tag = "##")
        # filter out non-ASCII lines
        lines = [x for x in lines if entropy.tools.is_valid_ascii(x)]
        return lines

    def dep_rewrite_parser(self, sys_set):

        cached = getattr(self, '_mod_rewrite_data', None)
        if cached is not None:
            return cached

        rewrite_file = etpConst['etpdatabasedeprewritefile']
        rewrite_content = self.__generic_parser(rewrite_file)

        data = {}
        for line in rewrite_content:
            params = line.split()
            if len(params) < 3:
                continue
            pkg_match, pattern, replaces = params[0], params[1], params[2:]
            try:
                compiled_pattern = re.compile(pattern)
            except re.error:
                # invalid pattern
                continue
            # use this key to make sure to not overwrite similar entries
            data[(pkg_match, pattern)] = (compiled_pattern, replaces)

        self._mod_rewrite_data = data
        return data

    def server_parser(self, sys_set):

        """
        Parses Entropy server system configuration file.

        @return dict data
        """

        data = {
            'repositories': etpConst['server_repositories'].copy(),
            'qa_langs': ["en_US", "C"],
            'default_repository_id': etpConst['defaultserverrepositoryid'],
            'base_repository_id': None,
            'packages_expiration_days': etpConst['packagesexpirationdays'],
            'database_file_format': etpConst['etpdatabasefileformat'],
            'disabled_eapis': set(),
            'exp_based_scope': etpConst['expiration_based_scope'],
            'nonfree_packages_dir_support': False, # disabled by default for now
            'sync_speed_limit': None,
            'rss': {
                'enabled': etpConst['rss-feed'],
                'name': etpConst['rss-name'],
                'base_url': etpConst['rss-base-url'],
                'website_url': etpConst['rss-website-url'],
                'editor': etpConst['rss-managing-editor'],
                'max_entries': etpConst['rss-max-entries'],
                'light_max_entries': etpConst['rss-light-max-entries'],
            },
        }

        fake_instance = self._helper.fake_default_repo

        if not os.access(etpConst['serverconf'], os.R_OK):
            return data

        with open(etpConst['serverconf'], "r") as server_f:
            serverconf = [x.strip() for x in server_f.readlines() if x.strip()]

        default_repo_changed = False

        for line in serverconf:

            split_line = line.split("|")
            split_line_len = len(split_line)

            # NOTE: remove this in future, supported for backward compat.
            if (line.find("officialserverrepositoryid|") != -1) and \
                (not line.startswith("#")) and (split_line_len == 2):

                # NOTE: added for backward and mixed compat.
                if default_repo_changed:
                    continue

                if not fake_instance:
                    data['default_repository_id'] = split_line[1].strip()

            elif (line.find("default-repository|") != -1) and \
                (not line.startswith("#")) and (split_line_len == 2):

                if not fake_instance:
                    data['default_repository_id'] = split_line[1].strip()
                default_repo_changed = True

            elif line.startswith("expiration-days|") and (split_line_len == 2):

                mydays = split_line[1].strip()
                try:
                    mydays = int(mydays)
                    data['packages_expiration_days'] = mydays
                except ValueError:
                    continue

            elif line.startswith("expiration-based-scope|") and \
                (split_line_len == 2):

                exp_opt = split_line[1].strip().lower()
                if exp_opt in ("enable", "enabled", "true", "1", "yes"):
                    data['exp_based_scope'] = True
                else:
                    data['exp_based_scope'] = False

            elif line.startswith("nonfree-packages-directory-support|") and \
                (split_line_len == 2):

                exp_opt = split_line[1].strip().lower()
                if exp_opt in ("enable", "enabled", "true", "1", "yes"):
                    data['nonfree_packages_dir_support'] = True
                else:
                    data['nonfree_packages_dir_support'] = False

            elif line.startswith("disabled-eapis|") and (split_line_len == 2):

                mydis = split_line[1].strip().split(",")
                try:
                    mydis = [int(x) for x in mydis]
                    mydis = set([x for x in mydis if x in (1, 2, 3,)])
                except ValueError:
                    continue
                if (len(mydis) < 3) and mydis:
                    data['disabled_eapis'] = mydis

            elif line.startswith("server-basic-languages|") and \
                (split_line_len == 2):
                data['qa_langs'] = split_line[1].strip().split()

            elif line.startswith("repository|") and (split_line_len in (5, 6)) \
                and (not fake_instance):

                repoid, repodata = \
                    ServerSystemSettingsPlugin.analyze_server_repo_string(line,
                        product = sys_set['repositories']['product'])
                if repoid in data['repositories']:
                    # just update mirrors
                    data['repositories'][repoid]['pkg_mirrors'].extend(
                        repodata['pkg_mirrors'])
                    data['repositories'][repoid]['repo_mirrors'].extend(
                        repodata['repo_mirrors'])
                else:
                    data['repositories'][repoid] = repodata.copy()

                # base_repository_id support
                if data['base_repository_id'] is None:
                    data['base_repository_id'] = repoid

            elif line.startswith("database-format|") and (split_line_len == 2):

                fmt = split_line[1]
                if fmt in etpConst['etpdatabasesupportedcformats']:
                    data['database_file_format'] = fmt

            elif line.startswith("syncspeedlimit|") and (split_line_len == 2):

                try:
                    speed_limit = int(split_line[1])
                except ValueError:
                    speed_limit = None
                data['sync_speed_limit'] = speed_limit

            elif line.startswith("rss-feed|") and (split_line_len == 2):

                feed = split_line[1]
                if feed in ("enable", "enabled", "true", "1"):
                    data['rss']['enabled'] = True
                elif feed in ("disable", "disabled", "false", "0", "no",):
                    data['rss']['enabled'] = False

            elif line.startswith("rss-name|") and (split_line_len == 2):

                feedname = line.split("rss-name|")[1].strip()
                data['rss']['name'] = feedname

            elif line.startswith("rss-base-url|") and (split_line_len == 2):

                data['rss']['base_url'] = line.split("rss-base-url|")[1].strip()
                if not data['rss']['base_url'][-1] == "/":
                    data['rss']['base_url'] += "/"

            elif line.startswith("rss-website-url|") and (split_line_len == 2):

                data['rss']['website_url'] = split_line[1].strip()

            elif line.startswith("managing-editor|") and (split_line_len == 2):

                data['rss']['editor'] = split_line[1].strip()

            elif line.startswith("max-rss-entries|") and (split_line_len == 2):

                try:
                    entries = int(split_line[1].strip())
                    data['rss']['max_entries'] = entries
                except (ValueError, IndexError,):
                    continue

            elif line.startswith("max-rss-light-entries|") and \
                (split_line_len == 2):

                try:
                    entries = int(split_line[1].strip())
                    data['rss']['light_max_entries'] = entries
                except (ValueError, IndexError,):
                    continue

        # add system database if community repository mode is enabled
        if self._helper.community_repo:
            data['repositories'][etpConst['clientserverrepoid']] = {}
            mydata = {}
            mydata['description'] = "Community Repositories System Database"
            mydata['pkg_mirrors'] = []
            mydata['repo_mirrors'] = []
            mydata['community'] = False
            data['repositories'][etpConst['clientserverrepoid']].update(mydata)
            # installed packages repository is now the base repository
            data['base_repository_id'] = etpConst['clientserverrepoid']

        # expand paths
        for repoid in data['repositories']:
            data['repositories'][repoid]['repo_basedir'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid
                            )
            data['repositories'][repoid]['packages_dir'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                etpConst['packagesrelativepath_basedir'],
                                etpConst['currentarch']
                            )
            data['repositories'][repoid]['packages_dir_nonfree'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                etpConst['packagesrelativepath_basedir_nonfree'],
                                etpConst['currentarch']
                            )
            data['repositories'][repoid]['packages_dir_restricted'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                etpConst['packagesrelativepath_basedir_restricted'],
                                etpConst['currentarch']
                            )
            data['repositories'][repoid]['store_dir'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                "store",
                                etpConst['currentarch']
                            )
            data['repositories'][repoid]['upload_basedir'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                "upload" # consider this a base dir
                            )
            data['repositories'][repoid]['database_dir'] = \
                os.path.join(   etpConst['entropyworkdir'],
                                "server",
                                repoid,
                                "database",
                                etpConst['currentarch']
                            )
            data['repositories'][repoid]['remote_repo_basedir'] = \
                os.path.join(   sys_set['repositories']['product'],
                                repoid
                            )
            data['repositories'][repoid]['database_relative_path'] = \
                os.path.join(   sys_set['repositories']['product'],
                                repoid,
                                "database",
                                etpConst['currentarch']
                            )

        # Support for shell variables
        shell_repoid = os.getenv('ETP_REPO')
        if shell_repoid:
            data['default_repository_id'] = shell_repoid

        expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
        if expiration_days:
            try:
                expiration_days = int(expiration_days)
                data['packages_expiration_days'] = expiration_days
            except ValueError:
                pass

        return data


class ServerFatscopeSystemSettingsPlugin(SystemSettingsPlugin):

    def repos_parser(self, sys_set):

        cached = getattr(self, '_repos_data', None)
        if cached is not None:
            return cached

        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']
        if not srv_parser_data['exp_based_scope']:
            return data

        # get expiration-based packages removal data from config files
        for repoid in srv_parser_data['repositories']:

            # filter out system repository if community repository
            # mode is enabled
            if repoid == etpConst['clientserverrepoid']:
                continue

            idpackages = set()
            exp_fp = self._helper._get_local_exp_based_pkgs_rm_whitelist_file(
                repo = repoid)
            dbconn = self._helper.open_server_repository(
                just_reading = True, repo = repoid)

            if os.access(exp_fp, os.R_OK) and os.path.isfile(exp_fp):
                pkgs = entropy.tools.generic_file_content_parser(exp_fp)
                if '*' in pkgs: # wildcard support
                    idpackages.add(-1)
                else:
                    for pkg in pkgs:
                        idpackage, rc_match = dbconn.atomMatch(pkg)
                        if rc_match:
                            continue
                        idpackages.add(idpackage)

            data[repoid] = idpackages

        self._repos_data = data
        return data

class ServerFakeClientSystemSettingsPlugin(SystemSettingsPlugin):

    def fake_cli_parser(self, sys_set):
        """
        This is just fake, doesn't bring any new metadata but just tweak
        Entropy client ones.
        """
        data = {}
        srv_plug_id = etpConst['system_settings_plugins_ids']['server_plugin']
        # if support is not enabled, don't waste time scanning files
        srv_parser_data = sys_set[srv_plug_id]['server']

        # now setup fake Entropy Client repositories, so that Entropy Server
        # can use Entropy Client interfaces transparently
        srv_repodata = srv_parser_data['repositories']
        cli_repodata = sys_set['repositories']
        # remove unavailable server repos in client metadata first
        cli_repodata['available'].clear()

        for repoid, repo_data in srv_repodata.items():
            xxx, my_data = sys_set._analyze_client_repo_string(
                "repository|%s|%s|http://--fake--|http://--fake--" % (
                    repoid, repo_data['description'],))
            my_data['repoid'] = repoid
            my_data['dbpath'] = self._helper._get_local_database_dir(
                repo = repoid)
            my_data['dbrevision'] = self._helper.get_local_repository_revision(
                repo = repoid)
            cli_repodata['available'][repoid] = my_data

        cli_repodata['default_repository'] = \
            srv_parser_data['default_repository_id']
        del cli_repodata['order'][:]
        if srv_parser_data['base_repository_id'] is not None:
            cli_repodata['order'].append(srv_parser_data['base_repository_id'])
        for repoid in sorted(srv_repodata):
            if repoid not in cli_repodata['order']:
                cli_repodata['order'].append(repoid)

        return data

class ServerQAInterfacePlugin(QAInterfacePlugin):

    def __init__(self, entropy_server_instance):
        self._server = entropy_server_instance

    def __check_package_using_spm(self, package_path):

        spm_class = get_spm_class()
        spm_rc, spm_msg = spm_class.execute_qa_tests(package_path)

        if spm_rc == 0:
            return True
        sys.stderr.write("QA Error: " + spm_msg + "\n")
        sys.stderr.flush()
        return False

    def __extract_edb_analyze_metadata(self, package_path):
        tmp_fd, tmp_f = tempfile.mkstemp(prefix = 'entropy.server')
        dbc = None
        try:
            found_edb = entropy.tools.dump_entropy_metadata(package_path, tmp_f)
            if not found_edb:
                return False
            dbc = self._server._open_temp_repository("test", temp_file = tmp_f)
            for package_id in dbc.listAllPackageIds():
                keywords = dbc.retrieveKeywords(package_id)
                if not keywords:
                    atom = dbc.retrieveAtom(package_id)
                    # bit PHAT warning !!
                    self._server.output(darkred("~"*40), level = "warning")
                    self._server.output("[%s, %s] %s" % (
                         brown(os.path.basename(package_path)), teal(atom),
                         purple(_("package has no keyword set, it will be masked !"))),
                        level = "warning", header = darkred(" !!! "))
                    self._server.output(darkred("~"*40), level = "warning")
                    time.sleep(10)
        finally:
            if dbc is not None:
                dbc.closeDB()
            os.close(tmp_fd)
            os.remove(tmp_f)

        return True

    def get_tests(self):
        return [self.__check_package_using_spm,
            self.__extract_edb_analyze_metadata]

    def get_id(self):
        return SERVER_QA_PLUGIN


class ServerSettingsMixin:

    def _get_branch_from_download_relative_uri(self, db_download_uri):
        return db_download_uri.split("/")[2]

    def _swap_branch_in_download_relative_uri(self, new_branch,
        db_download_uri):
        cur_branch = self._get_branch_from_download_relative_uri(
            db_download_uri)
        return db_download_uri.replace("/%s/" % (cur_branch,),
            "/%s/" % (new_branch,))

    def _get_basedir_pkg_listing(self, base_dir, repo = None, branch = None):

        pkgs_dir_types = self._get_pkg_dir_names()
        basedir_raw_content = []
        entropy.tools.recursive_directory_relative_listing(
            basedir_raw_content, base_dir)

        pkg_ext = etpConst['packagesext']
        pkg_list = [x for x in basedir_raw_content if \
            x.split(os.path.sep)[0] in pkgs_dir_types and \
                x.endswith(pkg_ext)]

        if branch is not None:
            branch_extractor = \
                self._get_branch_from_download_relative_uri
            pkg_list = [x for x in pkg_list if branch_extractor(x) == branch]

        return pkg_list

    def _get_pkg_dir_names(self):
        return [etpConst['packagesrelativepath_basedir'],
            etpConst['packagesrelativepath_basedir_nonfree'],
            etpConst['packagesrelativepath_basedir_restricted']]

    def _get_remote_database_relative_path(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['database_relative_path']

    def _get_local_database_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasefile'])

    def _get_local_store_directory(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['store_dir']

    def _get_local_upload_directory(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['upload_basedir']

    def _get_local_repository_base_directory(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['repo_basedir']

    def _get_local_database_taint_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasetaintfile'])

    def _get_local_database_revision_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaserevisionfile'])

    def _get_local_database_timestamp_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasetimestampfile'])

    def _get_local_database_ca_cert_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasecacertfile'])

    def _get_local_database_server_cert_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaseservercertfile'])

    def _get_local_database_mask_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasemaskfile'])

    def _get_local_database_system_mask_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasesytemmaskfile'])

    def _get_local_database_confl_tagged_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaseconflictingtaggedfile'])

    def _get_local_database_licensewhitelist_file(self, repo = None,
        branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaselicwhitelistfile'])

    def _get_local_database_dep_rewrite_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasedeprewritefile'])

    def _get_local_database_rss_file(self, repo = None, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            srv_set['rss']['name'])

    def _get_local_database_rsslight_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['rss-light-name'])

    def _get_local_database_notice_board_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['rss-notice-board'])

    def _get_local_database_treeupdates_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaseupdatefile'])

    def _get_local_database_compressed_metafiles_file(self, repo = None,
        branch = None):

        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasemetafilesfile'])

    def _get_local_database_metafiles_not_found_file(self, repo = None,
        branch = None):

        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasemetafilesnotfound'])

    def _get_local_database_gpg_signature_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasegpgfile'])

    def _get_local_exp_based_pkgs_rm_whitelist_file(self, repo = None,
        branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaseexpbasedpkgsrm'])

    def _get_local_pkglist_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasepkglist'])

    def _get_local_database_sets_dir(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['confsetsdirname'])

    def _get_local_post_branch_mig_script(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etp_post_branch_hop_script'])

    def _get_local_post_branch_upg_script(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etp_post_branch_upgrade_script'])

    def _get_local_post_repo_update_script(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etp_post_repo_update_script'])

    def _get_local_critical_updates_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasecriticalfile'])

    def _get_local_restricted_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabaserestrictedfile'])

    def _get_local_database_keywords_file(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo, branch),
            etpConst['etpdatabasekeywordsfile'])

    def _get_local_database_dir(self, repo = None, branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(srv_set['repositories'][repo]['database_dir'],
            branch)

    def _get_missing_dependencies_blacklist_file(self, repo = None,
        branch = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        if branch is None:
            branch = self._settings['repositories']['branch']
        return os.path.join(srv_set['repositories'][repo]['database_dir'],
            branch, etpConst['etpdatabasemissingdepsblfile'])

    def _get_database_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo),
            etpConst['etpdatabaselockfile'])

    def _get_database_download_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        return os.path.join(self._get_local_database_dir(repo),
            etpConst['etpdatabasedownloadlockfile'])

    def _create_local_database_download_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        lock_file = self._get_database_download_lockfile(repo)
        f_lock = open(lock_file, "w")
        f_lock.write("download locked")
        f_lock.flush()
        f_lock.close()

    def _create_local_database_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        lock_file = self._get_database_lockfile(repo)
        f_lock = open(lock_file, "w")
        f_lock.write("database locked")
        f_lock.flush()
        f_lock.close()

    def _remove_local_database_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        lock_file = self._get_database_lockfile(repo)
        if os.path.isfile(lock_file):
            os.remove(lock_file)

    def _remove_local_database_download_lockfile(self, repo = None):
        if repo is None:
            repo = self.default_repository
        lock_file = self._get_database_download_lockfile(repo)
        if os.path.isfile(lock_file):
            os.remove(lock_file)

    def complete_remote_package_relative_path(self, pkg_rel_url, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return os.path.join(
            srv_set['repositories'][repo]['remote_repo_basedir'], pkg_rel_url)

    def complete_local_upload_package_path(self, pkg_rel_url, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return os.path.join(srv_set['repositories'][repo]['upload_basedir'],
            pkg_rel_url)

    def complete_local_package_path(self, pkg_rel_url, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return os.path.join(srv_set['repositories'][repo]['repo_basedir'],
            pkg_rel_url)

    def get_remote_repository_mirrors(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['repo_mirrors'][:]

    def get_remote_packages_mirrors(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if repo is None:
            repo = self.default_repository
        return srv_set['repositories'][repo]['pkg_mirrors'][:]

    def get_local_repository_revision(self, repo = None):

        if repo is None:
            repo = self.default_repository

        dbrev_file = self._get_local_database_revision_file(repo)
        if not os.path.isfile(dbrev_file):
            return 0

        f_rev = open(dbrev_file)
        rev = f_rev.readline().strip()
        f_rev.close()
        try:
            rev = int(rev)
        except ValueError:
            self.output(
                "[repo:%s] %s: %s - %s" % (
                        darkgreen(repo),
                        blue(_("invalid database revision")),
                        bold(rev),
                        blue(_("defaulting to 0")),
                    ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            rev = 0
        return rev

    def get_remote_repository_revision(self, repo = None):

        if repo is None:
            repo = self.default_repository

        remote_status =  self.Mirrors.get_remote_repositories_status(repo)
        if not [x for x in remote_status if x[1]]:
            remote_revision = 0
        else:
            remote_revision = max([x[1] for x in remote_status])

        return remote_revision

    def repositories(self):
        """
        Return a list of available Entropy Server repositories.

        @return: list of available Entropy Server repositories
        @rtype: list
        """
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return sorted(srv_set['repositories'])


class ServerLoadersMixin:

    def QA(self):
        """
        Get Entropy QA Interface instance.

        @return: Entropy QA Interface instance
        @rtype: entropy.qa.QAInterface instance
        """
        qa_plugin = ServerQAInterfacePlugin(self)
        qa = _Client.QA(self)
        qa.add_plugin(qa_plugin)
        return qa

    def Spm(self):
        """
        Get Source Package Manager interface instance.

        @return: Source Package Manager interface instance
        @rtype: entropy.spm.plugins.skel.SpmPlugin based instance
        """
        return get_spm(self)

    def Spm_class(self):
        """
        Get Source Package Manager interface class.
        """
        return get_spm_class()

    def Transceiver(self, uri):
        """
        Get EntropyTransceiver interface instance.

        @param uri: EntropyTransceiver URI
        @type uri: string
        @return: EntropyTransceiver instance
        @rtype: entropy.transceivers.EntropyTransceiver
        """
        txc = EntropyTransceiver(uri)
        txc.set_output_interface(self)
        return txc


class ServerPackageDepsMixin:

    def sets_available(self, *args, **kwargs):
        sets = _Client.Sets(self)
        return sets.available(*args, **kwargs)

    def sets_search(self, *args, **kwargs):
        sets = _Client.Sets(self)
        return sets.search(*args, **kwargs)

    def sets_match(self, *args, **kwargs):
        sets = _Client.Sets(self)
        return sets.match(*args, **kwargs)

    def atom_match(self, *args, **kwargs):
        # disable masked packages for server-side repos
        kwargs['mask_filter'] = False
        return _Client.atom_match(self, *args, **kwargs)

    def match_packages(self, packages, repo = None):

        dbconn = self.open_server_repository(read_only = True,
            no_upload = True, repo = repo)
        if ("world" in packages) or not packages:
            return dbconn.listAllPackageIds(), True
        else:
            idpackages = set()
            for package in packages:
                matches = dbconn.atomMatch(package, multiMatch = True)
                if matches[1] == 0:
                    idpackages |= matches[0]
                else:
                    mytxt = "%s: %s: %s" % (
                        red(_("Attention")),
                        blue(_("cannot match")),
                        bold(package),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "warning",
                        header = darkred(" !!! ")
                    )
            return idpackages, False

    def mask_packages(self, packages, repo = None):
        """
        Mask given package dependencies for given repository, if any (otherwise
        use default one).

        @param packages: list of package dependency strings
        @type packages: list
        @keyword repo: repository identifier
        @type repo: string
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_database_mask_file(repo = repo)
        current_packages = []

        if os.path.isfile(mask_file) and os.access(mask_file, os.R_OK):
            current_packages = entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False)
        # this is untrusted input, it's fine because that config file is
        # untrusted too
        current_packages.extend(packages)

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        with open(mask_file_tmp, "w") as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")
            mask_f.flush()
        os.rename(mask_file_tmp, mask_file)

        return True

    def unmask_packages(self, packages, repo = None):
        """
        Unmask given package dependencies for given repository, if any (otherwise
        use default one).

        @param packages: list of package dependency strings
        @type packages: list
        @keyword repo: repository identifier
        @type repo: string
        @return: mask status, True if ok, False if not
        @rtype: bool
        """
        mask_file = self._get_local_database_mask_file(repo = repo)
        current_packages = []

        if os.path.isfile(mask_file) and os.access(mask_file, os.R_OK):
            current_packages = entropy.tools.generic_file_content_parser(
                mask_file, comment_tag = "##", filter_comments = False)

        def mask_filter(package):
            if package.startswith("#"):
                # comment, always valid
                return True
            in_file_pkg_match = self.atom_match(package)
            for req_package in packages:
                if package == req_package:
                    # of course remove if it's equal
                    return False
                req_package_match = self.atom_match(req_package)
                if req_package_match == in_file_pkg_match:
                    # drop it, they point to the same package match
                    return False
            return True

        current_packages = list(filter(mask_filter, current_packages))

        mask_file_tmp = mask_file + ".mask_packages_tmp"
        with open(mask_file_tmp, "w") as mask_f:
            for package in current_packages:
                mask_f.write(package + "\n")
            mask_f.flush()
        os.rename(mask_file_tmp, mask_file)

        return True

class ServerPackagesHandlingMixin:

    def initialize_server_repository(self, repo = None, show_warnings = True):

        if repo is None:
            repo = self.default_repository

        self.close_repositories()

        mytxt = red("%s ...") % (_("Initializing Entropy database"),)
        self.output(
            mytxt, importance = 1,
            level = "info", header = darkgreen(" * "),
            back = True
        )

        if os.path.isfile(self._get_local_database_file(repo)):

            # test out
            if show_warnings:
                dbconn = self.open_server_repository(read_only = True,
                    no_upload = True, repo = repo, warnings = show_warnings)
                self.close_repository(dbconn)

                mytxt = "%s: %s: %s" % (
                    bold(_("WARNING")),
                    red(_("database already exists")),
                    self._get_local_database_file(repo),
                )
                self.output(
                    mytxt,
                    importance = 1,
                    level = "warning",
                    header = darkred(" !!! ")
                )

                rc_question = self.ask_question(_("Do you want to continue ?"))
                if rc_question == _("No"):
                    return

            os.remove(self._get_local_database_file(repo))

        # initialize
        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo, is_new = True)
        dbconn.initializeRepository()

        dbconn.commitChanges()
        self.close_repositories()

        return 0

    def tag_packages(self, package_tag, idpackages, repo = None, ask = True):

        # check package_tag "no spaces"

        try:
            package_tag = str(package_tag)
            if " " in package_tag:
                raise ValueError
        except (UnicodeDecodeError, UnicodeEncodeError, ValueError,):
            self.output(
                "%s: %s" % (
                    blue(_("Invalid tag specified")),
                    package_tag,
                ),
                importance = 1, level = "error", header = darkred(" !! ")
            )
            return 1, package_tag

        if repo is None:
            repo = self.default_repository

        # sanity check
        invalid_atoms = []
        dbconn = self.open_server_repository(read_only = True,
            no_upload = True, repo = repo)
        for idpackage in idpackages:
            ver_tag = dbconn.retrieveTag(idpackage)
            if ver_tag:
                invalid_atoms.append(dbconn.retrieveAtom(idpackage))

        if invalid_atoms:
            self.output(
                "%s: %s" % (
                    blue(_("Packages already tagged, action aborted")),
                    ', '.join([darkred(str(x)) for x in invalid_atoms]),
                ),
                importance = 1, level = "error", header = darkred(" !! ")
            )
            return 2, invalid_atoms

        matches = [(x, repo) for x in idpackages]
        status = 0
        data = self.move_packages(
            matches, to_repo = repo, from_repo = repo, ask = ask,
            do_copy = True, new_tag = package_tag
        )
        return status, data

    def flushback_packages(self, from_branches, repo = None, ask = True):
        """
        When creating a new branch, for space reasons, packages are not
        moved to a new location. This works fine until old branch is removed.
        To avoid inconsistences, before deciding to do that, all the packages
        in the old branch should be flushed back to the the currently configured
        branch.

        @param from_branches -- list of branches to move packages from
        @type from_branches -- list
        @param repo -- repository to work on
        @type repo -- str
        @param ask -- user interactivity
        @type ask -- bool

        @return status
        """

        status = True
        if repo is None:
            repo = self.default_repository
        branch = self._settings['repositories']['branch']

        if branch in from_branches:
            from_branches = [x for x in from_branches if x != branch]

        self.output(
            "[%s=>%s|%s] %s" % (
                darkgreen(', '.join(from_branches)),
                darkred(branch),
                brown(repo),
                blue(_("flushing back selected packages from branches")),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(read_only = True,
            no_upload = True, repo = repo)

        idpackage_map = dict(((x, [],) for x in from_branches))
        idpackages = dbconn.listAllPackageIds(order_by = 'atom')
        for idpackage in idpackages:
            download_url = dbconn.retrieveDownloadURL(idpackage)
            url_br = self._get_branch_from_download_relative_uri(
                download_url)
            if url_br in from_branches:
                idpackage_map[url_br].append(idpackage)

        mapped_branches = [x for x in idpackage_map if idpackage_map[x]]
        if not mapped_branches:
            self.output(
                "[%s=>%s|%s] %s !" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repo),
                    blue(_("nothing to do")),
                ),
                importance = 0,
                level = "warning",
                header = blue(" @@ ")
            )
            return status


        all_fine = True
        tmp_down_dir = tempfile.mkdtemp(prefix = "entropy.server")

        download_queue = {}
        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)

        def generate_queue(branch, repo, from_branch, down_q, idpackage_map):

            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repo),
                    brown(_("these are the packages that will be flushed")),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )


            for idpackage in idpackage_map[from_branch]:
                atom = dbconn.retrieveAtom(idpackage)
                self.output(
                    "[%s=>%s|%s] %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repo),
                        purple(atom),
                    ),
                    importance = 0,
                    level = "info",
                    header = blue("  # ")
                )
                pkg_fp = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
                pkg_fp = os.path.join(tmp_down_dir, pkg_fp)
                down_q.append((pkg_fp, idpackage,))


        for from_branch in sorted(mapped_branches):

            download_queue[from_branch] = []
            all_fine = False
            generate_queue(branch, repo, from_branch,
                download_queue[from_branch], idpackage_map)

            if ask:
                rc_question = self.ask_question(
                    _("Would you like to continue ?"))
                if rc_question == _("No"):
                    continue

            for uri in self.get_remote_packages_mirrors(repo):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                queue_map = {}

                for pkg_fp, idpackage in download_queue[from_branch]:
                    down_url = dbconn.retrieveDownloadURL(idpackage)
                    down_rel = self.complete_remote_package_relative_path(
                        down_url, repo = repo)
                    down_rel_dir = os.path.dirname(down_rel)
                    obj = queue_map.setdefault(down_rel_dir, [])
                    obj.append(pkg_fp)

                errors = False
                m_fine_uris = set()
                m_broken_uris = set()

                for down_rel_dir, downloader_queue in queue_map.items():

                    downloader = self.Mirrors.TransceiverServerHandler(
                        self,
                        [uri],
                        downloader_queue,
                        critical_files = downloader_queue,
                        txc_basedir = down_rel_dir,
                        local_basedir = tmp_down_dir,
                        download = True,
                        repo = repo
                    )
                    xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
                    if xerrors:
                        errors = True
                    m_fine_uris.update(xm_fine_uris)
                    m_broken_uris.update(xm_broken_uris)

                if not errors:
                    for downloaded_path, idpackage in \
                        download_queue[from_branch]:

                        self.output(
                            "[%s=>%s|%s|%s] %s: %s" % (
                                darkgreen(from_branch),
                                darkred(branch),
                                brown(repo),
                                dbconn.retrieveAtom(idpackage),
                                blue(_("checking package hash")),
                                darkgreen(os.path.basename(downloaded_path)),
                            ),
                            importance = 0,
                            level = "info",
                            header = brown("   "),
                            back = True
                        )

                        md5hash = entropy.tools.md5sum(downloaded_path)
                        db_md5hash = dbconn.retrieveDigest(idpackage)
                        if md5hash != db_md5hash:
                            errors = True
                            self.output(
                                "[%s=>%s|%s|%s] %s: %s" % (
                                    darkgreen(from_branch),
                                    darkred(branch),
                                    brown(repo),
                                    dbconn.retrieveAtom(idpackage),
                                    blue(_("hash does not match for")),
                                    darkgreen(os.path.basename(downloaded_path)),
                                ),
                                importance = 0,
                                level = "error",
                                header = brown("   ")
                            )
                            continue

                if errors:
                    reason = _("wrong md5")
                    if m_broken_uris:
                        my_broken_uris = [
                        (EntropyTransceiver.get_uri_name(x), y,) \
                            for x, y in m_broken_uris]
                        reason = my_broken_uris[0][1]

                    self.output(
                        "[%s=>%s|%s] %s, %s: %s" % (
                            darkgreen(from_branch),
                            darkred(branch),
                            brown(repo),
                            blue(_("download errors")),
                            blue(_("reason")),
                            reason,
                        ),
                        importance = 1,
                        level = "error",
                        header = darkred(" !!! ")
                    )
                    # continuing if possible
                    continue

                all_fine = True

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repo),
                        blue(_("download completed successfully")),
                        darkgreen(crippled_uri),
                    ),
                    importance = 1,
                    level = "info",
                    header = darkgreen(" * ")
                )

        if not all_fine:
            self.output(
                "[%s=>%s|%s] %s" % (
                    darkgreen(', '.join(from_branches)),
                    darkred(branch),
                    brown(repo),
                    blue(_("error downloading packages from mirrors")),
                ),
                importance = 2,
                level = "error",
                header = darkred(" !!! ")
            )
            return False

        for from_branch in sorted(mapped_branches):

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(from_branch),
                    darkred(branch),
                    brown(repo),
                    blue(_("working on branch")),
                    darkgreen(from_branch),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )

            down_queue = download_queue[from_branch]
            for package_path, idpackage in down_queue:

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repo),
                        blue(_("updating package")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   "),
                    back = True
                )

                # build new download url
                download_url = dbconn.retrieveDownloadURL(idpackage)
                download_url = \
                    self._swap_branch_in_download_relative_uri(
                        branch, download_url)

                # move files to upload
                new_package_path = self.complete_local_upload_package_path(
                    download_url, repo = repo)
                self._ensure_dir_path(os.path.dirname(new_package_path))

                try:
                    os.rename(package_path, new_package_path)
                except OSError:
                    shutil.move(package_path, new_package_path)

                # create md5 checksum
                entropy.tools.create_md5_file(new_package_path)

                # update database
                dbconn.setDownloadURL(idpackage, download_url)
                dbconn.switchBranch(idpackage, branch)
                dbconn.commitChanges()

                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(from_branch),
                        darkred(branch),
                        brown(repo),
                        blue(_("package flushed")),
                        darkgreen(os.path.basename(package_path)),
                    ),
                    importance = 1,
                    level = "info",
                    header = brown("   ")
                )

        try:
            os.rmdir(tmp_down_dir)
        except OSError:
            pass

        return True

    def move_packages(self, matches, to_repo, from_repo = None, ask = True,
        do_copy = False, new_tag = None, pull_deps = False):

        if from_repo is None:
            from_repo = self.default_repository
        switched = set()

        my_matches = list(matches)

        # avoid setting __default__ as default server repo
        if etpConst['clientserverrepoid'] in (to_repo, from_repo):
            self.output(
                "%s: %s" % (
                    blue(_("Cannot touch system database")),
                    red(etpConst['clientserverrepoid']),
                ),
                importance = 2, level = "warning", header = darkred(" @@ ")
            )
            return switched

        if not my_matches and from_repo:
            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = from_repo)
            my_matches = set( \
                [(x, from_repo) for x in \
                    dbconn.listAllPackageIds()]
            )

        mytxt = _("Preparing to move selected packages to")
        if do_copy:
            mytxt = _("Preparing to copy selected packages to")
        self.output(
            "%s %s:" % (
                blue(mytxt),
                red(to_repo),
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        self.output(
            "%s: %s" % (
                bold(_("Note")),
                red(_("all old packages with conflicting scope will be " \
                    "removed from destination repo unless injected")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        new_tag_string = ''
        if new_tag != None:
            new_tag_string = "[%s: %s]" % (darkgreen(_("new tag")),
                brown(new_tag),)

        # open both repos here to make sure it's all fine with them
        dbconn = self.open_server_repository(read_only = True,
            no_upload = True, repo = from_repo)
        todbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = to_repo)

        my_qa = self.QA()
        branch = self._settings['repositories']['branch']
        pull_deps_matches = []
        for idpackage, repo in my_matches:
            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = repo)
            self.output(
                "[%s=>%s|%s] %s " % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(branch),
                        blue(dbconn.retrieveAtom(idpackage)),
                ) + new_tag_string,
                importance = 0,
                level = "info",
                header = brown("    # ")
            )

            # check if there are pkgs that are going to be overwritten
            # and warn user about that.
            from_name, from_category, from_slot, from_injected = \
                dbconn.retrieveName(idpackage), \
                dbconn.retrieveCategory(idpackage), \
                dbconn.retrieveSlot(idpackage), \
                dbconn.isInjected(idpackage)
            to_rm_idpackages = todbconn.getPackagesToRemove(from_name,
                from_category, from_slot, from_injected)
            for to_rm_idpackage in to_rm_idpackages:
                self.output(
                    "    [=>%s|%s] %s" % (
                            darkred(to_repo),
                            bold(_("remove")),
                            blue(todbconn.retrieveAtom(to_rm_idpackage)),
                    ),
                    importance = 0,
                    level = "info",
                    header = purple("    # ")
                )

            # do we want to pull in also package dependencies?
            if pull_deps:
                dep_matches = [(x, repo) for x in \
                    my_qa.get_deep_dependency_list(dbconn, idpackage)]
                revdep_matches = self.get_reverse_queue(dep_matches,
                    system_packages = False)
                dep_matches += [x for x in revdep_matches if x not in \
                    dep_matches]

                for dep_idpackage, dep_repo in dep_matches:

                    my_dep_match = (dep_idpackage, dep_repo,)
                    if my_dep_match in pull_deps_matches:
                        continue
                    if my_dep_match in my_matches:
                        continue

                    pull_deps_matches.append(my_dep_match)
                    dep_dbconn = self.open_server_repository(read_only = True,
                        no_upload = True, repo = dep_repo)
                    dep_atom = dep_dbconn.retrieveAtom(dep_idpackage)
                    if my_dep_match in revdep_matches:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("reverse dependency")),
                                teal(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )
                    else:
                        self.output(
                            "[%s|%s] %s" % (
                                brown(branch),
                                blue(_("dependency")),
                                purple(dep_atom),
                            ),
                            importance = 0,
                            level = "info",
                            header = purple("    >> ")
                        )

        if pull_deps:
            # put deps first!
            my_matches = pull_deps_matches + [x for x in my_matches if x not \
                in pull_deps_matches]

        if ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                return switched

        for idpackage, repo in my_matches:

            dbconn = self.open_server_repository(read_only = False,
                no_upload = True, repo = repo)

            match_atom = dbconn.retrieveAtom(idpackage)
            package_rel_path = dbconn.retrieveDownloadURL(idpackage)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repo),
                    brown(branch),
                    blue(_("switching")),
                    darkgreen(match_atom),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            # move binary file
            from_file = self.complete_local_package_path(package_rel_path,
                repo = repo)
            if not os.path.isfile(from_file):
                from_file = self.complete_local_upload_package_path(
                    package_rel_path, repo = repo)
            if not os.path.isfile(from_file):
                self.output(
                    "[%s=>%s|%s] %s: %s -> %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(branch),
                        bold(_("cannot switch, package not found, skipping")),
                        darkgreen(match_atom),
                        red(from_file),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred(" !!! ")
                )
                continue

            # we need to ask SpmPlugin to re-extract metadata from pkg file
            # and grab the new "download" metadatum value using our
            # license check callback. It has to be done here because
            # we need the new path.

            def _package_injector_check_license(pkg_data):
                licenses = pkg_data['license'].split()
                return self._is_pkg_free(licenses, repo = to_repo)

            def _package_injector_check_restricted(pkg_data):
                pkgatom = entropy.tools.create_package_atom_string(
                    pkg_data['category'], pkg_data['name'], pkg_data['version'],
                    pkg_data['versiontag'])
                return self._is_pkg_restricted(pkgatom, pkg_data['slot'],
                    repo = to_repo)

            # check if pkg is restricted
            # and check if pkg is free, we must do this step in any case
            # NOTE: it sucks!
            tmp_data = self.Spm().extract_package_metadata(from_file,
                license_callback = _package_injector_check_license,
                restricted_callback = _package_injector_check_restricted)
            # NOTE: since ~0.tbz2 << revision is lost, we need to trick
            # the logic.
            updated_package_rel_path = os.path.join(
                os.path.dirname(tmp_data['download']),
                os.path.basename(package_rel_path))
            del tmp_data

            to_file = self.complete_local_upload_package_path(
                updated_package_rel_path, repo = to_repo)

            if new_tag != None:

                match_category = dbconn.retrieveCategory(idpackage)
                match_name = dbconn.retrieveName(idpackage)
                match_version = dbconn.retrieveVersion(idpackage)
                tagged_package_filename = \
                    entropy.tools.create_package_filename(
                        match_category, match_name, match_version, new_tag)

                to_file = self.complete_local_upload_package_path(
                    updated_package_rel_path, repo = to_repo)
                # directly move to correct place, tag changed, so file name
                to_file = os.path.join(os.path.dirname(to_file),
                    tagged_package_filename)

            self._ensure_dir_path(os.path.dirname(to_file))

            copy_data = [
                (from_file, to_file,),
                (from_file + etpConst['packagesmd5fileext'],
                    to_file + etpConst['packagesmd5fileext'],),
                (from_file + etpConst['packagesexpirationfileext'],
                    to_file + etpConst['packagesexpirationfileext'],)
            ]

            for from_item, to_item in copy_data:
                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(branch),
                        blue(_("moving file")),
                        darkgreen(os.path.basename(from_item)),
                    ),
                    importance = 0,
                    level = "info",
                    header = red(" @@ "),
                    back = True
                )
                if os.path.isfile(from_item):
                    shutil.copy2(from_item, to_item)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repo),
                    brown(branch),
                    blue(_("loading data from source database")),
                    darkgreen(repo),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            # install package into destination db
            data = dbconn.getPackageData(idpackage)
            if new_tag != None:
                data['versiontag'] = new_tag

            # need to set back data['download'], because pkg path might got
            # changed, due to license re-validation
            data['download'] = updated_package_rel_path

            # GPG
            # before inserting new pkg, drop GPG signature and re-sign
            old_gpg = copy.copy(data['signatures']['gpg'])
            data['signatures']['gpg'] = None
            try:
                repo_sec = RepositorySecurity()
            except RepositorySecurity.GPGError as err:
                if old_gpg:
                    self.output(
                        "[repo:%s] %s %s: %s." % (
                            darkgreen(to_repo),
                            darkred(_("GPG key was available in")),
                            bold(from_repo),
                            err,
                        ),
                        importance = 1,
                        level = "warning",
                        header = bold(" !!! ")
                    )
                repo_sec = None

            if repo_sec is not None:
                data['signatures']['gpg'] = self._get_gpg_signature(repo_sec,
                    to_repo, to_file)

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repo),
                    brown(branch),
                    blue(_("injecting data to destination database")),
                    darkgreen(to_repo),
                ),
                importance = 0,
                level = "info",
                header = red(" @@ "),
                back = True
            )
            new_idpackage, new_revision, new_data = todbconn.handlePackage(data)
            del data
            todbconn.commitChanges()

            if not do_copy:
                self.output(
                    "[%s=>%s|%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(to_repo),
                        brown(branch),
                        blue(_("removing entry from source database")),
                        darkgreen(repo),
                    ),
                    importance = 0,
                    level = "info",
                    header = red(" @@ "),
                    back = True
                )

                # remove package from old db
                dbconn.removePackage(idpackage)
                dbconn.commitChanges()

            self.output(
                "[%s=>%s|%s] %s: %s" % (
                    darkgreen(repo),
                    darkred(to_repo),
                    brown(branch),
                    blue(_("successfully handled atom")),
                    darkgreen(match_atom),
                ),
                importance = 0,
                level = "info",
                header = blue(" @@ ")
            )
            switched.add((idpackage, repo,))

        todbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = to_repo)
        todbconn.clean()

        # just run this to make dev aware
        self.dependencies_test(to_repo)

        return switched

    def _inject_database_into_packages(self, injection_data, repo = None):

        if repo is None:
            repo = self.default_repository

        # now inject metadata into tbz2 packages
        self.output(
            "[repo:%s] %s:" % (
                darkgreen(repo),
                blue(_("Injecting entropy metadata into built packages")),
            ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output(
                "[repo:%s] %s: %s" % (
                    darkgreen(repo),
                    blue(_("JFYI, GPG infrastructure failed to load")),
                    err,
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            repo_sec = None # gnupg not found, perhaps report it

        for idpackage, package_path in injection_data:
            self.output(
                "[repo:%s|%s] %s: %s" % (
                    darkgreen(repo),
                    brown(str(idpackage)),
                    blue(_("injecting entropy metadata")),
                    darkgreen(os.path.basename(package_path)),
                ),
                importance = 1,
                level = "info",
                header = blue(" @@ "),
                back = True
            )
            data = dbconn.getPackageData(idpackage)
            treeupdates_actions = dbconn.listAllTreeUpdatesActions()
            self._inject_entropy_database_into_package(
                package_path, data, treeupdates_actions)

            # GPG-sign package if GPG signature is set
            gpg_sign = None
            if repo_sec is not None:
                gpg_sign = self._get_gpg_signature(repo_sec, repo,
                    package_path)

            digest = entropy.tools.md5sum(package_path)
            # update digest
            dbconn.setDigest(idpackage, digest)
            # update signatures
            signatures = data['signatures'].copy()
            for hash_key in sorted(signatures):
                if hash_key == "gpg": # gpg already created
                    continue
                hash_func = getattr(entropy.tools, hash_key)
                signatures[hash_key] = hash_func(package_path)
            dbconn.setSignatures(idpackage, signatures['sha1'],
                signatures['sha256'], signatures['sha512'],
                gpg_sign)
            entropy.tools.create_md5_file(package_path)
            const_setup_file(package_path, etpConst['entropygid'], 0o664)
            self.output(
                "[repo:%s|%s] %s: %s" % (
                    darkgreen(repo),
                    brown(str(idpackage)),
                    blue(_("injection complete")),
                    darkgreen(os.path.basename(package_path)),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            dbconn.commitChanges()

    def remove_packages(self, idpackages, repo = None):

        if repo is None:
            repo = self.default_repository

        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)
        for idpackage in idpackages:
            atom = dbconn.retrieveAtom(idpackage)
            self.output(
                "[repo:%s] %s: %s" % (
                    darkgreen(repo),
                    blue(_("removing package")),
                    darkgreen(atom),
                ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            dbconn.removePackage(idpackage)
        self.close_repository(dbconn)
        self.output(
            "[repo:%s] %s" % (
                darkgreen(repo),
                blue(_("removal complete")),
            ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )

    def verify_remote_packages(self, packages, ask = True, repo = None):

        if repo is None:
            repo = self.default_repository

        self.output(
            "[%s] %s:" % (
                red("remote"),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ ")
        )

        idpackages, world = self.match_packages(packages)
        dbconn = self.open_server_repository(read_only = True, no_upload = True,
            repo = repo)
        branch = self._settings['repositories']['branch']

        if world:
            self.output(
                blue(
                    _("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )
        else:
            mytxt = red("%s:") % (
                _("This is the list of the packages that would be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "    "
            )
            for idpackage in idpackages:
                pkgatom = dbconn.retrieveAtom(idpackage)
                down_url = dbconn.retrieveDownloadURL(idpackage)
                pkgfile = os.path.basename(down_url)
                self.output(
                    red(pkgatom) + " -> " + bold(os.path.join(branch, pkgfile)),
                    importance = 1,
                    level = "info",
                    header = darkgreen("   - ")
                )

        if ask:
            rc_question = self.ask_question(
                _("Would you like to continue ?"))
            if rc_question == _("No"):
                return set(), set(), {}

        match = set()
        not_match = set()
        broken_packages = {}

        for uri in self.get_remote_packages_mirrors(repo):

            crippled_uri = EntropyTransceiver.get_uri_name(uri)
            self.output(
                "[repo:%s] %s: %s" % (
                    darkgreen(repo),
                    blue(_("Working on mirror")),
                    brown(crippled_uri),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )


            totalcounter = len(idpackages)
            currentcounter = 0

            txc = self.Transceiver(uri)
            txc.set_verbosity(False)
            with txc as handler:

                for idpackage in idpackages:

                    currentcounter += 1
                    pkgfile = dbconn.retrieveDownloadURL(idpackage)
                    pkgfile = self.complete_remote_package_relative_path(
                        pkgfile, repo = repo)
                    pkghash = dbconn.retrieveDigest(idpackage)

                    self.output(
                        "[%s] %s: %s" % (
                            brown(crippled_uri),
                            blue(_("checking hash")),
                            darkgreen(pkgfile),
                        ),
                        importance = 1,
                        level = "info",
                        header = blue(" @@ "),
                        back = True,
                        count = (currentcounter, totalcounter,)
                    )

                    ck_remote = handler.get_md5(pkgfile)
                    if ck_remote is None:
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("digest verification of")),
                                bold(pkgfile),
                                blue(_("not supported")),
                            ),
                            importance = 1,
                            level = "info",
                            header = blue(" @@ "),
                            count = (currentcounter, totalcounter,)
                        )
                        continue

                    if ck_remote == pkghash:
                        match.add(idpackage)
                    else:
                        not_match.add(idpackage)
                        self.output(
                            "[%s] %s: %s %s" % (
                                brown(crippled_uri),
                                blue(_("package")),
                                bold(pkgfile),
                                red(_("NOT healthy")),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred(" !!! "),
                            count = (currentcounter, totalcounter,)
                        )
                        if crippled_uri not in broken_packages:
                            broken_packages[crippled_uri] = []
                        broken_packages[crippled_uri].append(pkgfile)

            if broken_packages:
                mytxt = blue("%s:") % (
                    _("This is the list of broken packages"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = red(" * ")
                )
                for mirror in list(broken_packages.keys()):
                    mytxt = "%s: %s" % (
                        brown(_("Mirror")),
                        bold(mirror),
                    )
                    self.output(
                        mytxt,
                        importance = 1,
                        level = "info",
                        header = red("   <> ")
                    )
                    for broken_package in broken_packages[mirror]:
                        self.output(
                            blue(broken_package),
                            importance = 1,
                            level = "info",
                            header = red("      - ")
                        )

            self.output(
                "%s:" % (
                    blue(_("Statistics")),
                ),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    brown(_("Number of checked packages")),
                    brown(str(len(match) + len(not_match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkgreen(_("Number of healthy packages")),
                    darkgreen(str(len(match))),
                ),
                importance = 1,
                level = "info",
               header = brown("   # ")
            )
            self.output(
                "[%s] %s: %s" % (
                    red(crippled_uri),
                    darkred(_("Number of broken packages")),
                    darkred(str(len(not_match))),
                ),
                importance = 1,
                level = "info",
                header = brown("   # ")
            )

        return match, not_match, broken_packages

    def verify_local_packages(self, packages, ask = True, repo = None):

        if repo is None:
            repo = self.default_repository

        self.output(
            "[%s] %s:" % (
                red(_("local")),
                blue(_("Integrity verification of the selected packages")),
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        idpackages, world = self.match_packages(packages)
        dbconn = self.open_server_repository(read_only = True,
            repo = repo)

        if world:
            self.output(
                blue(_("All the packages in repository will be checked.")),
                importance = 1,
                level = "info",
                header = "    "
            )

        fine = set()
        failed = set()

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(idpackages, repo = repo,
                ask = ask)
        if not rc_status:
            return fine, failed, downloaded_fine, downloaded_errors

        my_qa = self.QA()

        totalcounter = str(len(available))
        currentcounter = 0
        for idpackage in available:
            currentcounter += 1
            pkg_path = dbconn.retrieveDownloadURL(idpackage)

            self.output(
                "%s: %s" % (
                    blue(_("checking status of")),
                    darkgreen(pkg_path),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            storedmd5 = dbconn.retrieveDigest(idpackage)
            pkgpath = self._get_package_path(repo, dbconn, idpackage)
            result = entropy.tools.compare_md5(pkgpath, storedmd5)
            qa_fine = my_qa.entropy_package_checks(pkgpath)
            if result and qa_fine:
                fine.add(idpackage)
            else:
                failed.add(idpackage)
                self.output(
                    "%s: %s -- %s: %s" % (
                            blue(_("package")),
                            darkgreen(pkg_path),
                            blue(_("is corrupted, stored checksum")),
                            brown(storedmd5),
                    ),
                    importance = 1,
                    level = "info",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )

        if failed:
            mytxt = blue("%s:") % (_("This is the list of broken packages"),)
            self.output(
                    mytxt,
                    importance = 1,
                    level = "warning",
                    header =  darkred("  # ")
            )
            for idpackage in failed:
                atom = dbconn.retrieveAtom(idpackage)
                down_p = dbconn.retrieveDownloadURL(idpackage)
                self.output(
                        blue("[atom:%s] %s" % (atom, down_p,)),
                        importance = 0,
                        level = "warning",
                        header =  brown("    # ")
                )

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    len(fine) + len(failed),
                    _("checked packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkgreen("%s => %s" % (
                    len(fine),
                    _("healthy packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    len(failed),
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return fine, failed, downloaded_fine, downloaded_errors

    def sign_local_packages(self, repo = None, ask = True):
        """
        Sign local packages in given repository using GPG key hopefully set
        for it.

        @raise OnlineMirrorError: if package path is not available after
            having tried to download it, this should never happen btw.
        """

        if repo is None:
            repo = self.default_repository

        self.output(
            "[%s] %s: %s" % (
                red(_("local")),
                blue(_("GPG signing packages for repository")),
                repo,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(repo = repo, read_only = False)
        idpackages = dbconn.listAllPackageIds()

        self.output(
            blue(_("All the missing packages in repository will be downloaded.")),
            importance = 1,
            level = "info",
            header = "    "
        )

        rc_status, available, downloaded_fine, downloaded_errors = \
            self._download_locally_missing_files(idpackages, repo = repo,
                ask = ask)
        if not rc_status:
            return False, 0, 0

        try:
            repo_sec = RepositorySecurity()
        except RepositorySecurity.GPGError as err:
            self.output("%s: %s" % (
                    darkgreen(_("GnuPG not available")),
                    err,
                ),
                level = "error"
            )
            return False, 0, 0

        kp_expired = False
        try:
            kp_avail = repo_sec.is_keypair_available(repo)
        except RepositorySecurity.KeyExpired:
            kp_avail = False
            kp_expired = True

        if kp_expired:
            for x in (1, 2, 3):
                # SPAM!
                self.output("%s: %s" % (
                        darkred(_("Keys for repository are expired")),
                        bold(repo),
                    ),
                    level = "warning",
                    header = bold(" !!! ")
                )
        elif not kp_avail:
            self.output("%s: %s" % (
                    darkgreen(_("Keys not available for")),
                    bold(repo),
                ),
                level = "error"
            )
            return False, 0, 0

        fine = 0
        failed = 0
        totalcounter = len(available)
        currentcounter = 0

        # clear all GPG signatures?

        # we can eventually sign!
        for idpackage in available:

            currentcounter += 1

            pkg_path = self._get_package_path(repo, dbconn, idpackage)
            if not os.path.isfile(pkg_path):
                pkg_path = self._get_upload_package_path(repo, dbconn,
                    idpackage)
            if not os.path.isfile(pkg_path):
                # wtf!?
                pkg_atom = dbconn.retrieveAtom(idpackage)
                raise OnlineMirrorError("WTF!?!?! => %s, %s" % (
                    pkg_path, pkg_atom,))

            self.output(
                "%s: %s" % (
                    blue(_("signing package")),
                    darkgreen(os.path.basename(pkg_path)),
                ),
                importance = 1,
                level = "info",
                header = "   ",
                back = True,
                count = (currentcounter, totalcounter,)
            )

            gpg_sign = self._get_gpg_signature(repo_sec, repo, pkg_path)
            if gpg_sign is None:
                self.output(
                    "%s: %s" % (
                        darkred(_("Unknown error signing package")),
                        darkgreen(os.path.basename(pkg_path)),
                    ),
                    importance = 1,
                    level = "error",
                    header = "   ",
                    count = (currentcounter, totalcounter,)
                )
                failed += 1
                continue

            # now inject gpg signature into repo
            sha1, sha256, sha512, old_gpg = dbconn.retrieveSignatures(idpackage)
            dbconn.setSignatures(idpackage, sha1, sha256, sha512,
                gpg = gpg_sign)

            fine += 1

        # print stats
        self.output(
            red("Statistics:"),
            importance = 1,
            level = "info",
            header = blue(" * ")
        )
        self.output(
            brown("%s => %s" % (
                    fine,
                    _("signed packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            darkred("%s => %s" % (
                    failed,
                    _("broken packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            blue("%s => %s" % (
                    len(downloaded_fine),
                    _("downloaded packages"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )
        self.output(
            bold("%s => %s" % (
                    len(downloaded_errors),
                    _("failed downloads"),
                )
            ),
            importance = 0,
            level = "info",
            header = brown("   # ")
        )

        self.close_repository(dbconn)
        return True, fine, failed

    def _download_locally_missing_files(self, idpackages, repo = None,
        ask = True):

        if repo is None:
            repo = self.default_repository

        dbconn = self.open_server_repository(read_only = True,
            repo = repo)

        to_download = set()
        available = set()
        for idpackage in idpackages:

            bindir_path = self._get_package_path(repo, dbconn, idpackage)
            uploaddir_path = self._get_upload_package_path(repo, dbconn,
                idpackage)
            pkg_path = dbconn.retrieveDownloadURL(idpackage)

            pkgatom = dbconn.retrieveAtom(idpackage)
            if os.path.isfile(bindir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkgreen(_("available")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                available.add(idpackage)
            elif os.path.isfile(uploaddir_path):
                self.output(
                    "[%s] %s :: %s" % (
                        darkred(_("upload/ignored")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
            else:
                self.output(
                    "[%s] %s :: %s" % (
                        brown(_("download")),
                        blue(pkgatom),
                        darkgreen(pkg_path),
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen("   # ")
                )
                to_download.add((idpackage, pkg_path,))

        if to_download and ask:
            rc_question = self.ask_question(_("Would you like to continue ?"))
            if rc_question == _("No"):
                # = downloaded fine, downloaded error
                return False, available, set(), set()

        if not to_download:
            # = downloaded fine, downloaded error
            return True, available, set(), set()

        downloaded_fine = set()
        downloaded_errors = set()

        not_downloaded = set()
        mytxt = blue("%s ...") % (_("Starting to download missing files"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = "   "
        )
        for uri in self.get_remote_packages_mirrors(repo):

            if not_downloaded:
                mytxt = blue("%s ...") % (
                    _("Searching missing/broken files on another mirror"),)
                self.output(
                    mytxt,
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                to_download = not_downloaded.copy()
                not_downloaded = set()

            for idpackage, pkg_path in to_download:
                rc_down = self.Mirrors.download_package(uri,
                    pkg_path, repo = repo)
                if rc_down:
                    downloaded_fine.add(idpackage)
                    available.add(idpackage)
                else:
                    not_downloaded.add(pkg_path)

            if not not_downloaded:
                self.output(
                    red(_("Binary packages downloaded successfully.")),
                    importance = 1,
                    level = "info",
                    header = "   "
                )
                break

        if not_downloaded:
            mytxt = blue("%s:") % (
                _("These are the packages that cannot be found online"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = "   "
            )
            for pkg_path in not_downloaded:
                downloaded_errors.add(pkg_path)
                self.output(
                        brown(pkg_path),
                        importance = 1,
                        level = "warning",
                        header = red("    * ")
                )
            downloaded_errors |= not_downloaded
            mytxt = "%s." % (_("They won't be checked"),)
            self.output(
                mytxt,
                importance = 1,
                level = "warning",
                header = "   "
            )

        return True, available, downloaded_fine, downloaded_errors

    def switch_packages_branch(self, from_branch, to_branch, repo = None):

        if repo is None:
            repo = self.default_repository

        if to_branch != self._settings['repositories']['branch']:
            mytxt = "%s: %s %s" % (
                blue(_("Please setup your branch to")),
                bold(to_branch),
                blue(_("and retry")),
            )
            self.output(
                mytxt,
                importance = 1,
                level = "error",
                header = darkred(" !! ")
            )
            return None

        mytxt = red("%s ...") % (_("Copying database (if not exists)"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )
        branch_dbdir = self._get_local_database_dir(repo)
        old_branch_dbdir = self._get_local_database_dir(repo, from_branch)

        # close all our databases
        self.close_repositories()

        # if database file did not exist got created as an empty file
        # we can just rm -rf it
        branch_dbfile = self._get_local_database_file(repo)
        if os.path.isfile(branch_dbfile):
            if entropy.tools.get_file_size(branch_dbfile) == 0:
                shutil.rmtree(branch_dbdir, True)

        if os.path.isdir(branch_dbdir):

            while True:
                rnd_num = entropy.tools.get_random_number()
                backup_dbdir = branch_dbdir + str(rnd_num)
                if not os.path.isdir(backup_dbdir):
                    break
            os.rename(branch_dbdir, backup_dbdir)

        if os.path.isdir(old_branch_dbdir):
            shutil.copytree(old_branch_dbdir, branch_dbdir)

        mytxt = red("%s ...") % (_("Switching packages"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" @@ ")
        )

        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo, lock_remote = False)
        try:
            dbconn.validate()
        except SystemDatabaseError:
            self._handle_uninitialized_repository(repo)
            dbconn = self.open_server_repository(read_only = False,
                no_upload = True, repo = repo, lock_remote = False)

        idpackages = dbconn.listAllPackageIds()
        already_switched = set()
        not_found = set()
        switched = set()
        ignored = set()
        no_checksum = set()

        maxcount = len(idpackages)
        count = 0
        for idpackage in idpackages:
            count += 1

            cur_branch = dbconn.retrieveBranch(idpackage)
            atom = dbconn.retrieveAtom(idpackage)
            if cur_branch == to_branch:
                already_switched.add(idpackage)
                self.output(
                    red("%s %s, %s %s" % (
                            _("Ignoring"),
                            bold(atom),
                            _("already in branch"),
                            cur_branch,
                        )
                    ),
                    importance = 0,
                    level = "info",
                    header = darkgreen(" @@ "),
                    count = (count, maxcount,)
                )
                ignored.add(idpackage)
                continue

            self.output(
                "[%s=>%s] %s" % (
                    brown(cur_branch),
                    bold(to_branch),
                    darkgreen(atom),
                ),
                importance = 0,
                level = "info",
                header = darkgreen(" @@ "),
                back = True,
                count = (count, maxcount,)
            )
            dbconn.switchBranch(idpackage, to_branch)
            dbconn.commitChanges()
            switched.add(idpackage)

        dbconn.commitChanges()

        # now migrate counters
        dbconn.moveSpmUidsToBranch(to_branch)

        self.close_repository(dbconn)
        mytxt = blue("%s.") % (_("migration loop completed"),)
        self.output(
            "[%s=>%s] %s" % (
                    brown(from_branch),
                    bold(to_branch),
                    mytxt,
            ),
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

        return switched, already_switched, ignored, not_found, no_checksum


class ServerQAMixin:

    def orphaned_spm_packages_test(self):

        mytxt = "%s %s" % (
            blue(_("Running orphaned SPM packages test")), red("..."),)
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        installed_packages = self.Spm().get_installed_packages()
        length = len(installed_packages)
        installed_packages.sort()
        not_found = {}
        count = 0
        for installed_package in installed_packages:
            count += 1
            self.output(
                "%s: %s" % (
                    darkgreen(_("Scanning package")),
                    brown(installed_package),),
                importance = 0,
                level = "info",
                back = True,
                count = (count, length),
                header = darkred(" @@ ")
            )
            key, slot = (entropy.tools.dep_getkey(installed_package),
                self.Spm().get_installed_package_metadata(installed_package,
                    "SLOT"),)
            pkg_atom = "%s%s%s" % (key, ":", slot,)
            try:
                tree_atoms = self.Spm().match_package(pkg_atom,
                    match_type = "match-all")
            except (KeyError,):
                tree_atoms = None # ouch!
            if not tree_atoms:
                not_found[installed_package] = pkg_atom
                self.output(
                    "%s: %s" % (
                        blue(pkg_atom),
                        darkred(_("not found anymore")),
                    ),
                    importance = 0,
                    level = "warning",
                    count = (count, length),
                    header = darkred(" @@ ")
                )

        if not_found:
            not_found_list = ' '.join([not_found[x] for x in sorted(not_found)])
            self.output(
                "%s: %s" % (
                        blue(_("Packages string")),
                        not_found_list,
                    ),
                importance = 0,
                level = "warning",
                count = (count, length),
                header = darkred(" @@ ")
            )

        return not_found

    def _deps_tester(self, default_repo = None):

        sys_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(sys_set['repositories'].keys())
        installed_packages = set()
        # if a default repository is passed, we will just test against it
        if default_repo:
            server_repos = [default_repo]

        for repo in server_repos:
            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = repo, do_treeupdates = False)
            installed_packages |= set([(x, repo) for x in \
                dbconn.listAllPackageIds()])


        deps_not_satisfied = set()
        length = str((len(installed_packages)))
        count = 0
        mytxt = _("Checking")

        for idpackage, repo in installed_packages:
            count += 1
            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = repo, do_treeupdates = False)

            if (count%150 == 0) or (count == length) or (count == 1):
                atom = dbconn.retrieveAtom(idpackage)
                self.output(
                    darkgreen(mytxt)+" "+bold(atom),
                    importance = 0,
                    level = "info",
                    back = True,
                    count = (count, length),
                    header = darkred(" @@  ")
                )

            # NOTE: this must also test build dependencies to make sure
            # that every packages comes out with all of them.
            xdeps = dbconn.retrieveDependencies(idpackage)
            for xdep in xdeps:
                xid, xuseless = self.atom_match(xdep)
                if xid == -1:
                    deps_not_satisfied.add(xdep)

        return deps_not_satisfied

    def dependencies_test(self, repo = None):

        mytxt = "%s %s" % (blue(_("Running dependencies test")), red("..."))
        self.output(
            mytxt,
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(srv_set['repositories'].keys())
        deps_not_matched = self._deps_tester(repo)

        if deps_not_matched:

            crying_atoms = {}
            for atom in deps_not_matched:
                for repo in server_repos:
                    dbconn = self.open_server_repository(just_reading = True,
                        repo = repo, do_treeupdates = False)
                    riddep = dbconn.searchDependency(atom)
                    if riddep == -1:
                        continue
                    ridpackages = dbconn.searchPackageIdFromDependencyId(riddep)
                    for i in ridpackages:
                        iatom = dbconn.retrieveAtom(i)
                        if atom not in crying_atoms:
                            crying_atoms[atom] = set()
                        crying_atoms[atom].add((iatom, repo))

            mytxt = blue("%s:") % (_("These are the dependencies not found"),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            mytxt = "%s:" % (_("Needed by"),)
            for atom in deps_not_matched:
                self.output(
                    red(atom),
                    importance = 1,
                    level = "info",
                    header = blue("   # ")
                )
                if atom in crying_atoms:
                    self.output(
                        red(mytxt),
                        importance = 0,
                        level = "info",
                        header = blue("      # ")
                    )
                    for my_dep, myrepo in crying_atoms[atom]:
                        self.output(
                            "[%s:%s] %s" % (
                                blue(_("by repo")),
                                darkred(myrepo),
                                darkgreen(my_dep),
                            ),
                            importance = 0,
                            level = "info",
                            header = blue("      # ")
                        )
        else:

            mytxt = blue(_("Every dependency is satisfied. It's all fine."))
            self.output(
                mytxt,
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )

        return deps_not_matched

    def test_shared_objects(self, get_files = False, repo = None,
        dump_results_to_file = False):

        pkg_list_path = None
        if dump_results_to_file:
            tmp_dir = tempfile.mkdtemp(prefix = "entropy.server")
            pkg_list_path = os.path.join(tmp_dir, "libtest_broken.txt")
            dmp_data = [
                (_("Broken and matched packages list"), pkg_list_path,),
            ]
            mytxt = "%s:" % (purple(_("Dumping results into these files")),)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = blue(" @@ ")
            )
            for txt, path in dmp_data:
                mytxt = "%s: %s" % (blue(txt), path,)
                self.output(
                    mytxt,
                    importance = 0,
                    level = "info",
                    header = darkgreen("   ## ")
                )


        # load db
        dbconn = self.open_server_repository(read_only = True,
            no_upload = True, repo = repo)
        QA = self.QA()
        packages_matched, brokenexecs, status = QA.test_shared_objects(dbconn,
            broken_symbols = True, dump_results_to_file = dump_results_to_file)
        if status != 0:
            return 1, None

        if get_files:
            return 0, brokenexecs

        if (not brokenexecs) and (not packages_matched):
            mytxt = "%s." % (_("System is healthy"),)
            self.output(
                blue(mytxt),
                importance = 2,
                level = "info",
                header = red(" @@ ")
            )
            return 0, None

        mytxt = "%s..." % (_("Matching libraries with Spm, please wait"),)
        self.output(
            blue(mytxt),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        real_brokenexecs = [os.path.realpath(x) for x in brokenexecs if \
            x != os.path.realpath(x)]
        brokenexecs.update(real_brokenexecs)
        packages = self.Spm().search_paths_owners(brokenexecs)

        if packages:
            mytxt = "%s:" % (_("These are the matched packages"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
            for my_atom, my_elf_id in packages:
                package_slot = my_atom, my_elf_id
                self.output(
                    "%s [elf:%s]" % (
                        purple(my_atom),
                        darkgreen(str(my_elf_id)),
                    ),
                    importance = 0,
                    level = "info",
                    header = red("     # ")
                )
                for filename in sorted(packages[package_slot]):
                    self.output(
                        darkgreen(filename),
                        importance = 0,
                        level = "info",
                        header = brown("       => ")
                    )

            pkgstring_list = sorted(["%s%s%s" % (
                entropy.tools.dep_getkey(x[0]), etpConst['entropyslotprefix'],
                    x[1],) for x in sorted(packages)])
            if pkg_list_path is not None:
                with open(pkg_list_path, "w") as pkg_f:
                    for pkgstr in pkgstring_list:
                        pkg_f.write(pkgstr + "\n")
                    pkg_f.flush()
            pkgstring = ' '.join(pkgstring_list)
            mytxt = "%s: %s" % (darkgreen(_("Packages string")), pkgstring,)
            self.output(
                mytxt,
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )
        else:
            self.output(
                red(_("No matched packages")),
                importance = 1,
                level = "info",
                header = red(" @@ ")
            )

        return 0, packages

class ServerRepositoryMixin:

    def close_repositories(self, mask_clear = False):
        for item in self._server_dbcache.keys():
            try:
                self._server_dbcache[item].closeDB()
            except ProgrammingError: # already closed?
                pass
        self._server_dbcache.clear()
        if mask_clear:
            self._settings.clear()

    def close_repository(self, dbinstance):
        found = None
        for item in self._server_dbcache:
            if dbinstance == self._server_dbcache[item]:
                found = item
                break
        if found:
            instance = self._server_dbcache.pop(found)
            instance.closeDB()

    def get_available_repositories(self):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        return srv_set['repositories'].copy()

    def switch_default_repository(self, repoid, save = None,
        handle_uninitialized = True):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        if save is None:
            save = self._save_repository
        if repoid not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        repoid,
                        _("repository not configured"),
                    )
            )
        self.close_repositories()
        srv_set['default_repository_id'] = repoid
        self.default_repository = repoid
        self._setup_services()
        if save:
            self._save_default_repository(repoid)

        self._setup_community_repositories_settings()
        self._show_interface_status()
        if handle_uninitialized and etpUi['warn']:
            self._handle_uninitialized_repository(repoid)

    def _setup_community_repositories_settings(self):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if self.community_repo:
            for repoid in srv_set['repositories']:
                srv_set['repositories'][repoid]['community'] = True

    def _handle_uninitialized_repository(self, repoid):

        if self._is_repository_initialized(repoid):
            return

        mytxt = blue("%s.") % (
            _("Your default repository is not initialized"),)
        self.output(
            "[%s:%s] %s" % (
                brown("repo"),
                purple(repoid),
                mytxt,
            ),
            importance = 1,
            level = "warning",
            header = darkred(" !!! ")
        )
        answer = self.ask_question(
            _("Do you want to initialize your default repository ?"))
        if answer == _("No"):
            mytxt = red("%s.") % (
                _("Continuing with an uninitialized repository"),)
            self.output(
                "[%s:%s] %s" % (
                    brown("repo"),
                    purple(repoid),
                    mytxt,
                ),
                importance = 1,
                level = "warning",
                header = darkred(" !!! ")
            )
        else:
            # move empty database for security sake
            dbfile = self._get_local_database_file(repoid)
            if os.path.isfile(dbfile):
                shutil.move(dbfile, dbfile+".backup")
            self.initialize_server_repository(repo = repoid,
                show_warnings = False)

    def _save_default_repository(self, repoid):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return

        if os.path.isfile(etpConst['serverconf']):
            f_srv = open(etpConst['serverconf'], "r")
            content = f_srv.readlines()
            f_srv.close()
            content = [x.strip() for x in content]
            found = False
            new_content = []
            for line in content:
                if line.strip().startswith("default-repository|"):
                    line = "default-repository|%s" % (repoid,)
                    found = True
                new_content.append(line)
            if not found:
                new_content.append("default-repository|%s" % (repoid,))
            f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp", "w")
            for line in new_content:
                f_srv_t.write(line+"\n")
            f_srv_t.flush()
            f_srv_t.close()
            os.rename(etpConst['serverconf']+".save_default_repo_tmp",
                etpConst['serverconf'])
        else:
            f_srv = open(etpConst['serverconf'], "w")
            f_srv.write("default-repository|%s\n" % (repoid,))
            f_srv.flush()
            f_srv.close()

    def toggle_repository(self, repoid, enable = True):

        # avoid setting __default__ as default server repo
        if repoid == etpConst['clientserverrepoid']:
            return False

        if not os.path.isfile(etpConst['serverconf']):
            return None
        f_srv = open(etpConst['serverconf'])
        tmpfile = etpConst['serverconf']+".switch"
        mycontent = [x.strip() for x in f_srv.readlines()]
        f_srv.close()
        f_tmp = open(tmpfile, "w")
        st = "repository|%s" % (repoid,)
        status = False
        for line in mycontent:
            if enable:
                if (line.find(st) != -1) and line.startswith("#") and \
                    (len(line.split("|")) == 5):
                    line = line[1:]
                    status = True
            else:
                if (line.find(st) != -1) and not line.startswith("#") and \
                    (len(line.split("|")) == 5):
                    line = "#"+line
                    status = True
            f_tmp.write(line+"\n")
        f_tmp.flush()
        f_tmp.close()
        shutil.move(tmpfile, etpConst['serverconf'])
        if status:
            self.close_repositories()
            self._settings.clear()
            self._setup_services()
            self._show_interface_status()
        return status

    def _is_repository_initialized(self, repo):

        def do_validate(dbc):
            try:
                dbc.validate()
                return True
            except SystemDatabaseError:
                return False

        dbc = self.open_server_repository(just_reading = True, repo = repo)
        valid = do_validate(dbc)
        if not valid:
            dbc = self.open_server_repository(read_only = False,
                no_upload = True, repo = repo, is_new = True)
            valid = do_validate(dbc)

        return valid

    def _server_repository_sync_lock(self, repo, no_upload):

        if repo is None:
            repo = self.default_repository

        # check if the database is locked locally
        lock_file = self._get_database_lockfile(repo)
        if os.path.isfile(lock_file):
            self.output(
                red(_("Entropy database is already locked by you :-)")),
                importance = 1,
                level = "info",
                header = red(" * ")
            )
        else:
            # check if the database is locked REMOTELY
            mytxt = "%s ..." % (_("Locking and Syncing Entropy database"),)
            self.output(
                red(mytxt),
                importance = 1,
                level = "info",
                header = red(" * "),
                back = True
            )
            for uri in self.get_remote_repository_mirrors(repo):

                crippled_uri = EntropyTransceiver.get_uri_name(uri)

                given_up = self.Mirrors._mirror_lock_check(uri, repo = repo)
                if given_up:
                    mytxt = "%s:" % (_("Mirrors status table"),)
                    self.output(
                        darkgreen(mytxt),
                        importance = 1,
                        level = "info",
                        header = brown(" * ")
                    )
                    dbstatus = self.Mirrors.get_mirrors_lock(repo = repo)
                    for db_uri, db_st1, db_st2 in dbstatus:
                        db_st1_info = darkgreen(_("Unlocked"))
                        if db_st1:
                            db_st1_info = red(_("Locked"))
                        db_st2_info = darkgreen(_("Unlocked"))
                        if db_st2:
                            db_st2_info = red(_("Locked"))

                        crippled_uri = EntropyTransceiver.get_uri_name(db_uri)
                        self.output(
                            "%s: [%s: %s] [%s: %s]" % (
                                bold(crippled_uri),
                                brown(_("database")),
                                db_st1_info,
                                brown(_("download")),
                                db_st2_info,
                            ),
                            importance = 1,
                            level = "info",
                            header = "\t"
                        )

                    raise OnlineMirrorError("OnlineMirrorError: %s %s" % (
                            _("cannot lock mirror"),
                            crippled_uri,
                        )
                    )

            # if we arrive here, it is because all the mirrors are unlocked
            self.Mirrors.lock_mirrors(True, repo = repo)
            self.Mirrors.sync_repositories(no_upload, repo = repo)


    def _init_generic_memory_server_repository(self, repoid, description,
        pkg_mirrors = None, repo_mirrors = None, community_repo = False,
        service_url = None, set_as_default = False):

        if pkg_mirrors is None:
            pkg_mirrors = []
        if repo_mirrors is None:
            repo_mirrors = []

        dbc = self._open_temp_repository(repoid, temp_file = ":memory:")
        self._memory_db_srv_instances[repoid] = dbc

        eapi3_port = int(etpConst['socket_service']['port'])
        eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
        # add to settings
        repodata = {
            'repoid': repoid,
            'description': description,
            'pkg_mirrors': pkg_mirrors,
            'repo_mirrors': repo_mirrors,
            'community': community_repo,
            'service_port': eapi3_port,
            'ssl_service_port': eapi3_ssl_port,
            'service_url': service_url,
            'handler': '', # not supported
            '__temporary__': True,
        }

        etpConst['server_repositories'][repoid] = repodata
        if set_as_default:
            etpConst['defaultserverrepositoryid'] = repoid
        sys_set = SystemSettings()
        sys_set.clear()

        etp_repo_meta = {
            'lock_remote': False,
            'no_upload': True,
            'output_interface': self,
            'read_only': False,
            'repo_name': repoid,
            'local_dbfile': '##this_path_does_not_exist_for_sure#' + repoid,
            '__temporary__': True,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        dbc.add_plugin(srv_plug)
        return dbc

    def _open_temp_repository(self, repo, temp_file = None):
        """
        Open temporary ServerPackagesRepository interface.

        @keyword dbname: database name
        @type dbname: string
        @keyword output_interface: entropy.output.TextInterface based instance
        @type output_interface: entropy.output.TextInterface based instance
        """
        if temp_file is None:
            tmp_fd, temp_file = tempfile.mkstemp(prefix = 'entropy.server')
            os.close(tmp_fd)

        conn = ServerPackagesRepository(
            readOnly = False,
            dbFile = temp_file,
            dbname = repo,
            xcache = False,
            indexing = False,
            skipChecks = True,
            temporary = True
        )
        conn.initializeRepository()
        return conn

    def open_repository(self, repoid):
        """
        This method aims to improve class usability by providing an easier
        method to open Entropy Server-side repositories like it happens
        with Entropy Client-side ones. If you just want open a read-only
        repository, feel free to use this wrapping method.

        @param repoid: repository identifier
        @type repoid: string
        @return: ServerPackagesRepository instance
        @rtype: entropy.server.interfaces.ServerPackagesRepository
        """
        return self.open_server_repository(repo = repoid, just_reading = True,
            do_treeupdates = False)

    def open_server_repository(
            self,
            read_only = True,
            no_upload = True,
            just_reading = False,
            repo = None,
            indexing = True,
            warnings = True,
            do_cache = True,
            use_branch = None,
            lock_remote = True,
            is_new = False,
            do_treeupdates = True
        ):

        if repo is None:
            repo = self.default_repository

        # in-memory server repos
        if repo in self._memory_db_srv_instances:
            return self._memory_db_srv_instances[repo]

        if repo == etpConst['clientserverrepoid'] and self.community_repo:
            return self.installed_repository()

        if just_reading:
            read_only = True
            no_upload = True

        local_dbfile = self._get_local_database_file(repo, use_branch)
        if do_cache:
            cached = self._server_dbcache.get(
                (repo, etpConst['systemroot'], local_dbfile, read_only,
                    no_upload, just_reading, use_branch, lock_remote,)
            )
            if cached != None:
                return cached

        local_dbfile_dir = os.path.dirname(local_dbfile)
        self._ensure_dir_path(local_dbfile_dir)

        if (not read_only) and (lock_remote) and \
            (repo not in self._sync_lock_cache):
            self._server_repository_sync_lock(repo, no_upload)
            self._sync_lock_cache.add(repo)

        conn = ServerPackagesRepository(
            readOnly = read_only,
            dbFile = local_dbfile,
            dbname = repo,
            xcache = False # always set to False, if you want to enable
            # you need to make sure that client-side and server-side caches
            # don't collide due to sharing ServerPackagesRepository.reponame
        )
        etp_repo_meta = {
            'lock_remote': lock_remote,
            'no_upload': no_upload,
            'output_interface': self,
            'read_only': read_only,
            'repo_name': repo,
            'local_dbfile': local_dbfile,
        }
        srv_plug = ServerEntropyRepositoryPlugin(self, metadata = etp_repo_meta)
        conn.add_plugin(srv_plug)

        valid = True
        try:
            conn.validate()
        except SystemDatabaseError:
            valid = False

        # verify if we need to update the database to sync
        # with portage updates, we just ignore being readonly in the case
        if (repo not in self._treeupdates_repos) and \
            (not just_reading):
            # sometimes, when filling a new server db
            # we need to avoid tree updates
            if valid:
                if do_treeupdates:
                    self._repository_packages_spm_sync(conn,
                        branch = use_branch, repo = repo)
            elif warnings and not is_new:
                mytxt = _("Entropy database is corrupted!")
                self.output(
                    darkred(mytxt),
                    importance = 1,
                    level = "warning",
                    header = bold(" !!! ")
                )

        if not read_only and valid and indexing:

            self.output(
                "[repo:%s|%s] %s" % (
                        blue(repo),
                        red(_("database")),
                        blue(_("indexing database")),
                    ),
                importance = 1,
                level = "info",
                header = brown(" @@ ")
            )
            conn.createAllIndexes()
            conn.commitChanges(no_plugins = True)

        if do_cache:
            # !!! also cache just_reading otherwise there will be
            # real issues if the connection is opened several times
            self._server_dbcache[
                (repo, etpConst['systemroot'], local_dbfile, read_only,
                no_upload, just_reading, use_branch, lock_remote,)] = conn

        return conn

    def _repository_packages_spm_sync(self, repo_db, branch = None,
        repo = None):
        """
        Service method used to sync package names with Source Package Manager.
        Source Package Manager can change package names, categories or slot
        and Entropy repositories must be kept in sync.
        """
        if branch is None:
            branch = self._settings['repositories']['branch']
        if repo is None:
            repo = self.default_repository
        self._treeupdates_repos.add(repo)
        self.Spm().package_names_update(repo_db, repo, self, branch)

    def setup_empty_repository(self, dbpath = None, repo = None):

        if dbpath is None:
            dbpath = self._get_local_database_file(repo)

        dbdir = os.path.dirname(dbpath)
        if not os.path.isdir(dbdir):
            os.makedirs(dbdir)

        mytxt = red("%s ...") % (_("Initializing an empty database"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * "),
            back = True
        )
        dbconn = self.open_generic_repository(dbpath)
        dbconn.initializeRepository()
        dbconn.commitChanges()
        dbconn.closeDB()
        mytxt = "%s %s %s." % (
            red(_("Entropy database file")),
            bold(dbpath),
            red(_("successfully initialized")),
        )
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = darkgreen(" * ")
        )

    def _get_whitelisted_licenses(self, repo = None):

        if repo is None:
            repo = self.default_repository

        wl_file = self._get_local_database_licensewhitelist_file(repo = repo)
        if not os.path.isfile(wl_file):
            return []
        return entropy.tools.generic_file_content_parser(wl_file)

    def _get_restricted_packages(self, repo = None):

        if repo is None:
            repo = self.default_repository

        rl_file = self._get_local_restricted_file(repo = repo)
        if not os.path.isfile(rl_file):
            return []
        return entropy.tools.generic_file_content_parser(rl_file,
            comment_tag = "##")

    def _is_pkg_restricted(self, pkg_atom, pkg_slot, repo = None):

        if repo is None:
            repo = self.default_repository

        restricted_pkgs = self._get_restricted_packages(repo = repo)
        if not restricted_pkgs:
            return False

        pkg_key = entropy.tools.dep_getkey(pkg_atom)
        for r_dep in restricted_pkgs:
            r_key, r_slot = entropy.tools.dep_getkey(r_dep), \
                entropy.tools.dep_getslot(r_dep)
            if r_slot is None:
                r_slot = pkg_slot

            if (r_key == pkg_key) and (r_slot == pkg_slot):
                return True

        return False

    def _is_pkg_free(self, pkg_licenses, repo = None):

        if repo is None:
            repo = self.default_repository

        # check if nonfree directory support is enabled, if not,
        # always return True.
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        if not srv_set['nonfree_packages_dir_support']:
            return True

        wl_licenses = self._get_whitelisted_licenses(repo = repo)

        if not pkg_licenses:
            return True # free if no licenses provided
        if not wl_licenses:
            return True # no whitelist !

        for lic in pkg_licenses:
            if lic not in wl_licenses:
                return False
        return True

    def _package_injector(self, package_file, inject = False, repo = None):

        if repo is None:
            repo = self.default_repository
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']

        def _package_injector_check_license(pkg_data):
            licenses = pkg_data['license'].split()
            return self._is_pkg_free(licenses, repo = repo)

        def _package_injector_check_restricted(pkg_data):
            pkgatom = entropy.tools.create_package_atom_string(
                pkg_data['category'], pkg_data['name'], pkg_data['version'],
                pkg_data['versiontag'])
            return self._is_pkg_restricted(pkgatom, pkg_data['slot'],
                repo = repo)

        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)
        self.output(
            red("[repo: %s] %s: %s" % (
                    darkgreen(repo),
                    _("adding package"),
                    bold(os.path.basename(package_file)),
                )
            ),
            importance = 1,
            level = "info",
            header = brown(" * "),
            back = True
        )
        mydata = self.Spm().extract_package_metadata(package_file,
            license_callback = _package_injector_check_license,
            restricted_callback = _package_injector_check_restricted)
        self._pump_extracted_package_metadata(mydata, repo,
            {'injected': inject,})
        idpackage, revision, mydata = dbconn.handlePackage(mydata)

        myserver_repos = list(srv_set['repositories'].keys())

        ### since we are handling more repositories, we need to make sure
        ### that there are no packages in other repositories with same atom
        ### and greater revision
        rev_test_atom = mydata['atom']
        max_rev = -1
        for myrepo in myserver_repos:

            # not myself
            if myrepo == repo:
                continue

            mydbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = myrepo)

            myrepo_idpackages = mydbconn.getPackageIds(rev_test_atom)
            for myrepo_idpackage in myrepo_idpackages:
                myrev = mydbconn.retrieveRevision(myrepo_idpackage)
                if myrev > max_rev:
                    max_rev = myrev

        if max_rev >= revision:
            max_rev += 1
            revision = max_rev
            mydata['revision'] = revision
            # update revision for pkg now
            dbconn.setRevision(idpackage, revision)


        # set trashed counters
        trashing_counters = set()

        for myrepo in myserver_repos:
            mydbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = myrepo)
            mylist = mydbconn.getPackagesToRemove(
                    mydata['name'],
                    mydata['category'],
                    mydata['slot'],
                    mydata['injected']
            )
            for myitem in mylist:
                trashing_counters.add(mydbconn.retrieveSpmUid(myitem))

        for mycounter in trashing_counters:
            dbconn.setTrashedUid(mycounter)

        # add package info to our current server repository
        dbconn.dropInstalledPackageFromStore(idpackage)
        dbconn.storeInstalledPackage(idpackage, repo)
        atom = dbconn.retrieveAtom(idpackage)

        self.output(
            "[repo:%s] %s: %s %s: %s" % (
                    darkgreen(repo),
                    blue(_("added package")),
                    darkgreen(atom),
                    blue(_("rev")), # as in revision
                    bold(str(revision)),
                ),
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )

        manual_deps = sorted(dbconn.retrieveManualDependencies(idpackage))
        if manual_deps:
            self.output(
                "[repo:%s] %s: %s" % (
                        darkgreen(repo),
                        blue(_("manual dependencies for")),
                        darkgreen(atom),
                    ),
                importance = 1,
                level = "warning",
                header = darkgreen("   ## ")
            )
            for m_dep in manual_deps:
                self.output(
                    brown(m_dep),
                    importance = 1,
                    level = "warning",
                    header = darkred("    # ")
                )

        download_url = self._setup_repository_package_filename(idpackage,
            repo = repo)
        destination_path = self.complete_local_upload_package_path(
            download_url, repo = repo)
        destination_dir = os.path.dirname(destination_path)
        self._ensure_dir_path(destination_dir)

        try:
            os.rename(package_file, destination_path)
        except OSError:
            shutil.move(package_file, destination_path)

        dbconn.commitChanges()
        return idpackage, destination_path

    # this function changes the final repository package filename
    def _setup_repository_package_filename(self, idpackage, repo = None):

        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)

        downloadurl = dbconn.retrieveDownloadURL(idpackage)
        packagerev = dbconn.retrieveRevision(idpackage)
        downloaddir = os.path.dirname(downloadurl)
        downloadfile = os.path.basename(downloadurl)
        # add revision
        downloadfile = downloadfile[:-5]+"~%s%s" % (packagerev,
            etpConst['packagesext'],)
        downloadurl = os.path.join(downloaddir, downloadfile)

        # update url
        dbconn.setDownloadURL(idpackage, downloadurl)

        return downloadurl

    def add_packages_to_repository(self, packages_data, ask = True,
        repo = None):

        if repo is None:
            repo = self.default_repository

        mycount = 0
        maxcount = len(packages_data)
        idpackages_added = set()
        to_be_injected = set()
        my_qa = self.QA()
        for package_filepath, inject in packages_data:

            mycount += 1
            self.output(
                "[repo:%s] %s: %s" % (
                    darkgreen(repo),
                    blue(_("adding package")),
                    darkgreen(os.path.basename(package_filepath)),
                ),
                importance = 1,
                level = "info",
                header = blue(" @@ "),
                count = (mycount, maxcount,)
            )

            try:
                # add to database
                idpackage, destination_path = self._package_injector(
                    package_filepath,
                    inject = inject,
                    repo = repo
                )
                idpackages_added.add(idpackage)
                to_be_injected.add((idpackage, destination_path))
            except Exception as err:
                entropy.tools.print_traceback()
                self.output(
                    "[repo:%s] %s: %s" % (
                        darkgreen(repo),
                        darkred(_("Exception caught, closing tasks")),
                        darkgreen(str(err)),
                    ),
                    importance = 1,
                    level = "error",
                    header = bold(" !!! "),
                    count = (mycount, maxcount,)
                )
                # reinit librarypathsidpackage table
                if idpackages_added:
                    dbconn = self.open_server_repository(read_only = False,
                        no_upload = True, repo = repo)
                    my_qa.test_missing_dependencies(
                        idpackages_added,
                        dbconn,
                        ask = ask,
                        repo = repo,
                        self_check = True,
                        black_list = \
                            self._get_missing_dependencies_blacklist(
                                repo = repo),
                        black_list_adder = \
                            self._add_missing_dependencies_blacklist_items
                    )
                    my_qa.test_reverse_dependencies_linking(idpackages_added,
                        dbconn, repo = repo)
                if to_be_injected:
                    self._inject_database_into_packages(to_be_injected,
                        repo = repo)
                self.close_repositories()
                raise

        # make sure packages are really available, it can happen
        # after a previous failure to have garbage here
        dbconn = self.open_server_repository(just_reading = True, repo = repo)
        idpackages_added = set((x for x in idpackages_added if \
            dbconn.isPackageIdAvailable(x)))

        if idpackages_added:
            dbconn = self.open_server_repository(read_only = False,
                no_upload = True, repo = repo)
            my_qa.test_missing_dependencies(
                idpackages_added,
                dbconn,
                ask = ask,
                repo = repo,
                self_check = True,
                black_list = \
                    self._get_missing_dependencies_blacklist(repo = repo),
                black_list_adder = \
                    self._add_missing_dependencies_blacklist_items
            )
            my_qa.test_reverse_dependencies_linking(idpackages_added, dbconn,
                repo = repo)

        # inject database into packages
        self._inject_database_into_packages(to_be_injected, repo = repo)

        return idpackages_added

    def _taint_database(self, repo = None):
        if repo is None:
            repo = self.default_repository

        # taint the database status
        db_file = self._get_local_database_file(repo = repo)
        taint_file = self._get_local_database_taint_file(repo = repo)
        f = open(taint_file, "w")
        f.write("database tainted\n")
        f.flush()
        f.close()
        const_setup_file(taint_file, etpConst['entropygid'], 0o664)
        ServerRepositoryStatus().set_tainted(db_file)

    def _bump_database(self, repo = None):
        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)
        self._taint_database(repo = repo)
        self.close_repository(dbconn)

class ServerMiscMixin:

    def _ensure_paths(self, repo):
        upload_dir = self._get_local_upload_directory(repo)
        db_dir = self._get_local_database_dir(repo)
        for mydir in [upload_dir, db_dir]:
            if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
                os.makedirs(mydir, 0o755)
                const_setup_perms(mydir, etpConst['entropygid'],
                    recursion = False, uid = etpConst['uid'])

    def _ensure_dir_path(self, dir_path):
        if not os.path.isdir(dir_path):
            os.makedirs(dir_path, 0o755)
            const_setup_perms(dir_path, etpConst['entropygid'],
                recursion = False, uid = etpConst['uid'])

    def _setup_services(self):
        self._setup_entropy_settings()
        self._backup_entropy_settings()
        self.Mirrors = MirrorsServer(self)

    def _setup_entropy_settings(self, repo = None):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        backup_list = [
            'etpdatabaseclientfilepath',
            'clientdbid',
            {'server': srv_set.copy()},
        ]
        for setting in backup_list:
            if setting not in self._settings_to_backup:
                self._settings_to_backup.append(setting)
        # setup client database
        if not self.community_repo:
            etpConst['etpdatabaseclientfilepath'] = \
                self._get_local_database_file(repo = repo)
            etpConst['clientdbid'] = etpConst['serverdbid']
        const_create_working_dirs()

    def _show_interface_status(self):
        type_txt = _("server-side repository")
        if self.community_repo:
            type_txt = _("community repository")
        # ..on repository: <repository_name>
        mytxt = _("Entropy Server Interface Instance on repository")
        self.output(
            blue("%s: %s, %s: %s (%s: %s)" % (
                    mytxt,
                    red(self.default_repository),
                    _("current branch"),
                    darkgreen(self._settings['repositories']['branch']),
                    purple(_("type")),
                    bold(type_txt),
                )
            ),
            importance = 2,
            level = "info",
            header = red(" @@ ")
        )
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        repos = list(srv_set['repositories'].keys())
        mytxt = blue("%s:") % (_("Currently configured repositories"),)
        self.output(
            mytxt,
            importance = 1,
            level = "info",
            header = red(" @@ ")
        )
        for repo in repos:
            self.output(
                darkgreen(repo),
                importance = 0,
                level = "info",
                header = brown("   # ")
            )

    def _backup_entropy_settings(self):
        for setting in self._settings_to_backup:
            if isinstance(setting, const_get_stringtype()):
                self._backup_constant(setting)
            elif isinstance(setting, dict):
                self._settings.set_persistent_setting(setting)

    def _get_gpg_signature(self, repo_sec, repo, pkg_path):
        try:
            if not repo_sec.is_keypair_available(repo):
                return None # GPG is not enabled
        except RepositorySecurity.KeyExpired as err:
            self.output(
                "[repo:%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG key expired")),
                    err,
                    darkred(_("please frigging fix")),
                ),
                importance = 1,
                level = "warning",
                header = bold(" !!! ")
            )
            return None
        except RepositorySecurity.GPGError as err:
            self.output(
                "[repo:%s] %s: %s, %s." % (
                    darkgreen(repo),
                    darkred(_("GPG got unexpected error")),
                    err,
                    darkred(_("skipping")),
                ),
                importance = 1,
                level = "warning",
                header = red(" @@ ")
            )
            return None
        gpg_sign_path = repo_sec.sign_file(repo, pkg_path)
        # read file content and add to 'gpg' signature
        with open(gpg_sign_path, "rb") as gpg_f:
            return gpg_f.read()
        os.remove(gpg_sign_path)

    def _check_config_file_updates(self):
        self.output(
            "[%s] %s" % (
                red(_("config files")), # something short please
                blue(_("checking system")),
            ),
            importance = 1,
            level = "info",
            header = blue(" @@ "),
            back = True
        )
        # scanning for config files not updated
        scandata = self.FileUpdates.scan(dcache = False)
        if scandata:
            self.output(
                "[%s] %s" % (
                    red(_("config files")), # something short please
                    blue(_("there are configuration files not updated yet")),
                ),
                importance = 1,
                level = "error",
                header = darkred(" @@ ")
            )
            for key in scandata:
                self.output(
                    "%s" % (brown(etpConst['systemroot'] + \
                        scandata[key]['destination'])),
                    importance = 1,
                    level = "info",
                    header = "\t"
                )
            return True
        return False

    def _get_missing_dependencies_blacklist(self, repo = None, branch = None):
        if repo is None:
            repo = self.default_repository
        if branch is None:
            branch = self._settings['repositories']['branch']
        wl_file = self._get_missing_dependencies_blacklist_file(repo, branch)
        wl_data = []
        if os.path.isfile(wl_file) and os.access(wl_file, os.R_OK):
            f_wl = open(wl_file, "r")
            wl_data = [x.strip() for x in f_wl.readlines() if x.strip() and \
                not x.strip().startswith("#")]
            f_wl.close()
        return set(wl_data)

    def _add_missing_dependencies_blacklist_items(self, items, repo = None,
        branch = None):

        if repo is None:
            repo = self.default_repository
        if branch is None:
            branch = self._settings['repositories']['branch']
        wl_file = self._get_missing_dependencies_blacklist_file(repo, branch)
        wl_dir = os.path.dirname(wl_file)
        if not (os.path.isdir(wl_dir) and os.access(wl_dir, os.W_OK)):
            return
        if os.path.isfile(wl_file) and not os.access(wl_file, os.W_OK):
            return
        f_wl = open(wl_file, "a+")
        f_wl.write('\n'.join(items)+'\n')
        f_wl.flush()
        f_wl.close()

    def _get_package_path(self, repo, dbconn, idpackage):
        """
        Given EntropyRepository instance and package identifier, return local
        path of package. This method does not check for path validity though.
        """
        pkg_rel_url = dbconn.retrieveDownloadURL(idpackage)
        complete_path = self.complete_local_package_path(pkg_rel_url,
            repo = repo)
        return complete_path

    def _get_upload_package_path(self, repo, dbconn, idpackage):
        """
        Given ServerPackagesRepository instance and package identifier,
        return local path of package.
        This method does not check for path validity though.
        """
        pkg_path = dbconn.retrieveDownloadURL(idpackage)
        return os.path.join(self._get_local_upload_directory(repo = repo),
            pkg_path)

    def scan_package_changes(self):

        spm = self.Spm()

        spm_packages = spm.get_installed_packages()
        installed_packages = []
        for spm_package in spm_packages:
            pkg_counter = spm.get_installed_package_metadata(spm_package,
                "COUNTER")
            try:
                pkg_counter = int(pkg_counter)
            except ValueError:
                continue
            installed_packages.append((spm_package, pkg_counter,))

        installed_counters = set()
        to_be_added = set()
        to_be_removed = set()
        to_be_injected = set()
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        exp_based_scope = my_settings['exp_based_scope']

        server_repos = list(my_settings['repositories'].keys())

        # packages to be added
        for spm_atom, spm_counter in installed_packages:
            found = False
            for server_repo in server_repos:
                installed_counters.add(spm_counter)
                server_dbconn = self.open_server_repository(read_only = True,
                    no_upload = True, repo = server_repo)
                counter = server_dbconn.isSpmUidAvailable(spm_counter)
                if counter:
                    found = True
                    break
            if not found:
                to_be_added.add((spm_atom, spm_counter,))

        # packages to be removed from the database
        database_counters = {}
        for server_repo in server_repos:
            server_dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = server_repo)
            database_counters[server_repo] = server_dbconn.listAllSpmUids()

        ordered_counters = set()
        for server_repo in database_counters:
            for data in database_counters[server_repo]:
                ordered_counters.add((data, server_repo))
        database_counters = ordered_counters

        for (counter, idpackage,), xrepo in database_counters:

            if counter < 0:
                continue # skip packages without valid counter

            if counter in installed_counters:
                continue

            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = xrepo)

            dorm = True
            # check if the package is in to_be_added
            if to_be_added:

                dorm = False
                atom = dbconn.retrieveAtom(idpackage)
                atomkey = entropy.tools.dep_getkey(atom)
                atomtag = entropy.tools.dep_gettag(atom)
                atomslot = dbconn.retrieveSlot(idpackage)

                add = True
                for spm_atom, spm_counter in to_be_added:
                    addslot = self.Spm().get_installed_package_metadata(
                        spm_atom, "SLOT")
                    addkey = entropy.tools.dep_getkey(spm_atom)
                    # workaround for ebuilds not having slot
                    if addslot is None:
                        addslot = '0'
                    # atomtag != None is for handling tagged pkgs correctly
                    if (atomkey == addkey) and \
                        ((str(atomslot) == str(addslot)) or (atomtag != None)):
                        # do not add to to_be_removed
                        add = False
                        break

                if not add:
                    continue
                dorm = True

            # checking if we are allowed to remove stuff on this repo
            # it xrepo is not the default one, we MUST skip this to
            # avoid touching what developer doesn't expect
            if dorm and (xrepo == self.default_repository):
                trashed = self._is_spm_uid_trashed(counter)
                if trashed:
                    # search into portage then
                    try:
                        key, slot = dbconn.retrieveKeySlot(idpackage)
                        slot = slot.split(",")[0]
                        trashed = self.Spm().match_installed_package(
                            key+":"+slot)
                    except TypeError: # referred to retrieveKeySlot
                        trashed = True
                if not trashed:

                    dbtag = dbconn.retrieveTag(idpackage)
                    if dbtag:
                        is_injected = dbconn.isInjected(idpackage)
                        if not is_injected:
                            to_be_injected.add((idpackage, xrepo))

                    elif exp_based_scope:

                        # check if support for this is set
                        plg_id = self.sys_settings_fatscope_plugin_id
                        exp_data = self._settings[plg_id]['repos'].get(
                            xrepo, set())

                        # only some packages are set, check if our is
                        # in the list
                        if (idpackage not in exp_data) and (-1 not in exp_data):
                            to_be_removed.add((idpackage, xrepo))
                            continue

                        idpackage_expired = self._is_match_expired((idpackage,
                            xrepo,))

                        if idpackage_expired:
                            # expired !!!
                            # add this and its depends (reverse deps)

                            rm_match = (idpackage, xrepo)
                            #to_be_removed.add(rm_match)
                            revdep_matches = self.get_reverse_queue([rm_match],
                                system_packages = False)
                            to_be_removed.update(revdep_matches)

                    else:
                        to_be_removed.add((idpackage, xrepo))

        return to_be_added, to_be_removed, to_be_injected

    def _is_match_expired(self, match):

        idpackage, repoid = match
        dbconn = self.open_server_repository(repo = repoid, just_reading = True)
        # 3600 * 24 = 86400
        my_settings = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        pkg_exp_secs = my_settings['packages_expiration_days'] * 86400
        cur_unix_time = time.time()
        # if packages removal is triggered by expiration
        # we will have to check if our package is really
        # expired and remove its reverse deps too
        mydate = dbconn.retrieveCreationDate(idpackage)
        # cross fingers hoping that time is set correctly
        mydelta = cur_unix_time - float(mydate)
        if mydelta > pkg_exp_secs:
            return True
        return False

    def _is_spm_uid_trashed(self, counter):
        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        server_repos = list(srv_set['repositories'].keys())
        for repo in server_repos:
            dbconn = self.open_server_repository(read_only = True,
                no_upload = True, repo = repo)
            if dbconn.isSpmUidTrashed(counter):
                return True
        return False

    def _transform_package_into_injected(self, idpackage, repo = None):
        dbconn = self.open_server_repository(read_only = False,
            no_upload = True, repo = repo)
        counter = dbconn.getFakeSpmUid()
        dbconn.setSpmUid(idpackage, counter)
        dbconn.setInjected(idpackage)

    def _pump_extracted_package_metadata(self, pkg_meta, repo, extra_metadata):
        """
        Add to pkg_meta dict, server-side package metadata information before
        injecting it into repository database.
        """
        # add extra metadata
        pkg_meta.update(extra_metadata)
        # do not set GPG signature here for performance, just provide an
        # empty default. Real GPG signature will be written inside
        # _inject_database_into_packages()
        pkg_meta['signatures']['gpg'] = None

        # rewrite dependency strings using dep_rewrite metadata
        self.__handle_dep_rewrite(pkg_meta, repo)

    def __handle_dep_rewrite(self, pkg_meta, repo):

        dep_rewrite = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['dep_rewrite']

        # NOTE: to be able to match the package, we need to add it
        # to a temp repo
        tmp_repo = self._open_temp_repository("dep_rewrite_temp",
            temp_file = ":memory:")
        new_idpackage, new_revision, new_data = tmp_repo.handlePackage(pkg_meta)
        del new_revision
        del new_data
        pkg_atom = tmp_repo.retrieveAtom(new_idpackage)

        rewrites_enabled = []
        for dep_string_rewrite, dep_pattern in dep_rewrite:
            pkg_id, rc = tmp_repo.atomMatch(dep_string_rewrite)
            if rc == 0:
                rewrites_enabled.append((dep_string_rewrite, dep_pattern))
        tmp_repo.closeDB()

        if not rewrites_enabled:
            return

        self.output(
            "[repo:%s|%s] %s:" % (
                    blue(repo),
                    brown(pkg_atom),
                    teal(_("found available dep_rewrites for this package")),
                ),
            importance = 1,
            level = "info",
            header = brown(" @@ ")
        )
        for dep_string_rewrite, dep_pattern in rewrites_enabled:
            compiled_pattern, replaces = \
                dep_rewrite[(dep_string_rewrite, dep_pattern)]
            self.output(
                "%s => %s" % (
                    purple(dep_pattern),
                    ', '.join(replaces),
                ),
                importance = 1,
                level = "info",
                header = teal("   # ")
            )

        for dep_string, dep_value in pkg_meta['dependencies'].items():

            dep_string_matched = False
            matched_pattern = False

            for key in rewrites_enabled:

                compiled_pattern, replaces = dep_rewrite[key]

                if not compiled_pattern.match(dep_string):
                    # dep_string not matched, skipping
                    continue
                matched_pattern = True

                for replace in replaces:
                    new_dep_string, number_of_subs_made = \
                        compiled_pattern.subn(replace, dep_string)
                    if number_of_subs_made:
                        dep_string_matched = True
                        if new_dep_string and (new_dep_string != "-"):
                            pkg_meta['dependencies'][new_dep_string] = dep_value
                            self.output(
                                "%s: %s => %s" % (
                                    teal(_("replaced")),
                                    brown(dep_string),
                                    purple(new_dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )
                        else:
                            self.output(
                                "%s: %s => X" % (
                                    teal(_("removed")),
                                    brown(dep_string),
                                ),
                                importance = 1,
                                level = "info",
                                header = purple("   ! ")
                            )

                    else:
                        self.output(
                            "%s: %s + %s" % (
                                darkred(_("No dependency rewrite made for")),
                                brown(dep_string),
                                purple(replace),
                            ),
                            importance = 1,
                            level = "warning",
                            header = darkred("   !!! ")
                        )

            if dep_string_matched:
                del pkg_meta['dependencies'][dep_string]
            elif (not dep_string_matched) and matched_pattern:
                self.output(
                    "%s: %s :: %s" % (
                        darkred(_("No dependency rewrite made for")),
                        brown(pkg_atom),
                        purple(dep_string),
                    ),
                    importance = 1,
                    level = "warning",
                    header = darkred("   !x!x!x! ")
                )

    def _get_entropy_sets(self, repo = None, branch = None):

        if branch is None:
            branch = self._settings['repositories']['branch']
        if repo is None:
            repo = self.default_repository

        sets_dir = self._get_local_database_sets_dir(repo, branch)
        if not (os.path.isdir(sets_dir) and os.access(sets_dir, os.R_OK)):
            return {}

        mydata = {}
        items = os.listdir(sets_dir)
        for item in items:

            try:
                item_clean = str(item)
            except (UnicodeEncodeError, UnicodeDecodeError,):
                continue
            item_path = os.path.join(sets_dir, item)
            if not (os.path.isfile(item_path) and \
                os.access(item_path, os.R_OK)):
                continue
            item_elements = self._settings._extract_packages_from_set_file(
                item_path)
            if item_elements:
                mydata[item_clean] = item_elements.copy()

        return mydata

    def _get_configured_package_sets(self, repo = None, branch = None,
        validate = True):

        if branch is None:
            branch = self._settings['repositories']['branch']
        if repo is None:
            repo = self.default_repository

        # portage sets
        sets_data = self.Spm().get_package_sets(False)
        sets_data.update(self._get_entropy_sets(repo, branch))

        if validate:
            invalid_sets = set()
            # validate
            for setname in sets_data:
                good = True
                for atom in sets_data[setname]:
                    if atom.startswith(etpConst['packagesetprefix']):
                        # ignore nested package sets
                        continue
                    dbconn = self.open_server_repository(just_reading = True,
                        repo = repo)
                    match = dbconn.atomMatch(atom)
                    if match[0] == -1:
                        good = False
                        break
                if not good:
                    invalid_sets.add(setname)
            for invalid_set in invalid_sets:
                del sets_data[invalid_set]

        return sets_data

    def _update_database_package_sets(self, repo = None, dbconn = None):

        if repo is None:
            repo = self.default_repository
        package_sets = self._get_configured_package_sets(repo)
        if dbconn is None:
            dbconn = self.open_server_repository(
                read_only = False, no_upload = True, repo = repo,
                do_treeupdates = False)
        dbconn.clearPackageSets()
        if package_sets:
            dbconn.insertPackageSets(package_sets)
        dbconn.commitChanges()

class Server(ServerSettingsMixin, ServerLoadersMixin,
    ServerPackageDepsMixin, ServerPackagesHandlingMixin, ServerQAMixin,
    ServerRepositoryMixin, ServerMiscMixin, _Client):

    # Entropy Server cache directory, mainly used for storing commit changes
    CACHE_DIR = os.path.join(etpConst['entropyworkdir'], "server_cache")

    # SystemSettings class variables
    SYSTEM_SETTINGS_PLG_ID = etpConst['system_settings_plugins_ids']['server_plugin']

    def init_singleton(self, default_repository = None, save_repository = False,
            community_repo = False, fake_default_repo = False,
            fake_default_repo_id = None,
            fake_default_repo_desc = 'this is a fake repository'):

        self.__instance_destroyed = False
        if etpConst['uid'] != 0:
            mytxt = _("Entropy Server interface must be run as root")
            import warnings
            warnings.warn(mytxt)

        self._cacher = EntropyCacher()
        # settings
        self._memory_db_srv_instances = {}
        self._treeupdates_repos = set()
        self._server_dbcache = {}
        self._settings = SystemSettings()
        self.community_repo = community_repo
        etpSys['serverside'] = True
        self.fake_default_repo = fake_default_repo
        self.fake_default_repo_id = fake_default_repo_id
        self.indexing = False
        self.xcache = False
        self.Mirrors = None
        self._settings_to_backup = []
        self._save_repository = save_repository
        self._sync_lock_cache = set()

        self.sys_settings_fake_cli_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fake_client']
        self.sys_settings_fatscope_plugin_id = \
            etpConst['system_settings_plugins_ids']['server_plugin_fatscope']

        # create our SystemSettings plugin
        with self._settings:
            self.sys_settings_plugin = ServerSystemSettingsPlugin(
                Server.SYSTEM_SETTINGS_PLG_ID, self)
            self._settings.add_plugin(self.sys_settings_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fatscope_plugin = \
                ServerFatscopeSystemSettingsPlugin(
                    self.sys_settings_fatscope_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fatscope_plugin)

            # Fatscope support SystemSettings plugin
            self.sys_settings_fake_cli_plugin = \
                ServerFakeClientSystemSettingsPlugin(
                    self.sys_settings_fake_cli_plugin_id, self)
            self._settings.add_plugin(self.sys_settings_fake_cli_plugin)

        # setup fake repository
        if fake_default_repo:
            default_repository = fake_default_repo_id
            self._init_generic_memory_server_repository(fake_default_repo_id,
                fake_default_repo_desc, set_as_default = True)

        srv_set = self._settings[Server.SYSTEM_SETTINGS_PLG_ID]['server']
        self.default_repository = default_repository
        if self.default_repository is None:
            self.default_repository = srv_set['default_repository_id']

        if self.default_repository in srv_set['repositories']:
            self._ensure_paths(self.default_repository)

        if self.default_repository not in srv_set['repositories']:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                        self.default_repository,
                        _("repository not configured"),
                    )
            )
        if etpConst['clientserverrepoid'] == self.default_repository:
            raise PermissionDenied("PermissionDenied: %s %s" % (
                    etpConst['clientserverrepoid'],
                    _("protected repository id, can't use this, sorry dude..."),
                )
            )

        self.switch_default_repository(self.default_repository)
        # initialize Entropy Client superclass
        _Client.init_singleton(self,
            indexing = self.indexing,
            xcache = self.xcache,
            repo_validation = False,
            noclientdb = 1
        )

    def destroy(self):
        self.__instance_destroyed = True
        _Client.close_repositories(self, mask_clear = False)
        _Client.destroy(self)

        plug_id2 = self.sys_settings_fake_cli_plugin_id
        plug_id1 = self.sys_settings_fatscope_plugin_id
        plug_id = Server.SYSTEM_SETTINGS_PLG_ID
        # reverse insert order
        plugs = [plug_id2, plug_id1, plug_id]
        for plug in plugs:
            if plug is None:
                continue
            if not self._settings.has_plugin(plug):
                continue
            self._settings.remove_plugin(plug)

        self.close_repositories()

    def is_destroyed(self):
        return self.__instance_destroyed

    def __del__(self):
        self.destroy()