#!/usr/bin/python
import sys
sys.path.insert(0, '/usr/lib/entropy/lib')
sys.path.insert(0, '../lib')

import time
import os
import tempfile
import errno
import bz2

from entropy.i18n import _
from entropy.output import print_info, blue, teal, brown, darkgreen, purple, \
    print_error, print_warning, TextInterface
from entropy.exceptions import SystemDatabaseError
from entropy.client.interfaces.db import GenericRepository
from entropy.qa import QAInterface
from entropy.core.settings.base import SystemSettings
from entropy.exceptions import EntropyException

from entropy.const import const_convert_to_rawstring, etpConst
import entropy.dep
import entropy.tools

class WebinstallGenerator(TextInterface):

    SHELL_PREAMBLE = const_convert_to_rawstring("""\
#!/bin/sh

function execute_app() {
    equo install "$0" $@
    return ${?}
}

execute_app $@
exit ${?}

"""+ etpConst['databasestarttag'])

    class CalculationError(EntropyException):
        """Raised when an error occured while calculating the work queue"""

    def __init__(self, repository_id, entropy_repository, package_dirs,
        mirror_urls, regenerate = False):
        self._regenerate = regenerate
        self._repo_id = repository_id
        self._repo = entropy_repository
        self._package_dirs = package_dirs
        # this is part of the (unwritten) specification, don't change it!
        self._mirror_urls_str = "\n".join(mirror_urls)
        self._qa = QAInterface()

    def __copy_data(self, f_obj_source, f_obj_dest):
        while True:
            chunk = f_obj_source.read(16384)
            if not chunk:
                break
            f_obj_dest.write(chunk)
        f_obj_dest.flush()

    def _calculate_required_actions(self):
        """
        Compute the list of package identifiers requiring a webinstall package
        generation and collect the expired ones (that would be removed).
        """
        package_ids = self._repo.listAllPackageIds()
        download_map = dict((x, self._repo.retrieveDownloadURL(x)) for x in
            package_ids)

        package_dirs_cache = {}
        expired_webinstall_files = set()
        work_queue = []
        max_count = len(package_ids)
        count = 0

        def _determine_local_package_path(package_id):
            download_path = download_map[package_id]
            download_path_dir, download_file_name = os.path.split(download_path)
            local_package_dir = package_dirs_cache.get(download_path_dir)
            if local_package_dir is None:
                for package_dir in self._package_dirs:
                    if package_dir.endswith(download_path_dir):
                        local_package_dir = package_dir
                        package_dirs_cache[download_path_dir] = package_dir
                        break
            if local_package_dir is None:
                self.output("%s: %s" % (
                        brown("Cannot find local package directory for"),
                        download_path,
                    ),
                    header = teal(" @@ "),
                    importance = 1, back = True,
                    level = "error"
                )
                raise WebinstallGenerator.CalculationError(
                    "cannot find local package dir for: %s" % (download_path,))

            local_package_path = os.path.join(local_package_dir,
                download_file_name)
            return local_package_path

        def _determine_local_etp_path(local_package_dir, package_id):
            atom, category, name, version, slot, tag, revision, branch, \
                etpapi = self._repo.getScopeData(package_id)
            version += "%s%s" % (etpConst['entropyrevisionprefix'], revision,)
            local_etp_fn = entropy.dep.create_package_filename(category,
                name, version, tag, ext = etpConst['packagesext_webinstall'])
            local_etp_path = os.path.join(local_package_dir, local_etp_fn)
            return local_etp_path

        def _validate_etp_creation(local_package_path, local_etp_path, overwrite):
            if self._regenerate:
                # always generate the file in this case
                return True
            if not os.path.isfile(local_package_path) and \
                os.path.isfile(local_etp_path):
                expired_webinstall_files.add(local_etp_path)
                return False
            if not os.path.isfile(local_package_path):
                # local_etp_path does not exist and package file is not
                # available, skip!
                return False
            if not overwrite:
                if os.path.isfile(local_etp_path):
                    # already available
                    return False
            return True

        work_queue_cache = set()

        for package_id in sorted(package_ids, reverse = True):
            count += 1
            download_path = download_map[package_id]
            if (count % 150 == 0) or (count == 0) or (count == max_count):
                self.output("%s: %s" % (purple("scanning"), download_path),
                    header = teal(" @@ "),
                    count = (count, max_count),
                    importance = 0,
                    back = True)

            # don't enqueue the same package id twice
            if package_id in work_queue_cache:
                continue

            local_package_path = _determine_local_package_path(package_id)
            local_package_dir = os.path.dirname(local_package_path)
            local_etp_path = _determine_local_etp_path(local_package_dir,
                package_id)

            valid = _validate_etp_creation(local_package_path,
                local_etp_path, False)
            if not valid:
                continue

            work_queue.append((package_id, local_package_path, local_etp_path))
            work_queue_cache.add(package_id)

            # also pull in its reverse dependencies
            # this to ensure that packages referencing this packages will
            # have updated metadata.
            # in this case, etp files are going to be overwritten
            revdeps = self._repo.retrieveReverseDependencies(package_id)
            revdeps = [x for x in revdeps if x not in work_queue_cache]
            for rev_pkg_id in revdeps:
                local_package_path = _determine_local_package_path(rev_pkg_id)
                local_package_dir = os.path.dirname(local_package_path)
                local_etp_path = _determine_local_etp_path(local_package_dir,
                    rev_pkg_id)
                valid = _validate_etp_creation(local_package_path,
                    local_etp_path, True)
                if valid:
                    work_queue.append((rev_pkg_id, local_package_path,
                        local_etp_path))
                    work_queue_cache.add(rev_pkg_id)

        # collect really expired .etp files
        ext = etpConst['packagesext_webinstall']
        for package_dir in self._package_dirs:
            if not os.path.isdir(package_dir):
                continue
            for cur_dir, sub_dirs, files in os.walk(package_dir):
                for etp_file in files:
                    if not etp_file.endswith(ext):
                        continue
                    etp_file = os.path.join(package_dir, etp_file)
                    pkg_file = etp_file[:-len(ext)] + etpConst['packagesext']
                    if not os.path.isfile(pkg_file):
                        # we can drop our .etp
                        expired_webinstall_files.add(etp_file)

        return work_queue, expired_webinstall_files

    def _cleanup_expired_files(self, expired_webinstall_files):
        """
        Cleanup routine that removes expired webinstall package files.
        """
        for expired_file in sorted(expired_webinstall_files):
            try:
                os.remove(expired_file)
            except OSError as err:
                self.output("%s %s: %s" % (
                        teal("cannot remove"),
                        expired_file,
                        repr(err),
                    ),
                    header = brown(" @@ "),
                    level = "warning",
                    importance = 1
                )

    def _prepare_base_package_repository(self):
        """
        Prepare an empty Entropy Repository that will be used as base
        for embedding package metadata.
        """
        treeupdates_actions = self._repo.listAllTreeUpdatesActions()

        # generate empty repository file and re-use it every time
        # this improves the execution a lot
        orig_fd, tmp_repo_orig_path = tempfile.mkstemp(
            suffix="repo-webinst-gen")
        try:
            empty_repo = GenericRepository(
                readOnly = False,
                dbFile = tmp_repo_orig_path,
                name = "empty",
                xcache = False,
                indexing = False,
                skipChecks = True)
            empty_repo.initializeRepository()
            empty_repo.bumpTreeUpdatesActions(treeupdates_actions)
            empty_repo.commit()
            empty_repo.close()
            return tmp_repo_orig_path
        except Exception:
            os.remove(tmp_repo_orig_path)
            raise
        finally:
            os.close(orig_fd)

    def _generate_webinstall_package(self, base_repository_path,
        package_id, package_path, etp_path, cache_map):
        """
        Generate a webinstall package for given package_id matched inside
        the working Entropy Repository instance passed at constructor time.
        If no exceptions are raised, the generation went successful.
        The webinstall file generation is atomic.
        """

        # handle caching
        cache_map_len_threshold = 800
        cache_map_len = len(cache_map)
        if cache_map_len > cache_map_len_threshold:
            to_remove = cache_map_len_threshold - cache_map_len
            # LRU logic ! yay!
            for key in sorted(cache_map.keys(), key = lambda x: cache_map[x][0]):
                if to_remove < 1:
                    break
                to_remove -= 1
                cache_map.pop(key)

        # WARNING: usage of undocumented feature of
        # get_deep_dependency_list
        # NOTE: how about build deps?
        pkg_match = (package_id, self._repo)
        matches = self._qa.get_deep_dependency_list(None, pkg_match)

        tmp_fd, tmp_repo_path = tempfile.mkstemp(
            suffix="repo-webinst-gen")
        with os.fdopen(tmp_fd, "wb") as tmp_repo_f:
            with open(base_repository_path, "rb") as tmp_repo_source_f:
                self.__copy_data(tmp_repo_source_f, tmp_repo_f)

        dest_repo = None
        compressed_tmp_path = None
        compressed_fd = None
        try:

            repo_arch = self._repo.getSetting("arch")
            atom = self._repo.retrieveAtom(package_id)
            dest_repo = GenericRepository(
                readOnly = False,
                dbFile = tmp_repo_path,
                name = atom,
                xcache = False,
                indexing = False,
                skipChecks = True)

            deps_pkg_ids = set([pkg_id for pkg_id, _repo in matches])
            deps_pkg_ids.add(package_id)
            for dep_package_id in deps_pkg_ids:
                cached = cache_map.get(dep_package_id)
                if cached is None:
                    data = self._repo.getPackageData(dep_package_id,
                        get_content = False, get_changelog = False)
                    cache_map[dep_package_id] = (time.time(), data)
                else:
                    cache_t, data = cached

                if "original_repository" in data:
                    del data['original_repository']
                dest_package_id = dest_repo.addPackage(data,
                    revision = data['revision'],
                    formatted_content = True)

                source = etpConst['install_sources']['unknown']
                if dep_package_id == package_id:
                    source = etpConst['install_sources']['user']
                # required in order to make mirror URL to be
                # resolved correctly, and, to make only the main
                # package to be pulled in for install.
                dest_repo.storeInstalledPackage(dest_package_id,
                    self._repo_id, source = source)

            dest_repo._setSetting("plain_packages",
                self._mirror_urls_str)
            dest_repo._setSetting("arch", repo_arch)
            dest_repo.commit()
            dest_repo.close()
            dest_repo = None

            # ready to bzip2
            compressed_fd = None
            compressed_tmp_path = None
            try:
                compressed_fd, compressed_tmp_path = tempfile.mkstemp(
                        suffix="repo-webinst-gen")
                entropy.tools.compress_file(tmp_repo_path,
                    compressed_tmp_path, bz2.BZ2File)
                try:
                    os.rename(compressed_tmp_path, tmp_repo_path)
                except OSError as err:
                    if err.errno != errno.EXDEV:
                        raise
                    shutil.move(compressed_tmp_path, tmp_repo_path)
            finally:
                if compressed_fd is not None:
                    os.close(compressed_fd)

            tmp_etp_path = etp_path + "._etp_work"
            with open(tmp_etp_path, "wb") as etp_f:
                etp_f.write(WebinstallGenerator.SHELL_PREAMBLE)
                with open(tmp_repo_path, "rb") as bin_f:
                    while True:
                        chunk = bin_f.read(16384)
                        if not chunk:
                            break
                        etp_f.write(chunk)
                    bin_f.flush()
                etp_f.flush()
            os.rename(tmp_etp_path, etp_path)

        finally:
            if compressed_fd is not None:
                try:
                    os.close(compressed_fd)
                except OSError:
                    pass
            if compressed_tmp_path is not None:
                try:
                    os.remove(compressed_tmp_path)
                except (IOError, OSError):
                    pass
            if dest_repo is not None:
                dest_repo.close()
            try:
                os.close(tmp_fd)
            except (OSError, IOError):
                pass
            try:
                os.remove(tmp_repo_path)
            except (OSError, IOError):
                pass

    def sync(self):

        self.output(purple("Scanning..."),
            header = teal(" @@ "),
            importance = 1, back = True)

        work_queue, expired_webinstall_files = \
            self._calculate_required_actions()

        if not (work_queue or expired_webinstall_files):
            # nothing to do
            self.output(purple("Nothing to do."),
                header = teal(" @@ "),
                importance = 1)
            return True

        if work_queue:
            self.output(purple("Generating web-install packages..."),
                header = teal(" @@ "),
                importance = 1)

        tmp_repo_orig_path = self._prepare_base_package_repository()
        pkg_data_cache = {}
        max_count = len(work_queue)
        count = 0
        try:

            for package_id, package_path, etp_path in work_queue:

                count += 1
                atom = self._repo.retrieveAtom(package_id)
                self.output("%s: %s" % (purple("generating for"), atom),
                    header = teal(" @@ "),
                    count = (count, max_count),
                    importance = 0,
                    back = True)
                self._generate_webinstall_package(tmp_repo_orig_path,
                    package_id, package_path, etp_path, pkg_data_cache)
                self.output("%s: %s" % (purple("generated"), etp_path),
                    header = teal(" @@ "),
                    count = (count, max_count),
                    importance = 0)

        finally:
            try:
                os.remove(tmp_repo_orig_path)
            except (OSError, IOError):
                pass

            # help the garbage collector
            for key in list(pkg_data_cache.keys()):
                del pkg_data_cache[key]
            pkg_data_cache.clear()
            del pkg_data_cache

        if expired_webinstall_files:
            self._cleanup_expired_files(expired_webinstall_files)

        return True

def _print_help(args):
    app_name = os.path.basename(sys.argv[0])
    print_info("%s - %s" % (blue(app_name),
        teal(_("Repository web-install packages generator tool")),))
    print_info("  %s:\t%s %s" % (
        purple(_("generate packages")),
        brown(app_name),
        darkgreen("generate [--regen] <repository id> <repository file path> <packages dirs [list]> -- [<mirror urls [list]>]"))
    )
    print_info("    %s = %s" % (
        teal("<packages dirs [list]>"),
        _("dirs where package files are storied for repository"),)
    )
    print_info("    %s = %s" % (
        teal("<mirror urls [list]>"),
        _("list of package mirror urls (not mandatory)"),)
    )
    print_info("    %s = %s" % (
        teal("--regen"),
        _("regenerate all the package files"),)
    )
    print_info("  %s:\t\t%s %s" % (purple(_("this help")), brown(app_name),
        darkgreen("help")))
    if not args:
        return 1
    return 0

def _generate(args):

    regenerate = False
    if "--regen" in args:
        regenerate = True
        args.remove("--regen")

    if not args:
        print_error(brown(_("Invalid arguments")))
        return 1

    repository_id = args.pop(0)
    if not entropy.tools.validate_repository_id(repository_id):
        print_error(brown(_("Invalid repository identifier.")))
        return 1
    if not args:
        print_error(brown(_("Invalid Entropy repository file path")))
        return 1

    entropy_repository_path = args.pop(0)
    entropy_repository_path_dir = os.path.dirname(entropy_repository_path)
    if not (os.path.isdir(entropy_repository_path_dir) and \
        os.access(entropy_repository_path_dir, os.W_OK | os.R_OK)):
        print_error(brown(_("Invalid Entropy repository file path")))
        return 1

    packages_dirs = []
    mirror_urls = []
    do_packages_dirs = True
    for arg in args:
        if arg == "--":
            do_packages_dirs = False
            continue
        elif do_packages_dirs:
            packages_dirs.append(arg)
        else:
            mirror_urls.append(arg)

    if not packages_dirs:
        print_error(brown(_("Missing packages directories")))
        return 1

    if not mirror_urls:
        # coupled with ETP_REPOSITORIES_CONF
        print_info(brown(_("Using repositories.conf settings for mirrors")))
        sys_set = SystemSettings()
        mirror_urls = sys_set['repositories']['available'].get(
            repository_id, {}).get('plain_packages', [])

    if not mirror_urls:
        print_error(brown(_("Missing mirror urls")))
        return 1

    for package_dir in packages_dirs:
        if not (os.path.isdir(package_dir) and \
            os.access(package_dir, os.R_OK | os.W_OK)):
            print_error("%s: %s" % (
                brown(_("Insufficient permissions")),
                package_dir,))
            return 1

    lock_map = {}

    # acquire lock
    lock_file = entropy_repository_path + ".webinstall.lock"
    acquired = False
    try:

        acquired = entropy.tools.acquire_lock(lock_file, lock_map)
        if not acquired:
            print_error(brown(_("Another instance is running.")))
            return 1

        repo = GenericRepository(
                dbFile = entropy_repository_path,
                name = repository_id,
                indexing = True,
                readOnly = False,
                xcache = True)
        try:
            repo.validate()
            repo.integrity_check()
        except SystemDatabaseError:
            print_error(brown(_("Invalid repository.")))
            return 1
        # force indexing, if user is not in entropy group, indexing is
        # forced to False.
        repo.setIndexing(True)
        repo.createAllIndexes()

        generator = WebinstallGenerator(repository_id, repo, packages_dirs,
            mirror_urls, regenerate = regenerate)
        sts = generator.sync()
        repo.close()
        if sts:
            return 0
        return 1

    finally:
        if acquired:
            entropy.tools.release_lock(lock_file, lock_map)


if __name__ == "__main__":

    args_map = {
    'generate': _generate,
    'help': _print_help,
    '__fallback__': _print_help,
    }

    argv = sys.argv[1:]

    if not argv:
        argv.append("help")

    cmd, args = argv[0], argv[1:]
    func = args_map.get(cmd, args_map.get("__fallback__"))
    rc = func(args)
    raise SystemExit(rc)