diff --git a/libraries/entropy/client/interfaces.py b/libraries/entropy/client/interfaces/__init__.py similarity index 55% rename from libraries/entropy/client/interfaces.py rename to libraries/entropy/client/interfaces/__init__.py index 008066e1a..48c468fbc 100644 --- a/libraries/entropy/client/interfaces.py +++ b/libraries/entropy/client/interfaces/__init__.py @@ -32,4335 +32,18 @@ from entropy.const import * from entropy.exceptions import * from entropy.i18n import _ from entropy.db import dbapi2, LocalRepository -from entropy.misc import TimeScheduled, Lifo - - -class _ClientLoaders: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.__QA_cache = {} - self.__security_cache = {} - self.__spm_cache = {} - - def closeAllQA(self): - self.__QA_cache.clear() - - def closeAllSecurity(self): - self.__security_cache.clear() - - def Security(self): - chroot = etpConst['systemroot'] - cached = self.Client.__security_cache.get(chroot) - if cached != None: - return cached - from entropy.security import SecurityInterface - cached = SecurityInterface(self) - self.Client.__security_cache[chroot] = cached - return cached - - def QA(self): - chroot = etpConst['systemroot'] - cached = self.__QA_cache.get(chroot) - if cached != None: - return cached - from entropy.qa import QAInterface - cached = QAInterface(self) - self.__QA_cache[chroot] = cached - return cached - - def Triggers(self, *args, **kwargs): - return Trigger(self, *args, **kwargs) - - def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True): - return Repository(self, reponames = reponames, - forceUpdate = forceUpdate, noEquoCheck = noEquoCheck, - fetchSecurity = fetchSecurity) - - def Spm(self): - from entropy.spm import Spm - myroot = etpConst['systemroot'] - cached = self.__spm_cache.get(myroot) - if cached != None: return cached - conn = Spm(self) - self.__spm_cache[myroot] = conn.intf - return conn.intf - - def Package(self): - return Package(self) - -class _ClientCache: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.Cacher = self.Client.Cacher - self.updateProgress = self.Client.updateProgress - self.FileUpdates = self.Client.FileUpdates - self.entropyTools = self.Client.entropyTools - - def validate_repositories_cache(self): - # is the list of repos changed? - cached = self.Cacher.pop(etpCache['repolist']) - if cached == None: - # invalidate matching cache - try: self.repository_move_clear_cache() - except IOError: pass - elif isinstance(cached,tuple): - difflist = [x for x in cached if x not in etpRepositoriesOrder] - for repoid in difflist: - try: self.repository_move_clear_cache(repoid) - except IOError: pass - self.store_repository_list_cache() - - def store_repository_list_cache(self): - self.Cacher.push(etpCache['repolist'],tuple(etpRepositoriesOrder), async = False) - - def generate_cache(self, depcache = True, configcache = True, client_purge = True, install_queue = True): - self.Cacher.stop() - # clean first of all - self.purge_cache(client_purge = client_purge) - if depcache: - self.do_depcache(do_install_queue = install_queue) - if configcache: - self.do_configcache() - self.Cacher.start() - - def do_configcache(self): - self.updateProgress( - darkred(_("Configuration files")), - importance = 2, - type = "warning" - ) - self.updateProgress( - red(_("Scanning hard disk")), - importance = 1, - type = "warning" - ) - self.FileUpdates.scanfs(dcache = False, quiet = True) - self.updateProgress( - darkred(_("Cache generation complete.")), - importance = 2, - type = "info" - ) - - def do_depcache(self, do_install_queue = True): - - self.updateProgress( - darkgreen(_("Resolving metadata")), - importance = 1, - type = "warning" - ) - # we can barely ignore any exception from here - # especially cases where client db does not exist - try: - update, remove, fine = self.Client.calculate_world_updates() - del fine, remove - if do_install_queue: - self.Client.get_install_queue(update, False, False) - self.Client.calculate_available_packages() - except: - pass - - self.updateProgress( - darkred(_("Dependencies cache filled.")), - importance = 2, - type = "warning" - ) - - def purge_cache(self, showProgress = True, client_purge = True): - if self.entropyTools.is_user_in_entropy_group(): - skip = set() - if not client_purge: - skip.add("/"+etpCache['dbMatch']+"/"+etpConst['clientdbid']) # it's ok this way - skip.add("/"+etpCache['dbSearch']+"/"+etpConst['clientdbid']) # it's ok this way - for key in etpCache: - if showProgress: - self.Client.updateProgress( - darkred(_("Cleaning %s => dumps...")) % (etpCache[key],), - importance = 1, - type = "warning", - back = True - ) - self.clear_dump_cache(etpCache[key], skip = skip) - - if showProgress: - self.updateProgress( - darkgreen(_("Cache is now empty.")), - importance = 2, - type = "info" - ) - - def clear_dump_cache(self, dump_name, skip = []): - self.Cacher.sync(wait = True) - dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) - dump_dir = os.path.dirname(dump_path) - #dump_file = os.path.basename(dump_path) - for currentdir, subdirs, files in os.walk(dump_dir): - path = os.path.join(dump_dir,currentdir) - if skip: - found = False - for myskip in skip: - if path.find(myskip) != -1: - found = True - break - if found: continue - for item in files: - if item.endswith(etpConst['cachedumpext']): - item = os.path.join(path,item) - try: os.remove(item) - except OSError: pass - try: - if not os.listdir(path): - os.rmdir(path) - except OSError: - pass - - def update_ugc_cache(self, repository): - if not self.Client.UGC.is_repository_eapi3_aware(repository): - return None - status = True - - votes_dict, err_msg = self.Client.UGC.get_all_votes(repository) - if isinstance(votes_dict,dict): - self.Client.UGC.UGCCache.save_vote_cache(repository, votes_dict) - else: - status = False - - downloads_dict, err_msg = self.Client.UGC.get_all_downloads(repository) - if isinstance(downloads_dict,dict): - self.Client.UGC.UGCCache.save_downloads_cache(repository, downloads_dict) - else: - status = False - return status - - def repository_move_clear_cache(self, repoid = None): - self.clear_dump_cache(etpCache['world_available']) - self.clear_dump_cache(etpCache['world_update']) - self.clear_dump_cache(etpCache['check_package_update']) - self.clear_dump_cache(etpCache['filter_satisfied_deps']) - self.clear_dump_cache(self.Client.atomMatchCacheKey) - self.clear_dump_cache(etpCache['dep_tree']) - if repoid != None: - self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) - self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,)) - - def get_available_packages_chash(self, branch): - # client digest not needed, cache is kept updated - return str(hash("%s%s%s" % ( - self.all_repositories_checksum(), - branch,self.Client.validRepositories,))) - - def all_repositories_checksum(self): - sum_hashes = '' - for repo in self.Client.validRepositories: - try: - dbconn = self.Client.open_repository(repo) - except (RepositoryError): - continue # repo not available - try: - sum_hashes += dbconn.database_checksum() - except self.Client.dbapi2.OperationalError: - pass - return sum_hashes - - def get_available_packages_cache(self, branch = etpConst['branch'], myhash = None): - if myhash == None: myhash = self.get_available_packages_chash(branch) - return self.Cacher.pop("%s%s" % (etpCache['world_available'],myhash)) - - def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None, ignore_spm_downgrades = False): - if self.Client.xcache: - if db_digest == None: db_digest = self.all_repositories_checksum() - c_hash = "%s%s" % (etpCache['world_update'], - self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades),) - disk_cache = self.Cacher.pop(c_hash) - if disk_cache != None: - try: - return disk_cache['r'] - except (KeyError, TypeError): - return None - - def get_world_update_cache_hash(self, db_digest, empty_deps, branch, ignore_spm_downgrades): - return str(hash("%s|%s|%s|%s|%s|%s" % ( - db_digest,empty_deps,self.Client.validRepositories, - etpRepositoriesOrder, branch, ignore_spm_downgrades, - ))) - - -class _ClientDepCalculators: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.Cacher = self.Client.Cacher - self.updateProgress = self.Client.updateProgress - self.entropyTools = self.Client.entropyTools - self.dumpTools = self.Client.dumpTools - self.SystemSettings = self.Client.SystemSettings - self.dbapi2 = self.Client.dbapi2 - - def dependencies_test(self, dbconn = None): - - if dbconn == None: - dbconn = self.Client.clientDbconn - # get all the installed packages - installedPackages = dbconn.listAllIdpackages() - - deps_not_matched = set() - # now look - length = len(installedPackages) - count = 0 - for xidpackage in installedPackages: - count += 1 - atom = dbconn.retrieveAtom(xidpackage) - self.updateProgress( - darkgreen(_("Checking %s") % (bold(atom),)), - importance = 0, - type = "info", - back = True, - count = (count,length), - header = darkred(" @@ ") - ) - - xdeps = dbconn.retrieveDependencies(xidpackage) - needed_deps = set() - for xdep in xdeps: - xmatch = dbconn.atomMatch(xdep) - if xmatch[0] == -1: - needed_deps.add(xdep) - - deps_not_matched |= needed_deps - - return deps_not_matched - - def find_belonging_dependency(self, matched_atoms): - crying_atoms = set() - for atom in matched_atoms: - for repo in self.Client.validRepositories: - rdbconn = self.Client.open_repository(repo) - riddep = rdbconn.searchDependency(atom) - if riddep != -1: - ridpackages = rdbconn.searchIdpackageFromIddependency(riddep) - for i in ridpackages: - i,r = rdbconn.idpackageValidator(i) - if i == -1: - continue - iatom = rdbconn.retrieveAtom(i) - crying_atoms.add((iatom,repo)) - return crying_atoms - - def __handle_multi_repo_matches(self, results, extended_results, valid_repos, server_inst): - - packageInformation = {} - versionInformation = {} - # .tbz2 repos have always the precedence, so if we find them, - # we should second what user wants, installing his tbz2 - tbz2repos = [x for x in results if x.endswith(etpConst['packagesext'])] - if tbz2repos: - del tbz2repos - newrepos = results.copy() - for x in newrepos: - if x.endswith(etpConst['packagesext']): - continue - del results[x] - - version_duplicates = set() - versions = set() - for repo in results: - packageInformation[repo] = {} - if extended_results: - version = results[repo][1] - packageInformation[repo]['versiontag'] = results[repo][2] - packageInformation[repo]['revision'] = results[repo][3] - else: - dbconn = self.__atom_match_open_db(repo, server_inst) - packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(results[repo]) - packageInformation[repo]['revision'] = dbconn.retrieveRevision(results[repo]) - version = dbconn.retrieveVersion(results[repo]) - packageInformation[repo]['version'] = version - versionInformation[version] = repo - if version in versions: - version_duplicates.add(version) - versions.add(version) - - newerVersion = self.entropyTools.getNewerVersion(list(versions))[0] - # if no duplicates are found or newer version is not in duplicates we're done - if (not version_duplicates) or (newerVersion not in version_duplicates): - reponame = versionInformation.get(newerVersion) - return (results[reponame],reponame) - - # we have two repositories with >two packages with the same version - # check package tag - - conflictingEntries = {} - tags_duplicates = set() - tags = set() - tagsInfo = {} - for repo in packageInformation: - if packageInformation[repo]['version'] != newerVersion: - continue - conflictingEntries[repo] = {} - versiontag = packageInformation[repo]['versiontag'] - if versiontag in tags: - tags_duplicates.add(versiontag) - tags.add(versiontag) - tagsInfo[versiontag] = repo - conflictingEntries[repo]['versiontag'] = versiontag - conflictingEntries[repo]['revision'] = packageInformation[repo]['revision'] - - # tags will always be != [] - newerTag = sorted(list(tags), reverse = True)[0] - if newerTag not in tags_duplicates: - reponame = tagsInfo.get(newerTag) - return (results[reponame],reponame) - - # in this case, we have >two packages with the same version and tag - # check package revision - - conflictingRevisions = {} - revisions = set() - revisions_duplicates = set() - revisionInfo = {} - for repo in conflictingEntries: - if conflictingEntries[repo]['versiontag'] == newerTag: - conflictingRevisions[repo] = {} - versionrev = conflictingEntries[repo]['revision'] - if versionrev in revisions: - revisions_duplicates.add(versionrev) - revisions.add(versionrev) - revisionInfo[versionrev] = repo - conflictingRevisions[repo]['revision'] = versionrev - - newerRevision = max(revisions) - if newerRevision not in revisions_duplicates: - reponame = revisionInfo.get(newerRevision) - return (results[reponame],reponame) - - # final step, in this case we have >two packages with the same version, tag and revision - # get the repository with the biggest priority - - for reponame in valid_repos: - if reponame in conflictingRevisions: - return (results[reponame],reponame) - - def __validate_atom_match_cache(self, cached_obj, multiMatch, extendedResults, multiRepo, server_inst): - - data, rc = cached_obj - if rc == 1: return cached_obj - - if multiRepo or multiMatch: - matches = data # set([(14789, 'sabayonlinux.org'), (14479, 'sabayonlinux.org')]) - if extendedResults: - # set([((14789, u'3.3.8b', u'', 0), 'sabayonlinux.org')]) - matches = [(x[0][0],x[1],) for x in data] - for m_id, m_repo in matches: - m_db = self.__atom_match_open_db(m_repo, server_inst) - if not m_db.isIDPackageAvailable(m_id): return None - else: - m_id, m_repo = cached_obj # (14479, 'sabayonlinux.org') - if extendedResults: - # ((14479, u'4.4.2', u'', 0), 'sabayonlinux.org') - m_id, m_repo = cached_obj[0][0],cached_obj[1] - m_db = self.__atom_match_open_db(m_repo, server_inst) - if not m_db.isIDPackageAvailable(m_id): return None - - return cached_obj - - def __atom_match_open_db(self, repoid, server_inst): - if server_inst != None: - dbconn = server_inst.openServerDatabase(just_reading = True, repo = repoid) - else: - dbconn = self.Client.open_repository(repoid) - return dbconn - - def atom_match(self, atom, caseSensitive = True, matchSlot = None, - matchBranches = (), matchTag = None, packagesFilter = True, - multiMatch = False, multiRepo = False, matchRevision = None, - matchRepo = None, server_repos = [], serverInstance = None, - extendedResults = False, useCache = True): - - # support match in repository from shell - # atom@repo1,repo2,repo3 - atom, repos = self.entropyTools.dep_get_match_in_repos(atom) - if (matchRepo == None) and (repos != None): - matchRepo = repos - - u_hash = "" - m_hash = "" - k_ms = "//" - k_mt = "@#@" - k_mr = "-1" - if isinstance(matchRepo,(list,tuple,set,)): u_hash = hash(frozenset(matchRepo)) - if isinstance(matchBranches,(list,tuple,set,)): m_hash = hash(frozenset(matchBranches)) - if isinstance(matchSlot,basestring): k_ms = matchSlot - if isinstance(matchTag,basestring): k_mt = matchTag - if isinstance(matchRevision,basestring): k_mr = matchRevision - - c_hash = "|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" % ( - atom,k_ms,k_mt,hash(packagesFilter), - hash(frozenset(self.Client.validRepositories)), - hash(frozenset(etpRepositories)), - hash(multiMatch),hash(multiRepo),hash(caseSensitive), - k_mr,hash(extendedResults), - u_hash, m_hash - ) - c_hash = "%s%s" % (self.Client.atomMatchCacheKey,hash(c_hash),) - - if self.Client.xcache and useCache: - cached = self.Cacher.pop(c_hash) - if cached != None: - try: - cached = self.__validate_atom_match_cache(cached, multiMatch, extendedResults, multiRepo, serverInstance) - except (TypeError,ValueError,IndexError,KeyError,): - cached = None - if cached != None: - return cached - - if server_repos: - if not serverInstance: - t = _("server_repos needs serverInstance") - raise IncorrectParameter("IncorrectParameter: %s" % (t,)) - valid_repos = server_repos[:] - else: - valid_repos = self.Client.validRepositories - if matchRepo and (type(matchRepo) in (list,tuple,set)): - valid_repos = list(matchRepo) - - repoResults = {} - for repo in valid_repos: - - # search - dbconn = self.__atom_match_open_db(repo, serverInstance) - use_cache = useCache - while 1: - try: - query_data, query_rc = dbconn.atomMatch( - atom, - caseSensitive = caseSensitive, - matchSlot = matchSlot, - matchBranches = matchBranches, - matchTag = matchTag, - packagesFilter = packagesFilter, - matchRevision = matchRevision, - extendedResults = extendedResults, - useCache = use_cache - ) - if query_rc == 0: - # package found, add to our dictionary - if extendedResults: - repoResults[repo] = (query_data[0],query_data[2],query_data[3],query_data[4]) - else: - repoResults[repo] = query_data - except TypeError: - if not use_cache: - raise - use_cache = False - continue - break - - dbpkginfo = (-1,1) - if extendedResults: - dbpkginfo = ((-1,None,None,None),1) - - if multiRepo and repoResults: - - data = set() - for repoid in repoResults: - data.add((repoResults[repoid],repoid)) - dbpkginfo = (data,0) - - elif len(repoResults) == 1: - # one result found - repo = repoResults.keys()[0] - dbpkginfo = (repoResults[repo],repo) - - elif len(repoResults) > 1: - - # we have to decide which version should be taken - mypkginfo = self.__handle_multi_repo_matches(repoResults, extendedResults, valid_repos, serverInstance) - if mypkginfo != None: dbpkginfo = mypkginfo - - # multimatch support - if multiMatch: - - if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure - if multiRepo: - data = set() - for q_id,q_repo in dbpkginfo[0]: - dbconn = self.__atom_match_open_db(q_repo, serverInstance) - query_data, query_rc = dbconn.atomMatch( - atom, - caseSensitive = caseSensitive, - matchSlot = matchSlot, - matchBranches = matchBranches, - matchTag = matchTag, - packagesFilter = packagesFilter, - multiMatch = True, - extendedResults = extendedResults - ) - if extendedResults: - for item in query_data: - data.add(((item[0],item[2],item[3],item[4]),q_repo)) - else: - for x in query_data: data.add((x,q_repo)) - dbpkginfo = (data,0) - else: - dbconn = self.__atom_match_open_db(dbpkginfo[1], serverInstance) - query_data, query_rc = dbconn.atomMatch( - atom, - caseSensitive = caseSensitive, - matchSlot = matchSlot, - matchBranches = matchBranches, - matchTag = matchTag, - packagesFilter = packagesFilter, - multiMatch = True, - extendedResults = extendedResults - ) - if extendedResults: - dbpkginfo = (set([((x[0],x[2],x[3],x[4]),dbpkginfo[1]) for x in query_data]),0) - else: - dbpkginfo = (set([(x,dbpkginfo[1]) for x in query_data]),0) - - if self.Client.xcache and useCache: - self.Cacher.push(c_hash,dbpkginfo) - - return dbpkginfo - - # expands package sets, and in future something more perhaps - def packages_expand(self, packages): - new_packages = [] - - for pkg_id in range(len(packages)): - package = packages[pkg_id] - - # expand package sets - if package.startswith(etpConst['packagesetprefix']): - set_pkgs = sorted(list(self.package_set_expand(package, raise_exceptions = False))) - new_packages.extend([x for x in set_pkgs if x not in packages]) # atomMatch below will filter dupies - else: - new_packages.append(package) - - return new_packages - - def package_set_expand(self, package_set, raise_exceptions = True): - - max_recursion_level = 50 - recursion_level = 0 - - def do_expand(myset, recursion_level, max_recursion_level): - recursion_level += 1 - if recursion_level > max_recursion_level: - raise InvalidPackageSet('InvalidPackageSet: corrupted, too many recursions: %s' % (myset,)) - set_data, set_rc = self.package_set_match(myset[len(etpConst['packagesetprefix']):]) - if not set_rc: - raise InvalidPackageSet('InvalidPackageSet: not found: %s' % (myset,)) - (set_from, package_set, mydata,) = set_data - - mypkgs = set() - for fset in mydata: # recursively - if fset.startswith(etpConst['packagesetprefix']): - mypkgs |= do_expand(fset, recursion_level, max_recursion_level) - else: - mypkgs.add(fset) - - return mypkgs - - if not package_set.startswith(etpConst['packagesetprefix']): - package_set = "%s%s" % (etpConst['packagesetprefix'],package_set,) - - try: - mylist = do_expand(package_set, recursion_level, max_recursion_level) - except InvalidPackageSet: - if raise_exceptions: raise - mylist = set() - - return mylist - - def package_set_list(self, server_repos = [], serverInstance = None, matchRepo = None): - return self.package_set_match('', matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] - - def package_set_search(self, package_set, server_repos = [], serverInstance = None, matchRepo = None): - # search support - if package_set == '*': package_set = '' - return self.package_set_match(package_set, matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] - - def __package_set_match_open_db(self, repoid, server_inst): - if server_inst != None: - dbconn = server_inst.openServerDatabase(just_reading = True, repo = repoid) - else: - dbconn = self.Client.open_repository(repoid) - return dbconn - - def package_set_match(self, package_set, multiMatch = False, matchRepo = None, server_repos = [], serverInstance = None, search = False): - - # support match in repository from shell - # set@repo1,repo2,repo3 - package_set, repos = self.entropyTools.dep_get_match_in_repos(package_set) - if (matchRepo == None) and (repos != None): - matchRepo = repos - - if server_repos: - if not serverInstance: - t = _("server_repos needs serverInstance") - raise IncorrectParameter("IncorrectParameter: %s" % (t,)) - valid_repos = server_repos[:] - else: - valid_repos = self.Client.validRepositories - - if matchRepo and (type(matchRepo) in (list,tuple,set)): - valid_repos = list(matchRepo) - - # if we search, we return all the matches available - if search: multiMatch = True - - set_data = [] - - while 1: - - # check inside SystemSettings - if not server_repos: - if search: - mysets = [x for x in self.SystemSettings['system_package_sets'].keys() if (x.find(package_set) != -1)] - for myset in mysets: - mydata = self.SystemSettings['system_package_sets'].get(myset) - set_data.append((etpConst['userpackagesetsid'], unicode(myset), mydata.copy(),)) - else: - mydata = self.SystemSettings['system_package_sets'].get(package_set) - if mydata != None: - set_data.append((etpConst['userpackagesetsid'], unicode(package_set), mydata,)) - if not multiMatch: break - - for repoid in valid_repos: - dbconn = self.__package_set_match_open_db(repoid, serverInstance) - if search: - mysets = dbconn.searchSets(package_set) - for myset in mysets: - mydata = dbconn.retrievePackageSet(myset) - set_data.append((repoid, myset, mydata.copy(),)) - else: - mydata = dbconn.retrievePackageSet(package_set) - if mydata: set_data.append((repoid, package_set, mydata,)) - if not multiMatch: break - - break - - if not set_data: return (),False - if multiMatch: return set_data,True - return set_data.pop(0),True - - def get_unsatisfied_dependencies(self, dependencies, deep_deps = False, depcache = None): - - if self.Client.xcache: - c_data = sorted(dependencies) - client_checksum = self.Client.clientDbconn.database_checksum() - c_hash = hash("%s|%s|%s" % (c_data,deep_deps,client_checksum,)) - c_hash = "%s%s" % (etpCache['filter_satisfied_deps'],c_hash,) - cached = self.dumpTools.loadobj(c_hash) - if cached != None: return cached - - if not isinstance(depcache,dict): - depcache = {} - - cdb_am = self.Client.clientDbconn.atomMatch - am = self.atom_match - open_repo = self.Client.open_repository - intf_error = self.dbapi2.InterfaceError - cdb_getversioning = self.Client.clientDbconn.getVersioningData - #cdb_retrieveneededraw = self.clientDbconn.retrieveNeededRaw - etp_cmp = self.entropyTools.entropyCompareVersions - etp_get_rev = self.entropyTools.dep_get_entropy_revision - #do_needed_check = False - - def fm_dep(dependency): - - cached = depcache.get(dependency) - if cached != None: return cached - - ### conflict - if dependency.startswith("!"): - idpackage,rc = cdb_am(dependency[1:]) - if idpackage != -1: - depcache[dependency] = dependency - return dependency - depcache[dependency] = 0 - return 0 - - c_id,c_rc = cdb_am(dependency) - if c_id == -1: - depcache[dependency] = dependency - return dependency - - #if not deep_deps and not do_needed_check: - # depcache[dependency] = 0 - # return 0 - - r_id,r_repo = am(dependency) - if r_id == -1: - depcache[dependency] = dependency - return dependency - - #if do_needed_check: - # dbconn = open_repo(r_repo) - # installed_needed = cdb_retrieveneededraw(c_id) - # repo_needed = dbconn.retrieveNeededRaw(r_id) - # if installed_needed != repo_needed: - # return dependency - # #elif not deep_deps: - # # return 0 - - dbconn = open_repo(r_repo) - try: - repo_pkgver, repo_pkgtag, repo_pkgrev = dbconn.getVersioningData(r_id) - except (intf_error,TypeError,): - # package entry is broken - return dependency - - try: - installedVer, installedTag, installedRev = cdb_getversioning(c_id) - except TypeError: # corrupted entry? - installedVer = "0" - installedTag = '' - installedRev = 0 - - # support for app-foo/foo-123~-1 - # -1 revision means, always pull the latest - do_deep = deep_deps - if not do_deep: - string_rev = etp_get_rev(dependency) - if string_rev == -1: - do_deep = True - - vcmp = etp_cmp((repo_pkgver,repo_pkgtag,repo_pkgrev,), (installedVer,installedTag,installedRev,)) - if vcmp != 0: - if not do_deep and ((repo_pkgver,repo_pkgtag,) == (installedVer,installedTag,)) and (repo_pkgrev != installedRev): - depcache[dependency] = 0 - return 0 - depcache[dependency] = dependency - return dependency - depcache[dependency] = 0 - return 0 - - unsatisfied = map(fm_dep,dependencies) - unsatisfied = set([x for x in unsatisfied if x != 0]) - - if self.Client.xcache: - self.Cacher.push(c_hash,unsatisfied) - - return unsatisfied - - def get_masked_packages_tree(self, match, atoms = False, flat = False, matchfilter = None): - - if not isinstance(matchfilter,set): - matchfilter = set() - - maskedtree = {} - mybuffer = Lifo() - depcache = set() - treelevel = -1 - - match_id, match_repo = match - - mydbconn = self.Client.open_repository(match_repo) - myatom = mydbconn.retrieveAtom(match_id) - idpackage, idreason = mydbconn.idpackageValidator(match_id) - if idpackage == -1: - treelevel += 1 - if atoms: - mydict = {myatom: idreason,} - else: - mydict = {match: idreason,} - if flat: - maskedtree.update(mydict) - else: - maskedtree[treelevel] = mydict - - mydeps = mydbconn.retrieveDependencies(match_id) - for mydep in mydeps: mybuffer.push(mydep) - mydep = mybuffer.pop() - - open_db = self.Client.open_repository - am = self.atom_match - while mydep: - - if mydep in depcache: - mydep = mybuffer.pop() - continue - depcache.add(mydep) - - idpackage, repoid = am(mydep) - if (idpackage, repoid) in matchfilter: - mydep = mybuffer.pop() - continue - - if idpackage != -1: - # doing even here because atomMatch with packagesFilter = False can pull - # something different - matchfilter.add((idpackage, repoid)) - - # collect masked - if idpackage == -1: - idpackage, repoid = am(mydep, packagesFilter = False) - if idpackage != -1: - treelevel += 1 - if not maskedtree.has_key(treelevel) and not flat: - maskedtree[treelevel] = {} - dbconn = open_db(repoid) - vidpackage, idreason = dbconn.idpackageValidator(idpackage) - if atoms: - mydict = {dbconn.retrieveAtom(idpackage): idreason} - else: - mydict = {(idpackage,repoid): idreason} - if flat: maskedtree.update(mydict) - else: maskedtree[treelevel].update(mydict) - - # push its dep into the buffer - if idpackage != -1: - matchfilter.add((idpackage, repoid)) - dbconn = open_db(repoid) - owndeps = dbconn.retrieveDependencies(idpackage) - for owndep in owndeps: - mybuffer.push(owndep) - - mydep = mybuffer.pop() - - return maskedtree - - - def generate_dependency_tree(self, - matched_atom, empty_deps = False, deep_deps = False, matchfilter = None, - flat = False, filter_unsat_cache = None, treecache = None, keyslotcache = None): - - if not isinstance(matchfilter,set): - matchfilter = set() - if not isinstance(filter_unsat_cache,dict): - filter_unsat_cache = {} - if not isinstance(treecache,set): - treecache = set() - if not isinstance(keyslotcache,set): - keyslotcache = set() - - mydbconn = self.Client.open_repository(matched_atom[1]) - myatom = mydbconn.retrieveAtom(matched_atom[0]) - - # caches - # special events - deps_not_found = set() - conflicts = set() - - mydep = (1,myatom) - mybuffer = Lifo() - deptree = set() - if matched_atom not in matchfilter: - deptree.add((1,matched_atom)) - - virgin = True - open_repo = self.Client.open_repository - atom_match = self.atom_match - cdb_atom_match = self.Client.clientDbconn.atomMatch - lookup_conflict_replacement = self._lookup_conflict_replacement - lookup_library_breakages = self._lookup_library_breakages - lookup_inverse_dependencies = self._lookup_inverse_dependencies - get_unsatisfied_deps = self.get_unsatisfied_dependencies - - def my_dep_filter(x): - if x in treecache: return False - if tuple(x.split(":")) in keyslotcache: return False - return True - - while mydep: - - dep_level, dep_atom = mydep - - # already analyzed in this call - if dep_atom in treecache: - mydep = mybuffer.pop() - continue - treecache.add(dep_atom) - - if dep_atom == None: # corrupted entry - mydep = mybuffer.pop() - continue - - # conflicts - if dep_atom[0] == "!": - c_idpackage, xst = cdb_atom_match(dep_atom[1:]) - if c_idpackage != -1: - myreplacement = lookup_conflict_replacement(dep_atom[1:], c_idpackage, deep_deps = deep_deps) - if (myreplacement != None) and (myreplacement not in treecache): - mybuffer.push((dep_level+1,myreplacement)) - else: - conflicts.add(c_idpackage) - mydep = mybuffer.pop() - continue - - # atom found? - if virgin: - virgin = False - m_idpackage, m_repo = matched_atom - dbconn = open_repo(m_repo) - myidpackage, idreason = dbconn.idpackageValidator(m_idpackage) - if myidpackage == -1: m_idpackage = -1 - else: - m_idpackage, m_repo = atom_match(dep_atom) - if m_idpackage == -1: - deps_not_found.add(dep_atom) - mydep = mybuffer.pop() - continue - - # check if atom has been already pulled in - matchdb = open_repo(m_repo) - matchatom = matchdb.retrieveAtom(m_idpackage) - matchkey, matchslot = matchdb.retrieveKeySlot(m_idpackage) - if (dep_atom != matchatom) and (matchatom in treecache): - mydep = mybuffer.pop() - continue - - treecache.add(matchatom) - - # check if key + slot has been already pulled in - if (matchslot,matchkey) in keyslotcache: - mydep = mybuffer.pop() - continue - else: - keyslotcache.add((matchslot,matchkey)) - - match = (m_idpackage, m_repo,) - # result already analyzed? - if match in matchfilter: - mydep = mybuffer.pop() - continue - - # already analyzed by the calling function - if match in matchfilter: - mydep = mybuffer.pop() - continue - matchfilter.add(match) - - treedepth = dep_level+1 - - # all checks passed, well done - matchfilter.add(match) - deptree.add((dep_level,match)) # add match - - # extra hooks - cm_idpackage, cm_result = cdb_atom_match(matchkey, matchSlot = matchslot) - if cm_idpackage != -1: - broken_atoms = lookup_library_breakages(match, (cm_idpackage, cm_result,), deep_deps = deep_deps) - inverse_deps = lookup_inverse_dependencies(match, (cm_idpackage, cm_result,)) - if inverse_deps: - deptree.remove((dep_level,match)) - for ikey,islot in inverse_deps: - iks_str = '%s:%s' % (ikey,islot,) - if ((ikey,islot) not in keyslotcache) and (iks_str not in treecache): - mybuffer.push((dep_level,iks_str)) - keyslotcache.add((ikey,islot)) - deptree.add((treedepth,match)) - treedepth += 1 - for x in broken_atoms: - if (tuple(x.split(":")) not in keyslotcache) and (x not in treecache): - mybuffer.push((treedepth,x)) - - myundeps = filter(my_dep_filter,matchdb.retrieveDependenciesList(m_idpackage)) - if not empty_deps: - myundeps = filter(my_dep_filter,get_unsatisfied_deps(myundeps, deep_deps, depcache = filter_unsat_cache)) - - # PDEPENDs support - if myundeps: - post_deps = [x for x in matchdb.retrievePostDependencies(m_idpackage) if x in myundeps] - myundeps = [x for x in myundeps if x not in post_deps] - for x in post_deps: mybuffer.push((-1,x)) # always after the package itself - - for x in myundeps: mybuffer.push((treedepth,x)) - mydep = mybuffer.pop() - - if deps_not_found: - return list(deps_not_found),-2 - - if flat: return [x[1] for x in deptree],0 - - newdeptree = {} - for key,item in deptree: - if key not in newdeptree: newdeptree[key] = set() - newdeptree[key].add(item) - # conflicts - newdeptree[0] = conflicts - - return newdeptree,0 # note: newtree[0] contains possible conflicts - - - def _lookup_system_mask_repository_deps(self): - - data = self.SystemSettings['repos_system_mask'] - if not data: return [] - mydata = [] - cached_items = set() - for atom in data: - mymatch = self.atom_match(atom) - if mymatch[0] == -1: # ignore missing ones intentionally - continue - if mymatch in cached_items: - continue - if mymatch not in mydata: - # check if not found - myaction = self.Client.get_package_action(mymatch) - # only if the package is not installed - if myaction == 1: mydata.append(mymatch) - cached_items.add(mymatch) - return mydata - - def _lookup_conflict_replacement(self, conflict_atom, client_idpackage, deep_deps): - if self.entropyTools.isjustname(conflict_atom): - return None - conflict_match = self.atom_match(conflict_atom) - mykey, myslot = self.Client.clientDbconn.retrieveKeySlot(client_idpackage) - new_match = self.atom_match(mykey, matchSlot = myslot) - if (conflict_match == new_match) or (new_match[1] == 1): - return None - action = self.Client.get_package_action(new_match) - if (action == 0) and (not deep_deps): - return None - return "%s:%s" % (mykey,myslot,) - - def _lookup_inverse_dependencies(self, match, clientmatch): - - cmpstat = self.Client.get_package_action(match) - if cmpstat == 0: return set() - - keyslots = set() - mydepends = self.Client.clientDbconn.retrieveDepends(clientmatch[0]) - am = self.atom_match - cdb_rdeps = self.Client.clientDbconn.retrieveDependencies - cdb_rks = self.Client.clientDbconn.retrieveKeySlot - gpa = self.Client.get_package_action - keyslots_cache = set() - match_cache = {} - - for idpackage in mydepends: - try: - key, slot = cdb_rks(idpackage) - except TypeError: - continue - if (key,slot) in keyslots_cache: continue - keyslots_cache.add((key,slot)) - if (key,slot) in keyslots: continue - # grab its deps - mydeps = cdb_rdeps(idpackage) - found = False - for mydep in mydeps: - mymatch = match_cache.get(mydep, 0) - if mymatch == 0: - mymatch = am(mydep) - match_cache[mydep] = mymatch - if mymatch == match: - found = True - break - if not found: - mymatch = am(key, matchSlot = slot) - if mymatch[0] == -1: continue - cmpstat = gpa(mymatch) - if cmpstat == 0: continue - keyslots.add((key,slot)) - - return keyslots - - def __get_lib_breaks_client_and_repo_side(self, match_db, match_idpackage, client_idpackage): - repo_needed = match_db.retrieveNeeded(match_idpackage, extended = True, format = True) - client_needed = self.Client.clientDbconn.retrieveNeeded(client_idpackage, extended = True, format = True) - repo_split = [x.split(".so")[0] for x in repo_needed] - client_split = [x.split(".so")[0] for x in client_needed] - client_side = [x for x in client_needed if (x not in repo_needed) and (x.split(".so")[0] in repo_split)] - repo_side = [x for x in repo_needed if (x not in client_needed) and (x.split(".so")[0] in client_split)] - return repo_needed, client_side, repo_side - - def _lookup_library_breakages(self, match, clientmatch, deep_deps = False): - - # there is no need to update this cache when "match" will be installed, because at that point - # clientmatch[0] will differ. - c_hash = "%s|%s|%s" % (hash(tuple(match)),hash(deep_deps),hash(tuple(clientmatch)),) - c_hash = "%s%s" % (etpCache['library_breakage'],hash(c_hash),) - if self.Client.xcache: - cached = self.Cacher.pop(c_hash) - if cached != None: return cached - - # these should be pulled in before - repo_atoms = set() - # these can be pulled in after - client_atoms = set() - - matchdb = self.Client.open_repository(match[1]) - reponeeded, client_side, repo_side = self.__get_lib_breaks_client_and_repo_side(matchdb, - match[0], clientmatch[0]) - - # all the packages in client_side should be pulled in and updated - client_idpackages = set() - for needed in client_side: client_idpackages |= self.Client.clientDbconn.searchNeeded(needed) - - client_keyslots = set() - def mymf(idpackage): - if idpackage == clientmatch[0]: return 0 - return self.Client.clientDbconn.retrieveKeySlot(idpackage) - client_keyslots = set([x for x in map(mymf,client_idpackages) if x != 0]) - - # all the packages in repo_side should be pulled in too - repodata = {} - for needed in repo_side: - repodata[needed] = reponeeded[needed] - del repo_side,reponeeded - - repo_dependencies = matchdb.retrieveDependencies(match[0]) - matched_deps = set() - matched_repos = set() - for dependency in repo_dependencies: - depmatch = self.atom_match(dependency) - if depmatch[0] == -1: - continue - matched_repos.add(depmatch[1]) - matched_deps.add(depmatch) - - matched_repos = [x for x in etpRepositoriesOrder if x in matched_repos] - found_matches = set() - for needed in repodata: - for myrepo in matched_repos: - mydbc = self.Client.open_repository(myrepo) - solved_needed = mydbc.resolveNeeded(needed, elfclass = repodata[needed]) - found = False - for idpackage,myfile in solved_needed: - x = (idpackage,myrepo) - if x in matched_deps: - found_matches.add(x) - found = True - break - if found: - break - - for idpackage,repo in found_matches: - if not deep_deps: - cmpstat = self.Client.get_package_action((idpackage,repo)) - if cmpstat == 0: - continue - mydbc = self.Client.open_repository(repo) - repo_atoms.add(mydbc.retrieveAtom(idpackage)) - - for key, slot in client_keyslots: - idpackage, repo = self.atom_match(key, matchSlot = slot) - if idpackage == -1: - continue - if not deep_deps: - cmpstat = self.Client.get_package_action((idpackage, repo)) - if cmpstat == 0: - continue - mydbc = self.Client.open_repository(repo) - client_atoms.add(mydbc.retrieveAtom(idpackage)) - - client_atoms |= repo_atoms - - if self.Client.xcache: - self.Cacher.push(c_hash,client_atoms) - - return client_atoms - - - def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False, quiet = False): - - c_hash = "%s%s" % (etpCache['dep_tree'],hash("%s|%s|%s|%s" % ( - hash(frozenset(sorted(matched_atoms))),hash(empty_deps), - hash(deep_deps),self.Client.clientDbconn.database_checksum(), - )),) - if self.Client.xcache: - cached = self.Cacher.pop(c_hash) - if cached != None: return cached - - deptree = {} - deptree[0] = set() - - atomlen = len(matched_atoms); count = 0 - error_generated = 0 - error_tree = set() - - # check if there are repositories needing some mandatory packages - forced_matches = self._lookup_system_mask_repository_deps() - if forced_matches: - if isinstance(matched_atoms, list): - matched_atoms = forced_matches + [x for x in matched_atoms if x not in forced_matches] - elif isinstance(matched_atoms, set): # we cannot do anything about the order here - matched_atoms |= set(forced_matches) - - sort_dep_text = _("Sorting dependencies") - filter_unsat_cache = {} - treecache = set() - keyslotcache = set() - matchfilter = set() - for matched_atom in matched_atoms: - - if not quiet: - count += 1 - if (count%10 == 0) or (count == atomlen) or (count == 1): - self.updateProgress(sort_dep_text, importance = 0, type = "info", - back = True, header = ":: ", footer = " ::", - percent = True, count = (count,atomlen)) - - if matched_atom in matchfilter: continue - newtree, result = self.generate_dependency_tree( - matched_atom, empty_deps, deep_deps, - matchfilter = matchfilter, filter_unsat_cache = filter_unsat_cache, treecache = treecache, - keyslotcache = keyslotcache - ) - - if result == -2: # deps not found - error_generated = -2 - error_tree |= set(newtree) # it is a list, we convert it into set and update error_tree - elif (result != 0): - return newtree, result - elif newtree: - # add conflicts - max_parent_key = max(deptree) - deptree[0] |= newtree.pop(0) - levelcount = 0 - for mylevel in sorted(newtree.keys(), reverse = True): - levelcount += 1 - deptree[max_parent_key+levelcount] = newtree.get(mylevel) - - if error_generated != 0: - return error_tree,error_generated - - if self.Client.xcache: - self.Cacher.push(c_hash,(deptree,0)) - - return deptree,0 - - def _filter_depends_multimatched_atoms(self, idpackage, depends, monotree): - remove_depends = set() - for d_idpackage in depends: - mydeps = self.Client.clientDbconn.retrieveDependencies(d_idpackage) - for mydep in mydeps: - matches, rslt = self.Client.clientDbconn.atomMatch(mydep, multiMatch = True) - if rslt == 1: continue - if idpackage in matches and len(matches) > 1: - # are all in depends? - for mymatch in matches: - if mymatch not in depends and mymatch not in monotree: - remove_depends.add(d_idpackage) - break - depends -= remove_depends - return depends - - - def generate_depends_tree(self, idpackages, deep = False): - - c_hash = "%s%s" % (etpCache['depends_tree'],hash("%s|%s" % (hash(tuple(sorted(idpackages))),hash(deep),),),) - if self.Client.xcache: - cached = self.Cacher.pop(c_hash) - if cached != None: return cached - - dependscache = set() - treeview = set(idpackages) - treelevel = set(idpackages) - tree = {} - treedepth = 0 # I start from level 1 because level 0 is idpackages itself - tree[treedepth] = set(idpackages) - monotree = set(idpackages) # monodimensional tree - - # check if dependstable is sane before beginning - self.Client.clientDbconn.retrieveDepends(idpackages[0]) - count = 0 - - rem_dep_text = _("Calculating removable depends of") - while 1: - treedepth += 1 - tree[treedepth] = set() - for idpackage in treelevel: - - count += 1 - p_atom = self.Client.clientDbconn.retrieveAtom(idpackage) - self.updateProgress( - blue(rem_dep_text + " %s" % (red(p_atom),)), - importance = 0, - type = "info", - back = True, - header = '|/-\\'[count%4]+" " - ) - - systempkg = not self.Client.validate_package_removal(idpackage) - if (idpackage in dependscache) or systempkg: - if idpackage in treeview: - treeview.remove(idpackage) - continue - - # obtain its depends - depends = self.Client.clientDbconn.retrieveDepends(idpackage) - # filter already satisfied ones - depends = set([x for x in depends if x not in monotree]) - depends = set([x for x in depends if self.Client.validate_package_removal(x)]) - if depends: - depends = self._filter_depends_multimatched_atoms(idpackage, depends, monotree) - if depends: # something depends on idpackage - tree[treedepth] |= depends - monotree |= depends - treeview |= depends - elif deep: # if deep, grab its dependencies and check - - mydeps = set() - for x in self.Client.clientDbconn.retrieveDependencies(idpackage): - match = self.Client.clientDbconn.atomMatch(x) - if match[0] != -1: - mydeps.add(match[0]) - - # now filter them - mydeps = [x for x in mydeps if x not in monotree and not \ - (self.Client.clientDbconn.isSystemPackage(x) or \ - self.Client.is_installed_idpackage_in_system_mask(x) )] - for x in mydeps: - mydepends = self.Client.clientDbconn.retrieveDepends(x) - mydepends -= set([y for y in mydepends if y not in monotree]) - if not mydepends: - tree[treedepth].add(x) - monotree.add(x) - treeview.add(x) - - dependscache.add(idpackage) - if idpackage in treeview: - treeview.remove(idpackage) - - treelevel = treeview.copy() - if not treelevel: - if not tree[treedepth]: - del tree[treedepth] # probably the last one is empty then - break - - # now filter newtree - for count in sorted(tree.keys(), reverse = True): - x = 0 - while x < count: - tree[x] -= tree[count] - x += 1 - - if self.Client.xcache: - self.Cacher.push(c_hash,(tree,0)) - return tree,0 # treeview is used to show deps while tree is used to run the dependency code. - - def calculate_available_packages(self, use_cache = True): - - c_hash = self.Client.get_available_packages_chash(etpConst['branch']) - - if use_cache and self.Client.xcache: - cached = self.Client.get_available_packages_cache(myhash = c_hash) - if cached != None: - return cached - - available = [] - self.Client.setTotalCycles(len(self.Client.validRepositories)) - avail_dep_text = _("Calculating available packages for") - for repo in self.Client.validRepositories: - try: - dbconn = self.Client.open_repository(repo) - dbconn.validateDatabase() - except (RepositoryError,SystemDatabaseError): - self.Client.cycleDone() - continue - idpackages = [ x for x in dbconn.listAllIdpackages(branch = etpConst['branch'], branch_operator = "<=", order_by = 'atom') \ - if dbconn.idpackageValidator(x)[0] != -1 ] - count = 0 - maxlen = len(idpackages) - myavailable = [] - do_break = False - for idpackage in idpackages: - if do_break: - break - count += 1 - if (count % 10 == 0) or (count == 1) or (count == maxlen): - self.updateProgress( - avail_dep_text + " %s" % (repo,), - importance = 0, - type = "info", - back = True, - header = "::", - count = (count,maxlen), - percent = True, - footer = " ::" - ) - # get key + slot - try: - key, slot = dbconn.retrieveKeySlot(idpackage) - matches = self.Client.clientDbconn.searchKeySlot(key, slot) - except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,): - self.Client.cycleDone() - do_break = True - continue - if not matches: myavailable.append((idpackage,repo)) - available += myavailable[:] - self.Client.cycleDone() - - if self.Client.xcache: - self.Cacher.push("%s%s" % (etpCache['world_available'],c_hash),available) - return available - - def calculate_world_updates( - self, - empty_deps = False, - branch = etpConst['branch'], - ignore_spm_downgrades = etpConst['spm']['ignore-spm-downgrades'], - use_cache = True - ): - - db_digest = self.Client.all_repositories_checksum() - if use_cache and self.Client.xcache: - cached = self.Client.get_world_update_cache(empty_deps = empty_deps, - branch = branch, db_digest = db_digest, ignore_spm_downgrades = ignore_spm_downgrades) - if cached != None: return cached - - update = [] - remove = [] - fine = [] - - # get all the installed packages - idpackages = self.Client.clientDbconn.listAllIdpackages(order_by = 'atom') - maxlen = len(idpackages) - count = 0 - mytxt = _("Calculating world packages") - for idpackage in idpackages: - - count += 1 - if (count%10 == 0) or (count == maxlen) or (count == 1): - self.updateProgress( - mytxt, - importance = 0, - type = "info", - back = True, - header = ":: ", - count = (count,maxlen), - percent = True, - footer = " ::" - ) - - mystrictdata = self.Client.clientDbconn.getStrictData(idpackage) - # check against broken entries, or removed during iteration - if mystrictdata == None: - continue - use_match_cache = True - do_continue = False - while 1: - try: - match = self.atom_match( - mystrictdata[0], - matchSlot = mystrictdata[1], - matchBranches = (branch,), - extendedResults = True, - useCache = use_match_cache - ) - except self.dbapi2.OperationalError: - # ouch, but don't crash here - do_continue = True - break - try: - m_idpackage = match[0][0] - except TypeError: - if not use_match_cache: raise - use_match_cache = False - continue - break - if do_continue: continue - # now compare - # version: mystrictdata[2] - # tag: mystrictdata[3] - # revision: mystrictdata[4] - if (m_idpackage != -1): - repoid = match[1] - version = match[0][1] - tag = match[0][2] - revision = match[0][3] - if empty_deps: - if (m_idpackage,repoid) not in update: - update.append((m_idpackage,repoid)) - continue - elif (mystrictdata[2] != version): - # different versions - if (m_idpackage,repoid) not in update: - update.append((m_idpackage,repoid)) - continue - elif (mystrictdata[3] != tag): - # different tags - if (m_idpackage,repoid) not in update: - update.append((m_idpackage,repoid)) - continue - elif (mystrictdata[4] != revision): - # different revision - if mystrictdata[4] == 9999 and ignore_spm_downgrades: - # no difference, we're ignoring revision 9999 - fine.append(mystrictdata[5]) - continue - else: - if (m_idpackage,repoid) not in update: - update.append((m_idpackage,repoid)) - continue - else: - # no difference - fine.append(mystrictdata[5]) - continue - - # don't take action if it's just masked - maskedresults = self.atom_match(mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), packagesFilter = False) - if maskedresults[0] == -1: - remove.append(idpackage) - # look for packages that would match key with any slot (for eg: gcc, kernel updates) - matchresults = self.atom_match(mystrictdata[0], matchBranches = (branch,)) - if matchresults[0] != -1: - m_action = self.Client.get_package_action(matchresults) - if m_action > 0 and (matchresults not in update): - update.append(matchresults) - - if self.Client.xcache: - c_hash = self.Client.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades) - data = { - 'r': (update, remove, fine,), - 'empty_deps': empty_deps, - } - self.Cacher.push("%s%s" % (etpCache['world_update'],c_hash,), data, async = False) - - return update, remove, fine - - def check_package_update(self, atom, deep = False): - - c_hash = "%s%s" % (etpCache['check_package_update'],hash("%s%s" % (atom,hash(deep),)),) - if self.Client.xcache: - cached = self.Cacher.pop(c_hash) - if cached != None: - return cached - - found = False - match = self.Client.clientDbconn.atomMatch(atom) - matched = None - if match[0] != -1: - myatom = self.Client.clientDbconn.retrieveAtom(match[0]) - mytag = self.entropyTools.dep_gettag(myatom) - myatom = self.entropyTools.remove_tag(myatom) - myrev = self.Client.clientDbconn.retrieveRevision(match[0]) - pkg_match = "="+myatom+"~"+str(myrev) - if mytag != None: - pkg_match += "#%s" % (mytag,) - pkg_unsatisfied = self.get_unsatisfied_dependencies([pkg_match], deep_deps = deep) - if pkg_unsatisfied: - found = True - del pkg_unsatisfied - matched = self.atom_match(pkg_match) - del match - - if self.Client.xcache: - self.Cacher.push(c_hash,(found,matched)) - - return found, matched - - # This is the function that should be used by third party applications - # to retrieve a list of available updates, along with conflicts (removalQueue) and obsoletes - # (removed) - def get_world_queue(self, empty_deps = False, branch = etpConst['branch']): - update, remove, fine = self.calculate_world_updates(empty_deps = empty_deps, branch = branch) - del fine - data = {} - data['removed'] = list(remove) - data['runQueue'] = [] - data['removalQueue'] = [] - status = -1 - if update: - # calculate install+removal queues - install, removal, status = self.get_install_queue(update, empty_deps, deep_deps = False) - # update data['removed'] - data['removed'] = [x for x in data['removed'] if x not in removal] - data['runQueue'] += install - data['removalQueue'] += removal - return data,status - - def validate_package_removal(self, idpackage): - - pkgatom = self.Client.clientDbconn.retrieveAtom(idpackage) - pkgkey = self.entropyTools.dep_getkey(pkgatom) - - if self.Client.is_installed_idpackage_in_system_mask(idpackage): - idpackages = self.SystemSettings['repos_system_mask_installed_keys'].get(pkgkey) - if not idpackages: return False - if len(idpackages) > 1: - return True - return False # sorry! - - # did we store the bastard in the db? - system_pkg = self.Client.clientDbconn.isSystemPackage(idpackage) - if not system_pkg: return True - # check if the package is slotted and exist more than one installed first - sysresults = self.Client.clientDbconn.atomMatch(pkgkey, multiMatch = True) - if sysresults[1] == 0: - if len(sysresults[0]) < 2: return False - return True - return False - - - def get_removal_queue(self, idpackages, deep = False): - queue = [] - if not idpackages: - return queue - treeview, status = self.generate_depends_tree(idpackages, deep = deep) - if status == 0: - for x in range(len(treeview))[::-1]: queue.extend(treeview[x]) - return queue - - def get_install_queue(self, matched_atoms, empty_deps, deep_deps, quiet = False): - - install = [] - removal = [] - treepackages, result = self.get_required_packages(matched_atoms, empty_deps, deep_deps, quiet = quiet) - - if result == -2: - return treepackages,removal,result - - # format - removal = treepackages.pop(0, set()) - for x in sorted(treepackages.keys()): install.extend(treepackages[x]) - - # filter out packages that are in actionQueue comparing key + slot - if install and removal: - myremmatch = {} - for x in removal: - atom = self.Client.clientDbconn.retrieveAtom(x) - # XXX check if users removed idpackage while this whole instance is running - if atom == None: continue - myremmatch[(self.entropyTools.dep_getkey(atom),self.Client.clientDbconn.retrieveSlot(x),)] = x - for pkg_id, pkg_repo in install: - dbconn = self.Client.open_repository(pkg_repo) - testtuple = (self.entropyTools.dep_getkey(dbconn.retrieveAtom(pkg_id)),dbconn.retrieveSlot(pkg_id)) - removal.discard(myremmatch.get(testtuple)) - - return install, sorted(removal), 0 - - """ - XXX deprecated XXX - """ - - def packageSetMatch(self, *args, **kwargs): - import warning - warning.warn("deprecated, use package_set_match instead") - return self.package_set_match(*args, **kwargs) - - def packageSetSearch(self, *args, **kwargs): - import warning - warning.warn("deprecated, use package_set_search instead") - return self.package_set_search(*args, **kwargs) - - def packageSetList(self, *args, **kwargs): - import warning - warning.warn("deprecated, use package_set_list instead") - return self.package_set_list(*args, **kwargs) - - def packageSetExpand(self, *args, **kwargs): - import warning - warning.warn("deprecated, use package_set_expand instead") - return self.package_set_expand(*args, **kwargs) - - def packagesExpand(self, *args, **kwargs): - import warning - warning.warn("deprecated, use packages_expand instead") - return self.packages_expand(*args, **kwargs) - - def atomMatch(self, *args, **kwargs): - import warning - warning.warn("deprecated, use atom_match instead") - return self.atom_match(*args, **kwargs) - - def retrieveWorldQueue(self, *args, **kwargs): - import warning - warning.warn("deprecated, use get_world_queue instead") - return self.get_world_queue(*args, **kwargs) - - def validatePackageRemoval(self, *args, **kwargs): - import warning - warning.warn("deprecated, use validate_package_removal instead") - return self.validate_package_removal(*args, **kwargs) - - def retrieveInstallQueue(self, *args, **kwargs): - import warning - warning.warn("deprecated, use get_install_queue instead") - return self.get_install_queue(*args, **kwargs) - - def retrieveRemovalQueue(self, *args, **kwargs): - import warning - warning.warn("deprecated, use get_removal_queue instead") - return self.get_removal_queue(*args, **kwargs) - - -class _RepositoryMethods: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.MirrorStatus = self.Client.MirrorStatus - self.updateProgress = self.Client.updateProgress - self.SystemSettings = self.Client.SystemSettings - self.entropyTools = self.Client.entropyTools - - self.__repo_error_messages_cache = set() - self.__repodb_cache = {} - self._memory_db_instances = {} - - def validate_repositories(self): - self.MirrorStatus.clear() - self.__repo_error_messages_cache.clear() - self.Client._package_match_validator_cache.clear() - # valid repositories - del self.Client.validRepositories[:] - for repoid in etpRepositoriesOrder: - # open database - try: - dbc = self.open_repository(repoid) - dbc.listConfigProtectDirectories() - dbc.validateDatabase() - self.Client.validRepositories.append(repoid) - except RepositoryError: - t = _("Repository") + " " + repoid + " " + _("is not available") + ". " + _("Cannot validate") - t2 = _("Please update your repositories now in order to remove this message!") - self.updateProgress( - darkred(t), - importance = 1, - type = "warning" - ) - self.updateProgress( - purple(t2), - header = bold("!!! "), - importance = 1, - type = "warning" - ) - continue # repo not available - except (self.Client.dbapi2.OperationalError,self.Client.dbapi2.DatabaseError,SystemDatabaseError,): - t = _("Repository") + " " + repoid + " " + _("is corrupted") + ". " + _("Cannot validate") - self.updateProgress( - darkred(t), - importance = 1, - type = "warning" - ) - continue - # to avoid having zillions of open files when loading a lot of EquoInterfaces - self.close_all_repositories(mask_clear = False) - - def init_generic_memory_repository(self, repoid, description, package_mirrors = []): - dbc = self.Client.open_memory_database(dbname = repoid) - self._memory_db_instances[repoid] = dbc - - # add to etpRepositories - repodata = { - 'repoid': repoid, - 'in_memory': True, - 'description': description, - 'packages': package_mirrors, - 'dbpath': ':memory:', - } - self.add_repository(repodata) - return dbc - - def close_all_repositories(self, mask_clear = True): - for item in self.__repodb_cache: - self.__repodb_cache[item].closeDB() - self.__repodb_cache.clear() - if mask_clear: self.SystemSettings.clear() - - def open_repository(self, repoid): - t_ident = 1 # thread.get_ident() disabled for now - if not self.__repodb_cache.has_key((repoid,etpConst['systemroot'],t_ident,)): - dbconn = self.load_repository_database(repoid, xcache = self.Client.xcache, - indexing = self.Client.indexing) - try: - dbconn.checkDatabaseApi() - except: - pass - self.__repodb_cache[(repoid,etpConst['systemroot'],t_ident,)] = dbconn - return dbconn - else: - return self.__repodb_cache.get((repoid,etpConst['systemroot'],t_ident,)) - - def load_repository_database(self, repoid, xcache = True, indexing = True): - - if isinstance(repoid,basestring): - if repoid.endswith(etpConst['packagesext']): - xcache = False - - if repoid not in etpRepositories: - t = _("bad repository id specified") - if repoid not in self.__repo_error_messages_cache: - self.updateProgress( - darkred(t), - importance = 2, - type = "warning" - ) - self.__repo_error_messages_cache.add(repoid) - raise RepositoryError("RepositoryError: %s" % (t,)) - - dbfile = etpRepositories[repoid]['dbpath']+"/"+etpConst['etpdatabasefile'] - if not os.path.isfile(dbfile): - t = _("Repository %s hasn't been downloaded yet.") % (repoid,) - if repoid not in self.__repo_error_messages_cache: - self.updateProgress( - darkred(t), - importance = 2, - type = "warning" - ) - self.__repo_error_messages_cache.add(repoid) - raise RepositoryError("RepositoryError: %s" % (t,)) - - conn = LocalRepository( - readOnly = True, - dbFile = dbfile, - clientDatabase = True, - dbname = etpConst['dbnamerepoprefix']+repoid, - xcache = xcache, - indexing = indexing, - OutputInterface = self, - ServiceInterface = self - ) - # initialize CONFIG_PROTECT - if (etpRepositories[repoid]['configprotect'] == None) or \ - (etpRepositories[repoid]['configprotectmask'] == None): - self.setup_repository_config(repoid, conn) - - if (repoid not in etpConst['client_treeupdatescalled']) and \ - (self.entropyTools.is_user_in_entropy_group()) and \ - (not repoid.endswith(etpConst['packagesext'])): - updated = False - try: - updated = conn.clientUpdatePackagesData(self.Client.clientDbconn) - except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): - pass - if updated: - self.Client.clear_dump_cache(etpCache['world_update']) - self.Client.clear_dump_cache(etpCache['world']) - self.Client.clear_dump_cache(etpCache['install']) - self.Client.clear_dump_cache(etpCache['remove']) - self.Client.calculate_world_updates(use_cache = False) - return conn - - def setup_repository_config(self, repoid, dbconn): - - try: - etpRepositories[repoid]['configprotect'] = dbconn.listConfigProtectDirectories() - except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): - etpRepositories[repoid]['configprotect'] = [] - try: - etpRepositories[repoid]['configprotectmask'] = dbconn.listConfigProtectDirectories(mask = True) - except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): - etpRepositories[repoid]['configprotectmask'] = [] - - etpRepositories[repoid]['configprotect'] = [etpConst['systemroot']+x for \ - x in etpRepositories[repoid]['configprotect']] - etpRepositories[repoid]['configprotectmask'] = [etpConst['systemroot']+x for \ - x in etpRepositories[repoid]['configprotectmask']] - - etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for \ - x in etpConst['configprotect'] if etpConst['systemroot']+x not \ - in etpRepositories[repoid]['configprotect']] - etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for \ - x in etpConst['configprotectmask'] if etpConst['systemroot']+x not \ - in etpRepositories[repoid]['configprotectmask']] - - def get_repository_revision(self, reponame): - fname = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'] - revision = -1 - if os.path.isfile(fname) and os.access(fname,os.R_OK): - with open(fname,"r") as f: - try: - revision = int(f.readline().strip()) - except (OSError, IOError, ValueError,): - pass - return revision - - def update_repository_revision(self, reponame): - r = self.get_repository_revision(reponame) - etpRepositories[reponame]['dbrevision'] = "0" - if r != -1: - etpRepositories[reponame]['dbrevision'] = str(r) - - def get_repository_db_file_checksum(self, reponame): - fname = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'] - mhash = "-1" - if os.path.isfile(fname) and os.access(fname,os.R_OK): - with open(fname,"r") as f: - try: - mhash = f.readline().strip().split()[0] - except (OSError, IOError, IndexError,): - pass - return mhash - - def add_repository(self, repodata): - # update etpRepositories - try: - etpRepositories[repodata['repoid']] = {} - etpRepositories[repodata['repoid']]['description'] = repodata['description'] - etpRepositories[repodata['repoid']]['configprotect'] = None - etpRepositories[repodata['repoid']]['configprotectmask'] = None - except KeyError: - t = _("repodata dictionary is corrupted") - raise InvalidData("InvalidData: %s" % (t,)) - - if repodata['repoid'].endswith(etpConst['packagesext']) or repodata.get('in_memory'): # dynamic repository - try: - # no need # etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] - etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:] - smart_package = repodata.get('smartpackage') - if smart_package != None: etpRepositories[repodata['repoid']]['smartpackage'] = smart_package - etpRepositories[repodata['repoid']]['dbpath'] = repodata.get('dbpath') - etpRepositories[repodata['repoid']]['pkgpath'] = repodata.get('pkgpath') - except KeyError: - raise InvalidData("InvalidData: repodata dictionary is corrupted") - # put at top priority, shift others - etpRepositoriesOrder.insert(0,repodata['repoid']) - else: - # XXX it's boring to keep this in sync with entropyConstants stuff, solutions? - etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] - etpRepositories[repodata['repoid']]['packages'] = [x+"/"+etpConst['product'] for x in repodata['plain_packages']] - etpRepositories[repodata['repoid']]['plain_database'] = repodata['plain_database'] - etpRepositories[repodata['repoid']]['database'] = repodata['plain_database'] + \ - "/" + etpConst['product'] + "/database/" + etpConst['currentarch'] + "/" + etpConst['branch'] - if not repodata['dbcformat'] in etpConst['etpdatabasesupportedcformats']: - repodata['dbcformat'] = etpConst['etpdatabasesupportedcformats'][0] - etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat'] - etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + \ - "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch'] + "/" + etpConst['branch'] - # set dbrevision - myrev = self.get_repository_revision(repodata['repoid']) - if myrev == -1: - myrev = 0 - etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev) - if repodata.has_key("position"): - etpRepositoriesOrder.insert(repodata['position'],repodata['repoid']) - else: - etpRepositoriesOrder.append(repodata['repoid']) - if not repodata.has_key("service_port"): - repodata['service_port'] = int(etpConst['socket_service']['port']) - if not repodata.has_key("ssl_service_port"): - repodata['ssl_service_port'] = int(etpConst['socket_service']['ssl_port']) - etpRepositories[repodata['repoid']]['service_port'] = repodata['service_port'] - etpRepositories[repodata['repoid']]['ssl_service_port'] = repodata['ssl_service_port'] - self.Client.repository_move_clear_cache(repodata['repoid']) - # save new etpRepositories to file - self.entropyTools.saveRepositorySettings(repodata) - self.Client.reload_constants() - self.validate_repositories() - - def remove_repository(self, repoid, disable = False): - - # ensure that all dbs are closed - self.close_all_repositories() - - done = False - if etpRepositories.has_key(repoid): - del etpRepositories[repoid] - done = True - - if etpRepositoriesExcluded.has_key(repoid): - del etpRepositoriesExcluded[repoid] - done = True - - if done: - - if repoid in etpRepositoriesOrder: - etpRepositoriesOrder.remove(repoid) - - self.Client.repository_move_clear_cache(repoid) - # save new etpRepositories to file - repodata = {} - repodata['repoid'] = repoid - if disable: - self.entropyTools.saveRepositorySettings(repodata, disable = True) - else: - self.entropyTools.saveRepositorySettings(repodata, remove = True) - self.Client.reload_constants() - - self.validate_repositories() - - def shift_repository(self, repoid, toidx): - # update etpRepositoriesOrder - etpRepositoriesOrder.remove(repoid) - etpRepositoriesOrder.insert(toidx,repoid) - self.entropyTools.writeOrderedRepositoriesEntries() - self.Client.reload_constants() - self.Client.repository_move_clear_cache(repoid) - self.validate_repositories() - - def enable_repository(self, repoid): - self.Client.repository_move_clear_cache(repoid) - # save new etpRepositories to file - repodata = {} - repodata['repoid'] = repoid - self.entropyTools.saveRepositorySettings(repodata, enable = True) - self.Client.reload_constants() - self.validate_repositories() - - def disable_repository(self, repoid): - # update etpRepositories - done = False - try: - del etpRepositories[repoid] - done = True - except: - pass - - if done: - try: - etpRepositoriesOrder.remove(repoid) - except: - pass - # it's not vital to reset etpRepositoriesOrder counters - - self.Client.repository_move_clear_cache(repoid) - # save new etpRepositories to file - repodata = {} - repodata['repoid'] = repoid - self.entropyTools.saveRepositorySettings(repodata, disable = True) - self.Client.reload_constants() - self.validate_repositories() - - # every tbz2 file that would be installed must pass from here - def add_tbz2_to_repos(self, tbz2file): - atoms_contained = [] - basefile = os.path.basename(tbz2file) - cut_idx = -1*(len(etpConst['packagesext'])) - db_dir = etpConst['entropyunpackdir']+"/"+basefile[:cut_idx] - if os.path.isdir(db_dir): - shutil.rmtree(db_dir) - os.makedirs(db_dir) - dbfile = self.entropyTools.extractEdb(tbz2file, - dbpath = db_dir+"/packages.db") - if dbfile == None: - return -1,atoms_contained - etpSys['dirstoclean'].add(os.path.dirname(dbfile)) - # add dbfile - repodata = {} - repodata['repoid'] = basefile - repodata['description'] = "Dynamic database from "+basefile - repodata['packages'] = [] - repodata['dbpath'] = os.path.dirname(dbfile) - repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added - repodata['smartpackage'] = False # extra info added - - mydbconn = self.Client.open_generic_database(dbfile) - # read all idpackages - try: - myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files - except (AttributeError, self.Client.dbapi2.DatabaseError, \ - self.Client.dbapi2.IntegrityError, self.Client.dbapi2.OperationalError,): - return -2,atoms_contained - if len(myidpackages) > 1: - repodata[basefile]['smartpackage'] = True - for myidpackage in myidpackages: - compiled_arch = mydbconn.retrieveDownloadURL(myidpackage) - if compiled_arch.find("/"+etpSys['arch']+"/") == -1: - return -3,atoms_contained - atoms_contained.append((int(myidpackage),basefile)) - - self.add_repository(repodata) - self.validate_repositories() - if basefile not in self.Client.validRepositories: - self.remove_repository(basefile) - return -4,atoms_contained - mydbconn.closeDB() - del mydbconn - return 0,atoms_contained - - def reopen_client_repository(self): - self.clientDbconn.closeDB() - self.open_client_repository() - - def __client_repo_setup_const(self, conn): - if conn.doesTableExist('configprotect') and conn.doesTableExist('configprotectreference'): - etpConst['dbconfigprotect'] = conn.listConfigProtectDirectories() - if conn.doesTableExist('configprotectmask') and conn.doesTableExist('configprotectreference'): - etpConst['dbconfigprotectmask'] = conn.listConfigProtectDirectories(mask = True) - - etpConst['dbconfigprotect'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] - etpConst['dbconfigprotectmask'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] - - etpConst['dbconfigprotect'] += [etpConst['systemroot']+x for \ - x in etpConst['configprotect'] if etpConst['systemroot']+x not \ - in etpConst['dbconfigprotect']] - etpConst['dbconfigprotectmask'] += [etpConst['systemroot']+x for \ - x in etpConst['configprotectmask'] if etpConst['systemroot']+x not \ - in etpConst['dbconfigprotectmask']] - - def open_client_repository(self): - - def load_db_from_ram(): - self.Client.safe_mode = etpConst['safemodeerrors']['clientdb'] - mytxt = "%s, %s" % (_("System database not found or corrupted"), - _("running in safe mode using empty database from RAM"),) - self.updateProgress( - darkred(mytxt), - importance = 1, - type = "warning", - header = bold("!!!"), - ) - conn = self.open_memory_database(dbname = etpConst['clientdbid']) - return conn - - db_dir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) - if not os.path.isdir(db_dir): os.makedirs(db_dir) - - db_path = etpConst['etpdatabaseclientfilepath'] - if (not self.Client.noclientdb) and (not os.path.isfile(db_path)): - conn = load_db_from_ram() - self.entropyTools.printTraceback(f = self.Client.clientLog) - else: - conn = LocalRepository(readOnly = False, dbFile = db_path, - clientDatabase = True, dbname = etpConst['clientdbid'], - xcache = self.Client.xcache, indexing = self.Client.indexing, - OutputInterface = self, ServiceInterface = self - ) - # validate database - if not self.Client.noclientdb: - try: - conn.validateDatabase() - except SystemDatabaseError: - try: - conn.closeDB() - except: - pass - self.entropyTools.printTraceback(f = self.Client.clientLog) - conn = load_db_from_ram() - - if not etpConst['dbconfigprotect']: - self.__client_repo_setup_const(conn) - - self.Client.clientDbconn = conn - return self.Client.clientDbconn - - def client_repository_sanity_check(self): - self.updateProgress( - darkred(_("Sanity Check") + ": " + _("system database")), - importance = 2, - type = "warning" - ) - idpkgs = self.clientDbconn.listAllIdpackages() - length = len(idpkgs) - count = 0 - errors = False - scanning_txt = _("Scanning...") - for x in idpkgs: - count += 1 - self.updateProgress( - darkgreen(scanning_txt), - importance = 0, - type = "info", - back = True, - count = (count,length), - percent = True - ) - try: - self.clientDbconn.getPackageData(x) - except Exception ,e: - self.entropyTools.printTraceback() - errors = True - self.updateProgress( - darkred(_("Errors on idpackage %s, error: %s")) % (x,str(e)), - importance = 0, - type = "warning" - ) - - if not errors: - t = _("Sanity Check") + ": %s" % (bold(_("PASSED")),) - self.updateProgress( - darkred(t), - importance = 2, - type = "warning" - ) - return 0 - else: - t = _("Sanity Check") + ": %s" % (bold(_("CORRUPTED")),) - self.updateProgress( - darkred(t), - importance = 2, - type = "warning" - ) - return -1 - - def open_generic_database(self, dbfile, dbname = None, xcache = None, - readOnly = False, indexing_override = None, skipChecks = False): - if xcache == None: - xcache = self.Client.xcache - if indexing_override != None: - indexing = indexing_override - else: - indexing = self.Client.indexing - if dbname == None: - dbname = etpConst['genericdbid'] - return LocalRepository( - readOnly = readOnly, - dbFile = dbfile, - clientDatabase = True, - dbname = dbname, - xcache = xcache, - indexing = indexing, - OutputInterface = self, - skipChecks = skipChecks - ) - - def open_memory_database(self, dbname = None): - if dbname == None: - dbname = etpConst['genericdbid'] - dbc = LocalRepository( - readOnly = False, - dbFile = ':memory:', - clientDatabase = True, - dbname = dbname, - xcache = False, - indexing = False, - OutputInterface = self, - skipChecks = True - ) - dbc.initializeDatabase() - return dbc - - def backup_database(self, dbpath, backup_dir = None, silent = False, compress_level = 9): - - if compress_level not in range(1,10): - compress_level = 9 - - backup_dir = os.path.dirname(dbpath) - if not backup_dir: backup_dir = os.path.dirname(dbpath) - dbname = os.path.basename(dbpath) - bytes_required = 1024000*300 - if not (os.access(backup_dir,os.W_OK) and \ - os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \ - os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)): - if not silent: - mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),) - self.updateProgress( - mytxt, - importance = 1, - type = "error", - header = red(" @@ ") - ) - return False, mytxt - - def get_ts(): - from datetime import datetime - ts = datetime.fromtimestamp(time.time()) - return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second) - - comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),) - comp_dbpath = os.path.join(backup_dir,comp_dbname) - if not silent: - mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = blue(" @@ "), - back = True - ) - import bz2 - try: - self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) - except: - if not silent: - self.entropyTools.printTraceback() - return False, _("Unable to compress") - - if not silent: - mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = blue(" @@ "), - back = True - ) - return True, _("All fine") - - def restore_database(self, backup_path, db_destination, silent = False): - - bytes_required = 1024000*300 - db_dir = os.path.dirname(db_destination) - if not (os.access(db_dir,os.W_OK) and os.path.isdir(db_dir) and \ - os.path.isfile(backup_path) and os.access(backup_path,os.R_OK) and \ - self.entropyTools.check_required_space(db_dir, bytes_required)): - - if not silent: - mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")), - blue(backup_path),darkred(_("permission denied")),) - self.updateProgress( - mytxt, - importance = 1, - type = "error", - header = red(" @@ ") - ) - return False, mytxt - - if not silent: - mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")), - blue(os.path.basename(backup_path)),blue(db_destination),) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = blue(" @@ "), - back = True - ) - - import bz2 - try: - self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File) - except: - if not silent: - self.entropyTools.printTraceback() - return False, _("Unable to unpack") - - if not silent: - mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")), - blue(db_destination),) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = blue(" @@ "), - back = True - ) - self.Client.purge_cache() - return True, _("All fine") - - def list_backedup_client_databases(self, client_dbdir = None): - if not client_dbdir: - client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) - return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \ - if x.startswith(etpConst['dbbackupprefix']) and \ - os.access(os.path.join(client_dbdir,x),os.R_OK) - ] - - """ - XXX deprecated XXX - """ - - def backupDatabase(self, *args, **kwargs): - import warning - warning.warn("deprecated, use backup_database instead") - return self.backup_database(*args, **kwargs) - - def restoreDatabase(self, *args, **kwargs): - import warning - warning.warn("deprecated, use restore_database instead") - return self.restore_database(*args, **kwargs) - - def openGenericDatabase(self, *args, **kwargs): - import warning - warning.warn("deprecated, use open_generic_database instead") - return self.open_generic_database(*args, **kwargs) - - def openMemoryDatabase(self, *args, **kwargs): - import warning - warning.warn("deprecated, use open_memory_database instead") - return self.open_memory_database(*args, **kwargs) - - def clientDatabaseSanityCheck(self): - import warning - warning.warn("deprecated, use client_repository_sanity_check instead") - return self.client_repository_sanity_check() - - def openClientDatabase(self): - import warning - warning.warn("deprecated, use open_client_repository instead") - return self.open_client_repository() - - def reopenClientDbconn(self): - import warning - warning.warn("deprecated, use reopen_client_repository instead") - return self.reopen_client_repository() - - def openRepositoryDatabase(self, repoid): - import warning - warning.warn("deprecated, use open_repository instead") - return self.open_repository(repoid) - - def closeAllRepositoryDatabases(self, mask_clear = True): - import warning - warning.warn("deprecated, use close_all_repositories instead") - return self.close_all_repositories(mask_clear = mask_clear) - - def addRepository(self, repodata): - import warning - warning.warn("deprecated, use add_repository instead") - return self.add_repository(repodata) - - def removeRepository(self, repoid, disable = False): - import warning - warning.warn("deprecated, use remove_repository instead") - return self.remove_repository(repoid, disable = disable) - - def shiftRepository(self, repoid, toidx): - import warning - warning.warn("deprecated, use shift_repository instead") - return self.shift_repository(repoid, toidx) - - def enableRepository(self, repoid): - import warning - warning.warn("deprecated, use enable_repository instead") - return self.enable_repository(repoid) - - def disableRepository(self, repoid): - import warning - warning.warn("deprecated, use disable_repository instead") - return self.disable_repository(repoid) - - -class _MiscMethods: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.entropyTools = self.Client.entropyTools - self.updateProgress = self.Client.updateProgress - self.SystemSettings = self.Client.SystemSettings - - def reload_constants(self): - initConfig_entropyConstants(etpSys['rootdir']) - initConfig_clientConstants() - - def setup_default_file_perms(self, filepath): - # setup file permissions - os.chmod(filepath,0664) - if etpConst['entropygid'] != None: - os.chown(filepath,-1,etpConst['entropygid']) - - def _resources_run_create_lock(self): - self.create_pid_file_lock(etpConst['locks']['using_resources']) - - def _resources_run_remove_lock(self): - if os.path.isfile(etpConst['locks']['using_resources']): - os.remove(etpConst['locks']['using_resources']) - - def _resources_run_check_lock(self): - rc = self.check_pid_file_lock(etpConst['locks']['using_resources']) - return rc - - def check_pid_file_lock(self, pidfile): - if not os.path.isfile(pidfile): - return False # not locked - f = open(pidfile) - s_pid = f.readline().strip() - f.close() - try: - s_pid = int(s_pid) - except ValueError: - return False # not locked - # is it our pid? - mypid = os.getpid() - if (s_pid != mypid) and os.path.isdir("%s/proc/%s" % (etpConst['systemroot'],s_pid,)): - # is it running - return True # locked - return False - - def create_pid_file_lock(self, pidfile, mypid = None): - lockdir = os.path.dirname(pidfile) - if not os.path.isdir(lockdir): - os.makedirs(lockdir,0775) - const_setup_perms(lockdir,etpConst['entropygid']) - if mypid == None: - mypid = os.getpid() - f = open(pidfile,"w") - f.write(str(mypid)) - f.flush() - f.close() - - def application_lock_check(self, silent = False): - # check if another instance is running - etpConst['applicationlock'] = False - const_setupEntropyPid(just_read = True) - locked = self.entropyTools.applicationLockCheck(option = None, gentle = True, silent = True) - if locked: - if not silent: - self.updateProgress( - red(_("Another Entropy instance is currently active, cannot satisfy your request.")), - importance = 1, - type = "error", - header = darkred(" @@ ") - ) - return True - return False - - def lock_check(self, check_function): - - lock_count = 0 - max_lock_count = 600 - sleep_seconds = 0.5 - - # check lock file - while 1: - locked = check_function() - if not locked: - if lock_count > 0: - self.updateProgress( - blue(_("Resources unlocked, let's go!")), - importance = 1, - type = "info", - header = darkred(" @@ ") - ) - break - if lock_count >= max_lock_count: - mycalc = max_lock_count*sleep_seconds/60 - self.updateProgress( - blue(_("Resources still locked after %s minutes, giving up!")) % (mycalc,), - importance = 1, - type = "warning", - header = darkred(" @@ ") - ) - return True # gave up - lock_count += 1 - self.updateProgress( - blue(_("Resources locked, sleeping %s seconds, check #%s/%s")) % ( - sleep_seconds, - lock_count, - max_lock_count, - ), - importance = 1, - type = "warning", - header = darkred(" @@ "), - back = True - ) - time.sleep(sleep_seconds) - return False # yay! - - def backup_setting(self, setting_name): - if etpConst.has_key(setting_name): - myinst = etpConst[setting_name] - if type(etpConst[setting_name]) in (list,tuple): - myinst = etpConst[setting_name][:] - elif type(etpConst[setting_name]) in (dict,set): - myinst = etpConst[setting_name].copy() - else: - myinst = etpConst[setting_name] - etpConst['backed_up'].update({setting_name: myinst}) - else: - t = _("Nothing to backup in etpConst with %s key") % (setting_name,) - raise InvalidData("InvalidData: %s" % (t,)) - - def set_priority(self, low = 0): - return const_setNiceLevel(low) - - def reload_repositories_config(self, repositories = None): - if repositories == None: - repositories = self.Client.validRepositories - for repoid in repositories: - dbconn = self.Client.open_repository(repoid) - self.Client.setup_repository_config(repoid, dbconn) - - def switch_chroot(self, chroot = ""): - # clean caches - self.Client.purge_cache() - self.Client.close_all_repositories() - if chroot.endswith("/"): - chroot = chroot[:-1] - etpSys['rootdir'] = chroot - self.reload_constants() - self.Client.validate_repositories() - self.Client.reopen_client_repository() - if chroot: - try: self.Client.clientDbconn.resetTreeupdatesDigests() - except: pass - # I don't think it's safe to keep them open - # isn't it? - self.Client.closeAllSecurity() - self.Client.closeAllQA() - - def get_file_viewer(self): - viewer = None - if os.access("/usr/bin/less",os.X_OK): - viewer = "/usr/bin/less" - elif os.access("/bin/more",os.X_OK): - viewer = "/bin/more" - if not viewer: - viewer = self.get_file_editor() - return viewer - - def get_file_editor(self): - editor = None - if os.getenv("EDITOR"): - editor = "$EDITOR" - elif os.access("/bin/nano",os.X_OK): - editor = "/bin/nano" - elif os.access("/bin/vi",os.X_OK): - editor = "/bin/vi" - elif os.access("/usr/bin/vi",os.X_OK): - editor = "/usr/bin/vi" - elif os.access("/usr/bin/emacs",os.X_OK): - editor = "/usr/bin/emacs" - elif os.access("/bin/emacs",os.X_OK): - editor = "/bin/emacs" - return editor - - def add_user_package_set(self, set_name, set_atoms): - - def _ensure_package_sets_dir(): - sets_dir = etpConst['confsetsdir'] - if not os.path.isdir(sets_dir): - if os.path.lexists(sets_dir): - os.remove(sets_dir) - os.makedirs(sets_dir,0775) - const_setup_perms(sets_dir, etpConst['entropygid']) - - try: - set_name = str(set_name) - except (UnicodeEncodeError,UnicodeDecodeError,): - raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) - - if set_name.startswith(etpConst['packagesetprefix']): - raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) - set_match, rc = self.Client.package_set_match(set_name) - if rc: return -1,_("Name already taken") - - _ensure_package_sets_dir() - set_file = os.path.join(etpConst['confsetsdir'],set_name) - if os.path.isfile(set_file) and os.access(set_file,os.W_OK): - try: - os.remove(set_file) - except OSError: - return -2,_("Cannot remove the old element") - if not os.access(os.path.dirname(set_file),os.W_OK): - return -3,_("Cannot create the element") - - f = open(set_file,"w") - for x in set_atoms: f.write("%s\n" % (x,)) - f.flush() - f.close() - self.SystemSettings['system_package_sets'][set_name] = set(set_atoms) - return 0,_("All fine") - - def remove_user_package_set(self, set_name): - - try: - set_name = str(set_name) - except (UnicodeEncodeError,UnicodeDecodeError,): - raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) - - if set_name.startswith(etpConst['packagesetprefix']): - raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) - - set_match, rc = self.Client.package_set_match(set_name) - if not rc: return -1,_("Already removed") - set_id, set_x, set_y = set_match - - if set_id != etpConst['userpackagesetsid']: - return -2,_("Not defined by user") - set_file = os.path.join(etpConst['confsetsdir'],set_name) - if os.path.isfile(set_file) and os.access(set_file,os.W_OK): - os.remove(set_file) - if set_name in self.SystemSettings['system_package_sets']: - del self.SystemSettings['system_package_sets'][set_name] - return 0,_("All fine") - return -3,_("Set not found or unable to remove") - - def is_installed_idpackage_in_system_mask(self, idpackage): - if idpackage in self.SystemSettings['repos_system_mask_installed']: - return True - return False - - def get_branch_from_download_relative_uri(self, db_download_uri): - return db_download_uri.split("/")[2] - - def unused_packages_test(self, dbconn = None): - if dbconn == None: dbconn = self.Client.clientDbconn - return [x for x in dbconn.retrieveUnusedIdpackages() if self.Client.validate_package_removal(x)] - - def get_licenses_to_accept(self, install_queue): - if not install_queue: - return {} - licenses = {} - for match in install_queue: - repoid = match[1] - dbconn = self.Client.open_repository(repoid) - wl = self.SystemSettings['repos_license_whitelist'].get(repoid) - if not wl: - continue - keys = dbconn.retrieveLicensedataKeys(match[0]) - for key in keys: - if key not in wl: - found = self.Client.clientDbconn.isLicenseAccepted(key) - if found: - continue - if not licenses.has_key(key): - licenses[key] = set() - licenses[key].add(match) - return licenses - - def get_text_license(self, license_name, repoid): - dbconn = self.Client.open_repository(repoid) - text = dbconn.retrieveLicenseText(license_name) - tempfile = self.entropyTools.getRandomTempFile() - f = open(tempfile,"w") - f.write(text) - f.flush() - f.close() - return tempfile - - def libraries_test(self, dbconn = None, broken_symbols = False, task_bombing_func = None): - - if dbconn == None: - dbconn = self.Client.clientDbconn - - self.updateProgress( - blue(_("Libraries test")), - importance = 2, - type = "info", - header = red(" @@ ") - ) - - if not etpConst['systemroot']: - myroot = "/" - else: - myroot = etpConst['systemroot']+"/" - # run ldconfig first - subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) - # open /etc/ld.so.conf - if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"): - self.updateProgress( - blue(_("Cannot find "))+red(etpConst['systemroot']+"/etc/ld.so.conf"), - importance = 1, - type = "error", - header = red(" @@ ") - ) - return {},set(),-1 - - ldpaths = set(self.entropyTools.collectLinkerPaths()) - ldpaths |= self.entropyTools.collectPaths() - # speed up when /usr/lib is a /usr/lib64 symlink - if "/usr/lib64" in ldpaths and "/usr/lib" in ldpaths: - if os.path.realpath("/usr/lib64") == "/usr/lib": - ldpaths.discard("/usr/lib") - # some crappy packages put shit here too - ldpaths.add("/usr/share") - # always force /usr/libexec too - ldpaths.add("/usr/libexec") - - executables = set() - total = len(ldpaths) - count = 0 - sys_root_len = len(etpConst['systemroot']) - for ldpath in ldpaths: - if callable(task_bombing_func): task_bombing_func() - count += 1 - self.updateProgress( - blue("Tree: ")+red(etpConst['systemroot']+ldpath), - importance = 0, - type = "info", - count = (count,total), - back = True, - percent = True, - header = " " - ) - ldpath = ldpath.encode(sys.getfilesystemencoding()) - mywalk_iter = os.walk(etpConst['systemroot']+ldpath) - - def mywimf(dt): - - currentdir, subdirs, files = dt - - def mymf(item): - filepath = os.path.join(currentdir,item) - if filepath in etpConst['libtest_files_blacklist']: - return 0 - if not os.access(filepath,os.R_OK): - return 0 - if not os.path.isfile(filepath): - return 0 - if not self.entropyTools.is_elf_file(filepath): - return 0 - return filepath[sys_root_len:] - - return set([x for x in map(mymf,files) if type(x) != int]) - - for x in map(mywimf,mywalk_iter): executables |= x - - self.updateProgress( - blue(_("Collecting broken executables")), - importance = 2, - type = "info", - header = red(" @@ ") - ) - t = red(_("Attention")) + ": " + \ - blue(_("don't worry about libraries that are shown here but not later.")) - self.updateProgress( - t, - importance = 1, - type = "info", - header = red(" @@ ") - ) - - myQA = self.Client.QA() - - plain_brokenexecs = set() - total = len(executables) - count = 0 - scan_txt = blue("%s ..." % (_("Scanning libraries"),)) - for executable in executables: - if callable(task_bombing_func): task_bombing_func() - count += 1 - if (count%10 == 0) or (count == total) or (count == 1): - self.updateProgress( - scan_txt, - importance = 0, - type = "info", - count = (count,total), - back = True, - percent = True, - header = " " - ) - myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable) - def mymf2(mylib): - return not myQA.resolve_dynamic_library(mylib, executable) - mylibs = set(filter(mymf2,myelfs)) - broken_sym_found = set() - if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable) - if not (mylibs or broken_sym_found): - continue - - if mylibs: - alllibs = blue(' :: ').join(list(mylibs)) - self.updateProgress( - red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]", - importance = 1, - type = "info", - percent = True, - count = (count,total), - header = " " - ) - elif broken_sym_found: - allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)]) - if len(allsyms) > 50: allsyms = brown(_('various broken symbols')) - self.updateProgress( - red(etpConst['systemroot']+executable)+" { "+allsyms+" }", - importance = 1, - type = "info", - percent = True, - count = (count,total), - header = " " - ) - plain_brokenexecs.add(executable) - - del executables - packagesMatched = {} - - if not etpSys['serverside']: - - self.updateProgress( - blue(_("Matching broken libraries/executables")), - importance = 1, - type = "info", - header = red(" @@ ") - ) - matched = set() - for brokenlib in plain_brokenexecs: - idpackages = self.Client.clientDbconn.searchBelongs(brokenlib) - for idpackage in idpackages: - key, slot = self.Client.clientDbconn.retrieveKeySlot(idpackage) - mymatch = self.Client.atom_match(key, matchSlot = slot) - if mymatch[0] == -1: - matched.add(brokenlib) - continue - cmpstat = self.Client.get_package_action(mymatch) - if cmpstat == 0: - continue - if not packagesMatched.has_key(brokenlib): - packagesMatched[brokenlib] = set() - packagesMatched[brokenlib].add(mymatch) - matched.add(brokenlib) - plain_brokenexecs -= matched - - return packagesMatched,plain_brokenexecs,0 - - def move_to_branch(self, branch, pretend = False): - if pretend: return 0 - if branch != etpConst['branch']: - # update configuration - self.entropyTools.writeNewBranch(branch) - # reset treeupdatesactions - self.Client.clientDbconn.resetTreeupdatesDigests() - # clean cache - self.Client.purge_cache(showProgress = False) - # reopen Client Database, this will make treeupdates to be re-read - self.Client.close_all_repositories() - etpConst['branch'] = branch - self.Client.reload_constants() - etpConst['branch'] = branch - self.Client.validate_repositories() - self.Client.reopen_client_repository() - return 0 - - def get_meant_packages(self, search_term, from_installed = False, valid_repos = []): - - pkg_data = [] - atom_srch = False - if "/" in search_term: atom_srch = True - - if not valid_repos: valid_repos = self.Client.validRepositories - if from_installed: valid_repos = [1] - for repo in valid_repos: - if isinstance(repo,basestring): - dbconn = self.Client.open_repository(repo) - elif isinstance(repo,LocalRepository): - dbconn = repo - elif hasattr(self,'clientDbconn'): - dbconn = self.Client.clientDbconn - else: - continue - pkg_data.extend([(x,repo,) for x in dbconn.searchSimilarPackages(search_term, atom = atom_srch)]) - - return pkg_data - - def list_repo_categories(self): - categories = set() - for repo in self.Client.validRepositories: - dbconn = self.Client.open_repository(repo) - catsdata = dbconn.listAllCategories() - categories.update(set([x[1] for x in catsdata])) - return categories - - def list_repo_packages_in_category(self, category): - pkg_matches = [] - for repo in self.Client.validRepositories: - dbconn = self.Client.open_repository(repo) - catsdata = dbconn.searchPackagesByCategory(category, branch = etpConst['branch']) - pkg_matches.extend([(x[1],repo,) for x in catsdata if (x[1],repo,) not in pkg_matches]) - return pkg_matches - - def get_category_description_data(self, category): - - data = {} - for repo in self.Client.validRepositories: - try: - dbconn = self.Client.open_repository(repo) - except RepositoryError: - continue - try: - data = dbconn.retrieveCategoryDescription(category) - except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.IntegrityError,): - continue - if data: break - - return data - - def list_installed_packages_in_category(self, category): - pkg_matches = set([x[1] for x in self.Client.clientDbconn.searchPackagesByCategory(category)]) - return pkg_matches - - def inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None): - dbpath = self.get_tmp_dbpath() - dbconn = self.Client.open_generic_database(dbpath) - dbconn.initializeDatabase() - dbconn.addPackage(data, revision = data['revision']) - if treeupdates_actions != None: - dbconn.bumpTreeUpdatesActions(treeupdates_actions) - dbconn.commitChanges() - dbconn.closeDB() - self.entropyTools.aggregateEdb(tbz2file = package_filename, dbfile = dbpath) - return dbpath - - def get_tmp_dbpath(self): - dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) - while os.path.isfile(dbpath): - dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) - return dbpath - - def quickpkg(self, atomstring, savedir = None): - if savedir == None: - savedir = etpConst['packagestmpdir'] - if not os.path.isdir(etpConst['packagestmpdir']): - os.makedirs(etpConst['packagestmpdir']) - # match package - match = self.Client.clientDbconn.atomMatch(atomstring) - if match[0] == -1: - return -1,None,None - atom = self.Client.clientDbconn.atomMatch(match[0]) - pkgdata = self.Client.clientDbconn.getPackageData(match[0]) - resultfile = self.quickpkg_handler(pkgdata = pkgdata, dirpath = savedir) - if resultfile == None: - return -1,atom,None - else: - return 0,atom,resultfile - - def quickpkg_handler(self, pkgdata, dirpath, edb = True, - portdbPath = None, fake = False, compression = "bz2", shiftpath = ""): - - import stat - import tarfile - - if compression not in ("bz2","","gz"): - compression = "bz2" - - # getting package info - pkgtag = '' - pkgrev = "~"+str(pkgdata['revision']) - if pkgdata['versiontag']: pkgtag = "#"+pkgdata['versiontag'] - pkgname = pkgdata['name']+"-"+pkgdata['version']+pkgrev+pkgtag # + version + tag - pkgcat = pkgdata['category'] - #pkgfile = pkgname+etpConst['packagesext'] - dirpath += "/"+pkgname+etpConst['packagesext'] - if os.path.isfile(dirpath): - os.remove(dirpath) - tar = tarfile.open(dirpath,"w:"+compression) - - if not fake: - - contents = sorted([x for x in pkgdata['content']]) - id_strings = {} - - # collect files - for path in contents: - # convert back to filesystem str - encoded_path = path - path = path.encode('raw_unicode_escape') - path = shiftpath+path - try: - exist = os.lstat(path) - except OSError: - continue # skip file - arcname = path[len(shiftpath):] # remove shiftpath - if arcname.startswith("/"): - arcname = arcname[1:] # remove trailing / - ftype = pkgdata['content'][encoded_path] - if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype - if 'dir' == ftype and \ - not stat.S_ISDIR(exist.st_mode) and \ - os.path.isdir(path): # workaround for directory symlink issues - path = os.path.realpath(path) - - tarinfo = tar.gettarinfo(path, arcname) - tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) - tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) - - if stat.S_ISREG(exist.st_mode): - tarinfo.type = tarfile.REGTYPE - f = open(path) - try: - tar.addfile(tarinfo, f) - finally: - f.close() - else: - tar.addfile(tarinfo) - - tar.close() - - # appending xpak metadata - if etpConst['gentoo-compat']: - import entropy.xpak as xpak - Spm = self.Client.Spm() - - gentoo_name = self.entropyTools.remove_tag(pkgname) - gentoo_name = self.entropyTools.remove_entropy_revision(gentoo_name) - if portdbPath == None: - dbdir = Spm.get_vdb_path()+"/"+pkgcat+"/"+gentoo_name+"/" - else: - dbdir = portdbPath+"/"+pkgcat+"/"+gentoo_name+"/" - if os.path.isdir(dbdir): - tbz2 = xpak.tbz2(dirpath) - tbz2.recompose(dbdir) - - if edb: - self.inject_entropy_database_into_package(dirpath, pkgdata) - - if os.path.isfile(dirpath): - return dirpath - return None - - """ - XXX deprecated XXX - """ - - def switchChroot(self, *args, **kwargs): - import warning - warning.warn("deprecated, use switch_chroot instead") - return self.switch_chroot(*args, **kwargs) - - -class _MatchMethods: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.entropyTools = self.Client.entropyTools - self.updateProgress = self.Client.updateProgress - self.SystemSettings = self.Client.SystemSettings - self.Cacher = self.Client.Cacher - - def get_package_action(self, match): - """ - @input: matched atom (idpackage,repoid) - @output: - upgrade: int(2) - install: int(1) - reinstall: int(0) - downgrade: int(-1) - """ - dbconn = self.Client.open_repository(match[1]) - pkgkey, pkgslot = dbconn.retrieveKeySlot(match[0]) - results = self.Client.clientDbconn.searchKeySlot(pkgkey, pkgslot) - if not results: return 1 - - installed_idpackage = results[0][0] - pkgver, pkgtag, pkgrev = dbconn.getVersioningData(match[0]) - installedVer, installedTag, installedRev = self.Client.clientDbconn.getVersioningData(installed_idpackage) - pkgcmp = self.entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) - if pkgcmp == 0: - return 0 - elif pkgcmp > 0: - return 2 - return -1 - - def get_masked_package_reason(self, match): - idpackage, repoid = match - dbconn = self.Client.open_repository(repoid) - idpackage, idreason = dbconn.idpackageValidator(idpackage) - masked = False - if idpackage == -1: masked = True - return masked, idreason, self.SystemSettings['pkg_masking_reasons'].get(idreason) - - def get_match_conflicts(self, match): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - conflicts = dbconn.retrieveConflicts(m_id) - found_conflicts = set() - for conflict in conflicts: - my_m_id, my_m_rc = self.Client.clientDbconn.atomMatch(conflict) - if my_m_id != -1: - # check if the package shares the same slot - match_data = dbconn.retrieveKeySlot(m_id) - installed_match_data = self.Client.clientDbconn.retrieveKeySlot(my_m_id) - if match_data != installed_match_data: - found_conflicts.add(my_m_id) - return found_conflicts - - def is_match_masked(self, match, live_check = True): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) - if idpackage != -1: - return False - return True - - def is_match_masked_by_user(self, match, live_check = True): - # (query_status,masked?,) - m_id, m_repo = match - if m_repo not in self.Client.validRepositories: return False - dbconn = self.Client.open_repository(m_repo) - idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) - if idpackage != -1: return False #,False - myr = self.SystemSettings['pkg_masking_reference'] - user_masks = [myr['user_package_mask'],myr['user_license_mask'],myr['user_live_mask']] - if idreason in user_masks: - return True #,True - return False #,True - - def is_match_unmasked_by_user(self, match, live_check = True): - # (query_status,unmasked?,) - m_id, m_repo = match - if m_repo not in self.Client.validRepositories: return False - dbconn = self.Client.open_repository(m_repo) - idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) - if idpackage == -1: return False #,False - myr = self.SystemSettings['pkg_masking_reference'] - user_masks = [ - myr['user_package_unmask'],myr['user_live_unmask'],myr['user_package_keywords'], - myr['user_repo_package_keywords_all'], myr['user_repo_package_keywords'] - ] - if idreason in user_masks: - return True #,True - return False #,True - - def mask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): - if self.is_match_masked(match, live_check = False): return True - methods = { - 'atom': self.mask_match_by_atom, - 'keyslot': self.mask_match_by_keyslot, - } - rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) - if dry_run: # inject if done "live" - self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) - self.SystemSettings['live_packagemasking']['mask_matches'].add(match) - return rc - - def unmask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): - if not self.is_match_masked(match, live_check = False): return True - methods = { - 'atom': self.unmask_match_by_atom, - 'keyslot': self.unmask_match_by_keyslot, - } - rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) - if dry_run: # inject if done "live" - self.SystemSettings['live_packagemasking']['unmask_matches'].add(match) - self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) - return rc - - def _mask_unmask_match(self, match, method, methods_reference, dry_run = False, clean_all_cache = False): - - f = methods_reference.get(method) - if not callable(f): - raise IncorrectParameter('IncorrectParameter: %s: %s' % (_("not a valid method"),method,) ) - - self.Cacher.sync(wait = True) - done = f(match, dry_run) - if done: self.SystemSettings.clear() - - # clear atomMatch cache anyway - if clean_all_cache and not dry_run: - self.Client.clear_dump_cache(etpCache['world_available']) - self.Client.clear_dump_cache(etpCache['world_update']) - - self.Client.clear_dump_cache(etpCache['check_package_update']) - self.Client.clear_dump_cache(etpCache['filter_satisfied_deps']) - self.Client.clear_dump_cache(self.Client.atomMatchCacheKey) - self.Client.clear_dump_cache(etpCache['dep_tree']) - self.Client.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],)) - self.Client.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],)) - - self.Client._package_match_validator_cache.clear() - return done - - def unmask_match_by_atom(self, match, dry_run = False): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - atom = dbconn.retrieveAtom(m_id) - return self.unmask_match_generic(match, atom, dry_run = dry_run) - - def unmask_match_by_keyslot(self, match, dry_run = False): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) - return self.unmask_match_generic(match, keyslot, dry_run = dry_run) - - def mask_match_by_atom(self, match, dry_run = False): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - atom = dbconn.retrieveAtom(m_id) - return self.mask_match_generic(match, atom, dry_run = dry_run) - - def mask_match_by_keyslot(self, match, dry_run = False): - m_id, m_repo = match - dbconn = self.Client.open_repository(m_repo) - keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) - return self.mask_match_generic(match, keyslot, dry_run = dry_run) - - def unmask_match_generic(self, match, keyword, dry_run = False): - self.clear_match_mask(match, dry_run) - m_file = self.SystemSettings.etpSettingFiles['unmask'] - return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) - - def mask_match_generic(self, match, keyword, dry_run = False): - self.clear_match_mask(match, dry_run) - m_file = self.SystemSettings.etpSettingFiles['mask'] - return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) - - def _mask_unmask_match_generic(self, keyword, m_file, dry_run = False): - exist = False - if not os.path.isfile(m_file): - if not os.access(os.path.dirname(m_file),os.W_OK): - return False # cannot write - elif not os.access(m_file, os.W_OK): - return False - elif not dry_run: - exist = True - - if dry_run: - return True - - content = [] - if exist: - f = open(m_file,"r") - content = [x.strip() for x in f.readlines()] - f.close() - content.append(keyword) - m_file_tmp = m_file+".tmp" - f = open(m_file_tmp,"w") - for line in content: - f.write(line+"\n") - f.flush() - f.close() - shutil.move(m_file_tmp,m_file) - return True - - def clear_match_mask(self, match, dry_run = False): - masking_list = [self.SystemSettings.etpSettingFiles['mask'],self.SystemSettings.etpSettingFiles['unmask']] - return self._clear_match_generic(match, masking_list = masking_list, dry_run = dry_run) - - def _clear_match_generic(self, match, masking_list = [], dry_run = False): - - self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) - self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) - - if dry_run: return - - for mask_file in masking_list: - if not (os.path.isfile(mask_file) and os.access(mask_file,os.W_OK)): continue - f = open(mask_file,"r") - newf = self.entropyTools.open_buffer() - line = f.readline() - while line: - line = line.strip() - if line.startswith("#"): - newf.write(line+"\n") - line = f.readline() - continue - elif not line: - newf.write("\n") - line = f.readline() - continue - mymatch = self.Client.atom_match(line, packagesFilter = False) - if mymatch == match: - line = f.readline() - continue - newf.write(line+"\n") - line = f.readline() - f.close() - tmpfile = mask_file+".w_tmp" - f = open(tmpfile,"w") - f.write(newf.getvalue()) - f.flush() - f.close() - newf.close() - shutil.move(tmpfile,mask_file) - - -class _ClientFetchers: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.entropyTools = self.Client.entropyTools - self.updateProgress = self.Client.updateProgress - self.SystemSettings = self.Client.SystemSettings - self.Cacher = self.Client.Cacher - self.MirrorStatus = self.Client.MirrorStatus - - def check_needed_package_download(self, filepath, checksum = None): - # is the file available - if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath): - if checksum is None: - return 0 - else: - # check digest - md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum) - if (md5res): - return 0 - else: - return -2 - else: - return -1 - - def fetch_files(self, url_data_list, checksum = True, resume = True, fetch_file_abort_function = None): - """ - Fetch multiple files simultaneously on URLs. - - @param url_data_list list - [(url,dest_path [or None],checksum ['ab86fff46f6ec0f4b1e0a2a4a82bf323' or None],branch,),..] - @param digest bool, digest check (checksum) - @param resume bool enable resume support - @param fetch_file_abort_function callable method that could raise exceptions - @return general_status_code, {'url': (status_code,checksum,resumed,)}, data_transfer - """ - pkgs_bindir = etpConst['packagesbindir'] - url_path_list = [] - checksum_map = {} - count = 0 - for url, dest_path, cksum, branch in url_data_list: - count += 1 - filename = os.path.basename(url) - if dest_path == None: - dest_path = os.path.join(pkgs_bindir,branch,filename,) - - dest_dir = os.path.dirname(dest_path) - if not os.path.isdir(dest_dir): - os.makedirs(dest_dir,0755) - - url_path_list.append((url,dest_path,)) - if cksum != None: checksum_map[count] = cksum - - # load class - fetchConn = self.Client.MultipleUrlFetcher(url_path_list, resume = resume, - abort_check_func = fetch_file_abort_function, OutputInterface = self, - urlFetcherClass = self.Client.urlFetcher, checksum = checksum) - try: - data = fetchConn.download() - except KeyboardInterrupt: - return -100, {}, 0 - - diff_map = {} - if checksum_map and checksum: # verify checksums - diff_map = dict((url_path_list[x-1][0],checksum_map.get(x)) for x in checksum_map \ - if checksum_map.get(x) != data.get(x)) - - data_transfer = fetchConn.get_data_transfer() - if diff_map: - defval = -1 - for key, val in diff_map.items(): - if val == "-1": # general error - diff_map[key] = -1 - elif val == "-2": - diff_map[key] = -2 - elif val == "-4": # timeout - diff_map[key] = -4 - elif val == "-3": # not found - diff_map[key] = -3 - elif val == -100: - defval = -100 - return defval, diff_map, data_transfer - - return 0, diff_map, data_transfer - - def fetch_files_on_mirrors(self, download_list, checksum = False, fetch_abort_function = None): - """ - @param download_map list [(repository,branch,filename,checksum (digest),),..] - @param checksum bool verify checksum? - @param fetch_abort_function callable method that could raise exceptions - """ - repo_uris = dict(((x[0],etpRepositories[x[0]]['packages'][::-1],) for x in download_list)) - remaining = repo_uris.copy() - my_download_list = download_list[:] - - def get_best_mirror(repository): - try: - return remaining[repository][0] - except IndexError: - return None - - def update_download_list(down_list, failed_down): - newlist = [] - for repo,branch,fname,cksum in down_list: - myuri = get_best_mirror(repo) - myuri = os.path.join(myuri,fname) - if myuri not in failed_down: - continue - newlist.append((repo,branch,fname,cksum,)) - return newlist - - # return True: for failing, return False: for fine - def mirror_fail_check(repository, best_mirror): - # check if uri is sane - if not self.MirrorStatus.get_failing_mirror_status(best_mirror) >= 30: - return False - # set to 30 for convenience - self.MirrorStatus.set_failing_mirror_status(best_mirror, 30) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - mytxt += blue(" %s: ") % (_("Mirror"),) - mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) - mytxt += " - %s." % (_("maximum failure threshold reached"),) - self.updateProgress( - mytxt, - importance = 1, - type = "warning", - header = red(" ## ") - ) - - if self.MirrorStatus.get_failing_mirror_status(best_mirror) == 30: - self.MirrorStatus.add_failing_mirror(best_mirror,45) - elif self.MirrorStatus.get_failing_mirror_status(best_mirror) > 31: - self.MirrorStatus.add_failing_mirror(best_mirror,-4) - else: - self.MirrorStatus.set_failing_mirror_status(best_mirror, 0) - - remaining[repository].discard(best_mirror) - return True - - def show_download_summary(down_list): - # fetch_files_list.append((myuri,None,cksum,branch,)) - for repo, branch, fname, cksum in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - basef = os.path.basename(fname) - mytxt += "[%s] %s " % (brown(basef),blue("@"),) - mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) - # now fetch the new one - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = red(" ## ") - ) - - def show_successful_download(down_list, data_transfer): - for repo, branch, fname, cksum in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - basef = os.path.basename(fname) - mytxt += "[%s] %s %s " % (brown(basef),darkred(_("success")),blue("@"),) - mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = red(" ## ") - ) - mytxt = " %s: %s%s%s" % ( - blue(_("Aggregated transfer rate")), - bold(self.entropyTools.bytesIntoHuman(data_transfer)), - darkred("/"), - darkblue(_("second")), - ) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = red(" ## ") - ) - - def show_download_error(down_list, rc): - for repo, branch, fname, cksum in down_list: - best_mirror = get_best_mirror(repo) - mirrorcount = repo_uris[repo].index(best_mirror)+1 - mytxt = "( mirror #%s ) " % (mirrorcount,) - mytxt += blue("%s: %s") % ( - _("Error downloading from"), - red(self.entropyTools.spliturl(best_mirror)[1]), - ) - if rc == -1: - mytxt += " - %s." % (_("files not available on this mirror"),) - elif rc == -2: - self.MirrorStatus.add_failing_mirror(best_mirror,1) - mytxt += " - %s." % (_("wrong checksum"),) - elif rc == -3: - mytxt += " - %s." % (_("not found"),) - elif rc == -4: # timeout! - mytxt += " - %s." % (_("timeout error"),) - elif rc == -100: - mytxt += " - %s." % (_("discarded download"),) - else: - self.MirrorStatus.add_failing_mirror(best_mirror, 5) - mytxt += " - %s." % (_("unknown reason"),) - self.updateProgress( - mytxt, - importance = 1, - type = "warning", - header = red(" ## ") - ) - - def remove_failing_mirrors(repos): - for repo in repos: - best_mirror = get_best_mirror(repo) - if remaining[repo]: - remaining[repo].pop(0) - - def check_remaining_mirror_failure(repos): - return [x for x in repos if not remaining.get(x)] - - while 1: - - do_resume = True - timeout_try_count = 50 - while 1: - - fetch_files_list = [] - for repo, branch, fname, cksum in my_download_list: - best_mirror = get_best_mirror(repo) - if best_mirror != None: - mirror_fail_check(repo, best_mirror) - best_mirror = get_best_mirror(repo) - if best_mirror == None: - # at least one package failed to download - # properly, give up with everything - return 3, my_download_list - myuri = os.path.join(best_mirror,fname) - fetch_files_list.append((myuri,None,cksum,branch,)) - - try: - - show_download_summary(my_download_list) - rc, failed_downloads, data_transfer = self.fetch_files( - fetch_files_list, checksum = checksum, - fetch_file_abort_function = fetch_abort_function, - resume = do_resume - ) - if rc == 0: - show_successful_download(my_download_list, data_transfer) - return 0, [] - - # update my_download_list - my_download_list = update_download_list(my_download_list,failed_downloads) - if rc not in (-3,-4,-100,) and failed_downloads and do_resume: - # disable resume - do_resume = False - continue - else: - show_download_error(my_download_list, rc) - if rc == -4: # timeout - timeout_try_count -= 1 - if timeout_try_count > 0: - continue - elif rc == -100: # user discarded fetch - return 1, [] - myrepos = set([x[0] for x in my_download_list]) - remove_failing_mirrors(myrepos) - # make sure we don't have nasty issues - remaining_failure = check_remaining_mirror_failure(myrepos) - if remaining_failure: - return 3, my_download_list - break - except KeyboardInterrupt: - return 1, [] - return 0, [] - - - def fetch_file(self, url, branch, digest = None, resume = True, fetch_file_abort_function = None, filepath = None): - - filename = os.path.basename(url) - if not filepath: - filepath = os.path.join(etpConst['packagesbindir'],branch,filename) - filepath_dir = os.path.dirname(filepath) - if not os.path.isdir(filepath_dir): - os.makedirs(filepath_dir,0755) - - # load class - fetchConn = self.Client.urlFetcher(url, filepath, resume = resume, - abort_check_func = fetch_file_abort_function, OutputInterface = self) - fetchConn.progress = self.Client.progress - - # start to download - data_transfer = 0 - resumed = False - try: - fetchChecksum = fetchConn.download() - data_transfer = fetchConn.get_transfer_rate() - resumed = fetchConn.is_resumed() - except KeyboardInterrupt: - return -100, data_transfer, resumed - except NameError: - raise - except: - if etpUi['debug']: - self.updateProgress( - "fetch_file:", - importance = 1, - type = "warning", - header = red(" ## ") - ) - self.entropyTools.printTraceback() - return -1, data_transfer, resumed - if fetchChecksum == "-3": - # not found - return -3, data_transfer, resumed - elif fetchChecksum == "-4": - # timeout - return -4, data_transfer, resumed - - del fetchConn - if digest: - if fetchChecksum != digest: - # not properly downloaded - return -2, data_transfer, resumed - else: - return 0, data_transfer, resumed - return 0, data_transfer, resumed - - - def fetch_file_on_mirrors(self, repository, branch, filename, - digest = False, fetch_abort_function = None): - - uris = etpRepositories[repository]['packages'][::-1] - remaining = set(uris) - - mirrorcount = 0 - for uri in uris: - - if not remaining: - # tried all the mirrors, quitting for error - return 3 - - mirrorcount += 1 - mirrorCountText = "( mirror #%s ) " % (mirrorcount,) - url = uri+"/"+filename - - # check if uri is sane - if self.MirrorStatus.get_failing_mirror_status(uri) >= 30: - # ohohoh! - # set to 30 for convenience - self.MirrorStatus.set_failing_mirror_status(uri, 30) - mytxt = mirrorCountText - mytxt += blue(" %s: ") % (_("Mirror"),) - mytxt += red(self.entropyTools.spliturl(uri)[1]) - mytxt += " - %s." % (_("maximum failure threshold reached"),) - self.updateProgress( - mytxt, - importance = 1, - type = "warning", - header = red(" ## ") - ) - - if self.MirrorStatus.get_failing_mirror_status(uri) == 30: - # put to 75 then decrement by 4 so we - # won't reach 30 anytime soon ahahaha - self.MirrorStatus.add_failing_mirror(uri,45) - elif self.MirrorStatus.get_failing_mirror_status(uri) > 31: - # now decrement each time this point is reached, - # if will be back < 30, then equo will try to use it again - self.MirrorStatus.add_failing_mirror(uri,-4) - else: - # put to 0 - reenable mirror, welcome back uri! - self.MirrorStatus.set_failing_mirror_status(uri, 0) - - remaining.discard(uri) - continue - - do_resume = True - timeout_try_count = 50 - while 1: - try: - mytxt = mirrorCountText - mytxt += blue("%s: ") % (_("Downloading from"),) - mytxt += red(self.entropyTools.spliturl(uri)[1]) - # now fetch the new one - self.updateProgress( - mytxt, - importance = 1, - type = "warning", - header = red(" ## ") - ) - rc, data_transfer, resumed = self.fetch_file( - url, - branch, - digest, - do_resume, - fetch_file_abort_function = fetch_abort_function - ) - if rc == 0: - mytxt = mirrorCountText - mytxt += blue("%s: ") % (_("Successfully downloaded from"),) - mytxt += red(self.entropyTools.spliturl(uri)[1]) - mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),) - self.updateProgress( - mytxt, - importance = 1, - type = "info", - header = red(" ## ") - ) - - return 0 - elif resumed and rc not in (-3,-4,-100,): - do_resume = False - continue - else: - error_message = mirrorCountText - error_message += blue("%s: %s") % ( - _("Error downloading from"), - red(self.entropyTools.spliturl(uri)[1]), - ) - # something bad happened - if rc == -1: - error_message += " - %s." % (_("file not available on this mirror"),) - elif rc == -2: - self.MirrorStatus.add_failing_mirror(uri,1) - error_message += " - %s." % (_("wrong checksum"),) - elif rc == -3: - error_message += " - %s." % (_("not found"),) - elif rc == -4: # timeout! - timeout_try_count -= 1 - if timeout_try_count > 0: - error_message += " - %s." % (_("timeout, retrying on this mirror"),) - else: - error_message += " - %s." % (_("timeout, giving up"),) - elif rc == -100: - error_message += " - %s." % (_("discarded download"),) - else: - self.MirrorStatus.add_failing_mirror(uri, 5) - error_message += " - %s." % (_("unknown reason"),) - self.updateProgress( - error_message, - importance = 1, - type = "warning", - header = red(" ## ") - ) - if rc == -4: # timeout - if timeout_try_count > 0: - continue - elif rc == -100: # user discarded fetch - return 1 - remaining.discard(uri) - # make sure we don't have nasty issues - if not remaining: - return 3 - break - except KeyboardInterrupt: - return 1 - return 0 - - -class _MetadataExtractors: - - def __init__(self, ClientInterface): - if not isinstance(ClientInterface,Client): - mytxt = _("A valid Client instance or subclass is needed") - raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) - self.Client = ClientInterface - self.entropyTools = self.Client.entropyTools - self.updateProgress = self.Client.updateProgress - self.SystemSettings = self.Client.SystemSettings - - def _extract_pkg_metadata_generate_extraction_dict(self): - data = { - 'chost': { - 'path': etpConst['spm']['xpak_entries']['chost'], - 'critical': True, - }, - 'description': { - 'path': etpConst['spm']['xpak_entries']['description'], - 'critical': False, - }, - 'homepage': { - 'path': etpConst['spm']['xpak_entries']['homepage'], - 'critical': False, - }, - 'slot': { - 'path': etpConst['spm']['xpak_entries']['slot'], - 'critical': False, - }, - 'cflags': { - 'path': etpConst['spm']['xpak_entries']['cflags'], - 'critical': False, - }, - 'cxxflags': { - 'path': etpConst['spm']['xpak_entries']['cxxflags'], - 'critical': False, - }, - 'category': { - 'path': etpConst['spm']['xpak_entries']['category'], - 'critical': True, - }, - 'rdepend': { - 'path': etpConst['spm']['xpak_entries']['rdepend'], - 'critical': False, - }, - 'pdepend': { - 'path': etpConst['spm']['xpak_entries']['pdepend'], - 'critical': False, - }, - 'depend': { - 'path': etpConst['spm']['xpak_entries']['depend'], - 'critical': False, - }, - 'use': { - 'path': etpConst['spm']['xpak_entries']['use'], - 'critical': False, - }, - 'iuse': { - 'path': etpConst['spm']['xpak_entries']['iuse'], - 'critical': False, - }, - 'license': { - 'path': etpConst['spm']['xpak_entries']['license'], - 'critical': False, - }, - 'provide': { - 'path': etpConst['spm']['xpak_entries']['provide'], - 'critical': False, - }, - 'sources': { - 'path': etpConst['spm']['xpak_entries']['src_uri'], - 'critical': False, - }, - 'eclasses': { - 'path': etpConst['spm']['xpak_entries']['inherited'], - 'critical': False, - }, - 'counter': { - 'path': etpConst['spm']['xpak_entries']['counter'], - 'critical': False, - }, - 'keywords': { - 'path': etpConst['spm']['xpak_entries']['keywords'], - 'critical': False, - }, - } - return data - - def _extract_pkg_metadata_content(self, content_file, package_path): - - pkg_content = {} - - if os.path.isfile(content_file): - f = open(content_file,"r") - content = f.readlines() - f.close() - outcontent = set() - for line in content: - line = line.strip().split() - try: - datatype = line[0] - datafile = line[1:] - if datatype == 'obj': - datafile = datafile[:-2] - datafile = ' '.join(datafile) - elif datatype == 'dir': - datafile = ' '.join(datafile) - elif datatype == 'sym': - datafile = datafile[:-3] - datafile = ' '.join(datafile) - else: - myexc = "InvalidData: %s %s. %s." % ( - datafile, - _("not supported"), - _("Probably Portage API has changed"), - ) - raise InvalidData(myexc) - outcontent.add((datafile,datatype)) - except: - pass - - _outcontent = set() - for i in outcontent: - i = list(i) - datatype = i[1] - _outcontent.add((i[0],i[1])) - outcontent = sorted(_outcontent) - for i in outcontent: - pkg_content[i[0]] = i[1] - - else: - - # CONTENTS is not generated when a package is emerged with portage and the option -B - # we have to unpack the tbz2 and generate content dict - mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject" - if os.path.isdir(mytempdir): - shutil.rmtree(mytempdir) - if not os.path.isdir(mytempdir): - os.makedirs(mytempdir) - - self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True) - for currentdir, subdirs, files in os.walk(mytempdir): - pkg_content[currentdir[len(mytempdir):]] = "dir" - for item in files: - item = currentdir+"/"+item - if os.path.islink(item): - pkg_content[item[len(mytempdir):]] = "sym" - else: - pkg_content[item[len(mytempdir):]] = "obj" - - # now remove - shutil.rmtree(mytempdir,True) - try: os.rmdir(mytempdir) - except (OSError,): pass - - return pkg_content - - def _extract_pkg_metadata_needed(self, needed_file): - - pkg_needed = set() - lines = [] - - try: - f = open(needed_file,"r") - lines = [x.strip() for x in f.readlines() if x.strip()] - f.close() - except IOError: - return lines - - for line in lines: - needed = line.split() - if len(needed) == 2: - ownlib = needed[0] - ownelf = -1 - if os.access(ownlib,os.R_OK): - ownelf = self.entropyTools.read_elf_class(ownlib) - for lib in needed[1].split(","): - #if lib.find(".so") != -1: - pkg_needed.add((lib,ownelf)) - - return list(pkg_needed) - - def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False): - - pkg_messages = [] - - if os.path.isdir(log_dir): - - elogfiles = os.listdir(log_dir) - myelogfile = "%s:%s-%s" % (category, name, version,) - foundfiles = [x for x in elogfiles if x.startswith(myelogfile)] - if foundfiles: - elogfile = foundfiles[0] - if len(foundfiles) > 1: - # get the latest - mtimes = [] - for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item)) - mtimes = sorted(mtimes) - elogfile = mtimes[-1][1] - messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile)) - for message in messages: - message = message.replace("emerge","install") - pkg_messages.append(message) - - elif not silent: - - mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),) - self.updateProgress( - red(log_dir)+mytxt, - importance = 1, - type = "warning", - header = brown(" * ") - ) - - return pkg_messages - - def _extract_pkg_metadata_license_data(self, licenses_dir, license_string): - - pkg_licensedata = {} - if licenses_dir and os.path.isdir(licenses_dir): - licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())] - for mylicense in licdata: - licfile = os.path.join(licenses_dir,mylicense) - if os.access(licfile,os.R_OK): - if self.entropyTools.istextfile(licfile): - f = open(licfile) - pkg_licensedata[mylicense] = f.read() - f.close() - - return pkg_licensedata - - def _extract_pkg_metadata_mirror_links(self, Spm, sources_list): - - # =mirror://openoffice|link1|link2|link3 - pkg_links = [] - for i in sources_list: - if i.startswith("mirror://"): - # parse what mirror I need - mirrorURI = i.split("/")[2] - mirrorlist = Spm.get_third_party_mirrors(mirrorURI) - pkg_links.append([mirrorURI,mirrorlist]) - # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] - - return pkg_links - - def _extract_pkg_metadata_ebuild_entropy_tag(self, ebuild): - search_tag = etpConst['spm']['ebuild_pkg_tag_var'] - ebuild_tag = '' - f = open(ebuild,"r") - tags = [x.strip() for x in f.readlines() if x.strip() and x.strip().startswith(search_tag)] - f.close() - if not tags: return ebuild_tag - tag = tags[-1] - tag = tag.split("=")[-1].strip('"').strip("'").strip() - return tag - - # This function extracts all the info from a .tbz2 file and returns them - def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False): - - data = {} - info_package = bold(os.path.basename(package))+": " - - if not silent: - self.updateProgress( - red(info_package+_("Extracting package metadata")+" ..."), - importance = 0, - type = "info", - header = brown(" * "), - back = True - ) - - filepath = package - tbz2File = package - package = package.split(etpConst['packagesext'])[0] - package = self.entropyTools.remove_entropy_revision(package) - package = self.entropyTools.remove_tag(package) - # remove entropy category - if package.find(":") != -1: - package = ':'.join(package.split(":")[1:]) - - # pkgcat is always == "null" here - pkgcat, pkgname, pkgver, pkgrev = self.entropyTools.catpkgsplit(os.path.basename(package)) - if pkgrev != "r0": pkgver += "-%s" % (pkgrev,) - - # Fill Package name and version - data['name'] = pkgname - data['version'] = pkgver - data['digest'] = self.entropyTools.md5sum(tbz2File) - data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File)) - data['size'] = str(self.entropyTools.get_file_size(tbz2File)) - - tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/" - if not os.path.isdir(tbz2TmpDir): - if os.path.lexists(tbz2TmpDir): - os.remove(tbz2TmpDir) - os.makedirs(tbz2TmpDir) - self.entropyTools.extractXpak(tbz2File,tbz2TmpDir) - - data['injected'] = False - if inject: data['injected'] = True - data['branch'] = etpBranch - - portage_entries = self._extract_pkg_metadata_generate_extraction_dict() - for item in portage_entries: - value = '' - try: - f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r") - value = f.readline().strip() - f.close() - except IOError: - if portage_entries[item]['critical']: - raise - data[item] = value - - # setup vars - data['eclasses'] = data['eclasses'].split() - try: - data['counter'] = int(data['counter']) - except ValueError: - data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database - data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()] - if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords - needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed']) - data['needed'] = self._extract_pkg_metadata_needed(needed_file) - content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents']) - data['content'] = self._extract_pkg_metadata_content(content_file, filepath) - data['disksize'] = self.entropyTools.sum_file_sizes(data['content']) - - # [][][] Kernel dependent packages hook [][][] - data['versiontag'] = '' - kernelstuff = False - kernelstuff_kernel = False - for item in data['content']: - if item.startswith("/lib/modules/"): - kernelstuff = True - # get the version of the modules - kmodver = item.split("/lib/modules/")[1] - kmodver = kmodver.split("/")[0] - - lp = kmodver.split("-")[-1] - if lp.startswith("r"): - kname = kmodver.split("-")[-2] - kver = kmodver.split("-")[0]+"-"+kmodver.split("-")[-1] - else: - kname = kmodver.split("-")[-1] - kver = kmodver.split("-")[0] - break - # validate the results above - if kernelstuff: - matchatom = "linux-%s-%s" % (kname,kver,) - if (matchatom == data['name']+"-"+data['version']): - kernelstuff_kernel = True - - data['versiontag'] = kmodver - if not kernelstuff_kernel: - data['slot'] = kmodver # if you change this behaviour, - # you must change "reagent update" - # and "equo database gentoosync" consequentially - - file_ext = etpConst['spm']['ebuild_file_extension'] - ebuilds_in_path = [x for x in os.listdir(tbz2TmpDir) if x.endswith(".%s" % (file_ext,))] - if not data['versiontag'] and ebuilds_in_path: - # has the user specified a custom package tag inside the ebuild - ebuild_path = os.path.join(tbz2TmpDir,ebuilds_in_path[0]) - data['versiontag'] = self._extract_pkg_metadata_ebuild_entropy_tag(ebuild_path) - - - data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/" - data['download'] += self.entropyTools.create_package_filename(data['category'], data['name'], data['version'], data['versiontag']) - - - data['trigger'] = "" - if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']): - f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb") - data['trigger'] = f.read() - f.close() - - Spm = self.Client.Spm() - - # Get Spm ChangeLog - pkgatom = "%s/%s-%s" % (data['category'],data['name'],data['version'],) - try: - data['changelog'] = Spm.get_package_changelog(pkgatom) - except: - data['changelog'] = None - - portage_metadata = Spm.calculate_dependencies( - data['iuse'], data['use'], data['license'], data['depend'], - data['rdepend'], data['pdepend'], data['provide'], data['sources'] - ) - - data['provide'] = portage_metadata['PROVIDE'].split() - data['license'] = portage_metadata['LICENSE'] - data['useflags'] = [] - for x in data['use'].split(): - if x.startswith("+"): - x = x[1:] - elif x.startswith("-"): - x = x[1:] - if (x in portage_metadata['USE']) or (x in portage_metadata['USE_MASK']): - data['useflags'].append(x) - else: - data['useflags'].append("-"+x) - data['sources'] = portage_metadata['SRC_URI'].split() - data['dependencies'] = {} - for x in portage_metadata['RDEPEND'].split(): - if x.startswith("!") or (x in ("(","||",")","")): - continue - data['dependencies'][x] = etpConst['spm']['(r)depend_id'] - for x in portage_metadata['PDEPEND'].split(): - if x.startswith("!") or (x in ("(","||",")","")): - continue - data['dependencies'][x] = etpConst['spm']['pdepend_id'] - data['conflicts'] = [x[1:] for x in portage_metadata['RDEPEND'].split()+portage_metadata['PDEPEND'].split() if x.startswith("!") and not x in ("(","||",")","")] - - if (kernelstuff) and (not kernelstuff_kernel): - # add kname to the dependency - data['dependencies']["=sys-kernel/linux-"+kname+"-"+kver+"~-1"] = etpConst['spm']['(r)depend_id'] - - # Conflicting tagged packages support - key = data['category']+"/"+data['name'] - confl_data = self.SystemSettings['conflicting_tagged_packages'].get(key) - if confl_data != None: - for conflict in confl_data: data['conflicts'].append(conflict) - - # Get License text if possible - licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses') - data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license']) - data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources']) - - # write only if it's a systempackage - data['systempackage'] = False - system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()] - if data['category']+"/"+data['name'] in system_packages: - data['systempackage'] = True - - # write only if it's a systempackage - protect, mask = Spm.get_config_protect_and_mask() - data['config_protect'] = protect - data['config_protect_mask'] = mask - - log_dir = etpConst['logdir']+"/elog" - if not os.path.isdir(log_dir): os.makedirs(log_dir) - data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent) - data['etpapi'] = etpConst['etpapi'] - - # removing temporary directory - shutil.rmtree(tbz2TmpDir,True) - if os.path.isdir(tbz2TmpDir): - try: os.remove(tbz2TmpDir) - except OSError: pass - - if not silent: - self.updateProgress( - red(info_package+_("Package extraction complete")), importance = 0, - type = "info", header = brown(" * "), back = True - ) - return data - - - - -class Client(Singleton,TextInterface,_ClientLoaders,_ClientCache, - _ClientDepCalculators,_RepositoryMethods,_MiscMethods, - _MatchMethods,_ClientFetchers,_MetadataExtractors): +from entropy.misc import TimeScheduled + +from entropy.client.interfaces.loaders import Loaders +from entropy.client.interfaces.cache import Cache +from entropy.client.interfaces.dep import Calculators +from entropy.client.interfaces.methods import Repository as CRepository, Misc, Match +from entropy.client.interfaces.fetch import Fetchers +from entropy.client.interfaces.metadata import Extractors + +class Client(Singleton,TextInterface,Loaders,Cache, + Calculators,CRepository,Misc, + Match,Fetchers,Extractors): def init_singleton(self, indexing = True, noclientdb = 0, xcache = True, user_xcache = False, repo_validation = True, @@ -4422,14 +105,14 @@ class Client(Singleton,TextInterface,_ClientLoaders,_ClientCache, self.UGC = ugcClient(self) # class init - _ClientLoaders.__init__(self, self) - _ClientCache.__init__(self, self) - _ClientDepCalculators.__init__(self, self) - _RepositoryMethods.__init__(self, self) - _MiscMethods.__init__(self, self) - _MatchMethods.__init__(self, self) - _ClientFetchers.__init__(self, self) - _MetadataExtractors.__init__(self, self) + Loaders.__init__(self, self) + Cache.__init__(self, self) + Calculators.__init__(self, self) + CRepository.__init__(self, self) + Misc.__init__(self, self) + Match.__init__(self, self) + Fetchers.__init__(self, self) + Extractors.__init__(self, self) if noclientdb in (False,0): self.noclientdb = False diff --git a/libraries/entropy/client/interfaces/cache.py b/libraries/entropy/client/interfaces/cache.py new file mode 100644 index 000000000..119f9fe56 --- /dev/null +++ b/libraries/entropy/client/interfaces/cache.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +import os +from entropy.const import * +from entropy.exceptions import * +from entropy.output import red, darkred, darkgreen + +class Cache: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.Cacher = self.Client.Cacher + self.updateProgress = self.Client.updateProgress + self.FileUpdates = self.Client.FileUpdates + self.entropyTools = self.Client.entropyTools + + def validate_repositories_cache(self): + # is the list of repos changed? + cached = self.Cacher.pop(etpCache['repolist']) + if cached == None: + # invalidate matching cache + try: self.repository_move_clear_cache() + except IOError: pass + elif isinstance(cached,tuple): + difflist = [x for x in cached if x not in etpRepositoriesOrder] + for repoid in difflist: + try: self.repository_move_clear_cache(repoid) + except IOError: pass + self.store_repository_list_cache() + + def store_repository_list_cache(self): + self.Cacher.push(etpCache['repolist'],tuple(etpRepositoriesOrder), async = False) + + def generate_cache(self, depcache = True, configcache = True, client_purge = True, install_queue = True): + self.Cacher.stop() + # clean first of all + self.purge_cache(client_purge = client_purge) + if depcache: + self.do_depcache(do_install_queue = install_queue) + if configcache: + self.do_configcache() + self.Cacher.start() + + def do_configcache(self): + self.updateProgress( + darkred(_("Configuration files")), + importance = 2, + type = "warning" + ) + self.updateProgress( + red(_("Scanning hard disk")), + importance = 1, + type = "warning" + ) + self.FileUpdates.scanfs(dcache = False, quiet = True) + self.updateProgress( + darkred(_("Cache generation complete.")), + importance = 2, + type = "info" + ) + + def do_depcache(self, do_install_queue = True): + + self.updateProgress( + darkgreen(_("Resolving metadata")), + importance = 1, + type = "warning" + ) + # we can barely ignore any exception from here + # especially cases where client db does not exist + try: + update, remove, fine = self.Client.calculate_world_updates() + del fine, remove + if do_install_queue: + self.Client.get_install_queue(update, False, False) + self.Client.calculate_available_packages() + except: + pass + + self.updateProgress( + darkred(_("Dependencies cache filled.")), + importance = 2, + type = "warning" + ) + + def purge_cache(self, showProgress = True, client_purge = True): + if self.entropyTools.is_user_in_entropy_group(): + skip = set() + if not client_purge: + skip.add("/"+etpCache['dbMatch']+"/"+etpConst['clientdbid']) # it's ok this way + skip.add("/"+etpCache['dbSearch']+"/"+etpConst['clientdbid']) # it's ok this way + for key in etpCache: + if showProgress: + self.Client.updateProgress( + darkred(_("Cleaning %s => dumps...")) % (etpCache[key],), + importance = 1, + type = "warning", + back = True + ) + self.clear_dump_cache(etpCache[key], skip = skip) + + if showProgress: + self.updateProgress( + darkgreen(_("Cache is now empty.")), + importance = 2, + type = "info" + ) + + def clear_dump_cache(self, dump_name, skip = []): + self.Cacher.sync(wait = True) + dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name) + dump_dir = os.path.dirname(dump_path) + #dump_file = os.path.basename(dump_path) + for currentdir, subdirs, files in os.walk(dump_dir): + path = os.path.join(dump_dir,currentdir) + if skip: + found = False + for myskip in skip: + if path.find(myskip) != -1: + found = True + break + if found: continue + for item in files: + if item.endswith(etpConst['cachedumpext']): + item = os.path.join(path,item) + try: os.remove(item) + except OSError: pass + try: + if not os.listdir(path): + os.rmdir(path) + except OSError: + pass + + def update_ugc_cache(self, repository): + if not self.Client.UGC.is_repository_eapi3_aware(repository): + return None + status = True + + votes_dict, err_msg = self.Client.UGC.get_all_votes(repository) + if isinstance(votes_dict,dict): + self.Client.UGC.UGCCache.save_vote_cache(repository, votes_dict) + else: + status = False + + downloads_dict, err_msg = self.Client.UGC.get_all_downloads(repository) + if isinstance(downloads_dict,dict): + self.Client.UGC.UGCCache.save_downloads_cache(repository, downloads_dict) + else: + status = False + return status + + def repository_move_clear_cache(self, repoid = None): + self.clear_dump_cache(etpCache['world_available']) + self.clear_dump_cache(etpCache['world_update']) + self.clear_dump_cache(etpCache['check_package_update']) + self.clear_dump_cache(etpCache['filter_satisfied_deps']) + self.clear_dump_cache(self.Client.atomMatchCacheKey) + self.clear_dump_cache(etpCache['dep_tree']) + if repoid != None: + self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,)) + self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,)) + + def get_available_packages_chash(self, branch): + # client digest not needed, cache is kept updated + return str(hash("%s%s%s" % ( + self.all_repositories_checksum(), + branch,self.Client.validRepositories,))) + + def all_repositories_checksum(self): + sum_hashes = '' + for repo in self.Client.validRepositories: + try: + dbconn = self.Client.open_repository(repo) + except (RepositoryError): + continue # repo not available + try: + sum_hashes += dbconn.database_checksum() + except self.Client.dbapi2.OperationalError: + pass + return sum_hashes + + def get_available_packages_cache(self, branch = etpConst['branch'], myhash = None): + if myhash == None: myhash = self.get_available_packages_chash(branch) + return self.Cacher.pop("%s%s" % (etpCache['world_available'],myhash)) + + def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None, ignore_spm_downgrades = False): + if self.Client.xcache: + if db_digest == None: db_digest = self.all_repositories_checksum() + c_hash = "%s%s" % (etpCache['world_update'], + self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades),) + disk_cache = self.Cacher.pop(c_hash) + if disk_cache != None: + try: + return disk_cache['r'] + except (KeyError, TypeError): + return None + + def get_world_update_cache_hash(self, db_digest, empty_deps, branch, ignore_spm_downgrades): + return str(hash("%s|%s|%s|%s|%s|%s" % ( + db_digest,empty_deps,self.Client.validRepositories, + etpRepositoriesOrder, branch, ignore_spm_downgrades, + ))) diff --git a/libraries/entropy/client/interfaces/dep.py b/libraries/entropy/client/interfaces/dep.py new file mode 100644 index 000000000..76db50963 --- /dev/null +++ b/libraries/entropy/client/interfaces/dep.py @@ -0,0 +1,1526 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +from entropy.misc import Lifo +from entropy.const import * +from entropy.exceptions import * +from entropy.output import bold, darkgreen, darkred, blue, red + + +class Calculators: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.Cacher = self.Client.Cacher + self.updateProgress = self.Client.updateProgress + self.entropyTools = self.Client.entropyTools + self.dumpTools = self.Client.dumpTools + self.SystemSettings = self.Client.SystemSettings + self.dbapi2 = self.Client.dbapi2 + + def dependencies_test(self, dbconn = None): + + if dbconn == None: + dbconn = self.Client.clientDbconn + # get all the installed packages + installedPackages = dbconn.listAllIdpackages() + + deps_not_matched = set() + # now look + length = len(installedPackages) + count = 0 + for xidpackage in installedPackages: + count += 1 + atom = dbconn.retrieveAtom(xidpackage) + self.updateProgress( + darkgreen(_("Checking %s") % (bold(atom),)), + importance = 0, + type = "info", + back = True, + count = (count,length), + header = darkred(" @@ ") + ) + + xdeps = dbconn.retrieveDependencies(xidpackage) + needed_deps = set() + for xdep in xdeps: + xmatch = dbconn.atomMatch(xdep) + if xmatch[0] == -1: + needed_deps.add(xdep) + + deps_not_matched |= needed_deps + + return deps_not_matched + + def find_belonging_dependency(self, matched_atoms): + crying_atoms = set() + for atom in matched_atoms: + for repo in self.Client.validRepositories: + rdbconn = self.Client.open_repository(repo) + riddep = rdbconn.searchDependency(atom) + if riddep != -1: + ridpackages = rdbconn.searchIdpackageFromIddependency(riddep) + for i in ridpackages: + i,r = rdbconn.idpackageValidator(i) + if i == -1: + continue + iatom = rdbconn.retrieveAtom(i) + crying_atoms.add((iatom,repo)) + return crying_atoms + + def __handle_multi_repo_matches(self, results, extended_results, valid_repos, server_inst): + + packageInformation = {} + versionInformation = {} + # .tbz2 repos have always the precedence, so if we find them, + # we should second what user wants, installing his tbz2 + tbz2repos = [x for x in results if x.endswith(etpConst['packagesext'])] + if tbz2repos: + del tbz2repos + newrepos = results.copy() + for x in newrepos: + if x.endswith(etpConst['packagesext']): + continue + del results[x] + + version_duplicates = set() + versions = set() + for repo in results: + packageInformation[repo] = {} + if extended_results: + version = results[repo][1] + packageInformation[repo]['versiontag'] = results[repo][2] + packageInformation[repo]['revision'] = results[repo][3] + else: + dbconn = self.__atom_match_open_db(repo, server_inst) + packageInformation[repo]['versiontag'] = dbconn.retrieveVersionTag(results[repo]) + packageInformation[repo]['revision'] = dbconn.retrieveRevision(results[repo]) + version = dbconn.retrieveVersion(results[repo]) + packageInformation[repo]['version'] = version + versionInformation[version] = repo + if version in versions: + version_duplicates.add(version) + versions.add(version) + + newerVersion = self.entropyTools.getNewerVersion(list(versions))[0] + # if no duplicates are found or newer version is not in duplicates we're done + if (not version_duplicates) or (newerVersion not in version_duplicates): + reponame = versionInformation.get(newerVersion) + return (results[reponame],reponame) + + # we have two repositories with >two packages with the same version + # check package tag + + conflictingEntries = {} + tags_duplicates = set() + tags = set() + tagsInfo = {} + for repo in packageInformation: + if packageInformation[repo]['version'] != newerVersion: + continue + conflictingEntries[repo] = {} + versiontag = packageInformation[repo]['versiontag'] + if versiontag in tags: + tags_duplicates.add(versiontag) + tags.add(versiontag) + tagsInfo[versiontag] = repo + conflictingEntries[repo]['versiontag'] = versiontag + conflictingEntries[repo]['revision'] = packageInformation[repo]['revision'] + + # tags will always be != [] + newerTag = sorted(list(tags), reverse = True)[0] + if newerTag not in tags_duplicates: + reponame = tagsInfo.get(newerTag) + return (results[reponame],reponame) + + # in this case, we have >two packages with the same version and tag + # check package revision + + conflictingRevisions = {} + revisions = set() + revisions_duplicates = set() + revisionInfo = {} + for repo in conflictingEntries: + if conflictingEntries[repo]['versiontag'] == newerTag: + conflictingRevisions[repo] = {} + versionrev = conflictingEntries[repo]['revision'] + if versionrev in revisions: + revisions_duplicates.add(versionrev) + revisions.add(versionrev) + revisionInfo[versionrev] = repo + conflictingRevisions[repo]['revision'] = versionrev + + newerRevision = max(revisions) + if newerRevision not in revisions_duplicates: + reponame = revisionInfo.get(newerRevision) + return (results[reponame],reponame) + + # final step, in this case we have >two packages with the same version, tag and revision + # get the repository with the biggest priority + + for reponame in valid_repos: + if reponame in conflictingRevisions: + return (results[reponame],reponame) + + def __validate_atom_match_cache(self, cached_obj, multiMatch, extendedResults, multiRepo, server_inst): + + data, rc = cached_obj + if rc == 1: return cached_obj + + if multiRepo or multiMatch: + matches = data # set([(14789, 'sabayonlinux.org'), (14479, 'sabayonlinux.org')]) + if extendedResults: + # set([((14789, u'3.3.8b', u'', 0), 'sabayonlinux.org')]) + matches = [(x[0][0],x[1],) for x in data] + for m_id, m_repo in matches: + m_db = self.__atom_match_open_db(m_repo, server_inst) + if not m_db.isIDPackageAvailable(m_id): return None + else: + m_id, m_repo = cached_obj # (14479, 'sabayonlinux.org') + if extendedResults: + # ((14479, u'4.4.2', u'', 0), 'sabayonlinux.org') + m_id, m_repo = cached_obj[0][0],cached_obj[1] + m_db = self.__atom_match_open_db(m_repo, server_inst) + if not m_db.isIDPackageAvailable(m_id): return None + + return cached_obj + + def __atom_match_open_db(self, repoid, server_inst): + if server_inst != None: + dbconn = server_inst.openServerDatabase(just_reading = True, repo = repoid) + else: + dbconn = self.Client.open_repository(repoid) + return dbconn + + def atom_match(self, atom, caseSensitive = True, matchSlot = None, + matchBranches = (), matchTag = None, packagesFilter = True, + multiMatch = False, multiRepo = False, matchRevision = None, + matchRepo = None, server_repos = [], serverInstance = None, + extendedResults = False, useCache = True): + + # support match in repository from shell + # atom@repo1,repo2,repo3 + atom, repos = self.entropyTools.dep_get_match_in_repos(atom) + if (matchRepo == None) and (repos != None): + matchRepo = repos + + u_hash = "" + m_hash = "" + k_ms = "//" + k_mt = "@#@" + k_mr = "-1" + if isinstance(matchRepo,(list,tuple,set,)): u_hash = hash(frozenset(matchRepo)) + if isinstance(matchBranches,(list,tuple,set,)): m_hash = hash(frozenset(matchBranches)) + if isinstance(matchSlot,basestring): k_ms = matchSlot + if isinstance(matchTag,basestring): k_mt = matchTag + if isinstance(matchRevision,basestring): k_mr = matchRevision + + c_hash = "|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s" % ( + atom,k_ms,k_mt,hash(packagesFilter), + hash(frozenset(self.Client.validRepositories)), + hash(frozenset(etpRepositories)), + hash(multiMatch),hash(multiRepo),hash(caseSensitive), + k_mr,hash(extendedResults), + u_hash, m_hash + ) + c_hash = "%s%s" % (self.Client.atomMatchCacheKey,hash(c_hash),) + + if self.Client.xcache and useCache: + cached = self.Cacher.pop(c_hash) + if cached != None: + try: + cached = self.__validate_atom_match_cache(cached, multiMatch, extendedResults, multiRepo, serverInstance) + except (TypeError,ValueError,IndexError,KeyError,): + cached = None + if cached != None: + return cached + + if server_repos: + if not serverInstance: + t = _("server_repos needs serverInstance") + raise IncorrectParameter("IncorrectParameter: %s" % (t,)) + valid_repos = server_repos[:] + else: + valid_repos = self.Client.validRepositories + if matchRepo and (type(matchRepo) in (list,tuple,set)): + valid_repos = list(matchRepo) + + repoResults = {} + for repo in valid_repos: + + # search + dbconn = self.__atom_match_open_db(repo, serverInstance) + use_cache = useCache + while 1: + try: + query_data, query_rc = dbconn.atomMatch( + atom, + caseSensitive = caseSensitive, + matchSlot = matchSlot, + matchBranches = matchBranches, + matchTag = matchTag, + packagesFilter = packagesFilter, + matchRevision = matchRevision, + extendedResults = extendedResults, + useCache = use_cache + ) + if query_rc == 0: + # package found, add to our dictionary + if extendedResults: + repoResults[repo] = (query_data[0],query_data[2],query_data[3],query_data[4]) + else: + repoResults[repo] = query_data + except TypeError: + if not use_cache: + raise + use_cache = False + continue + break + + dbpkginfo = (-1,1) + if extendedResults: + dbpkginfo = ((-1,None,None,None),1) + + if multiRepo and repoResults: + + data = set() + for repoid in repoResults: + data.add((repoResults[repoid],repoid)) + dbpkginfo = (data,0) + + elif len(repoResults) == 1: + # one result found + repo = repoResults.keys()[0] + dbpkginfo = (repoResults[repo],repo) + + elif len(repoResults) > 1: + + # we have to decide which version should be taken + mypkginfo = self.__handle_multi_repo_matches(repoResults, extendedResults, valid_repos, serverInstance) + if mypkginfo != None: dbpkginfo = mypkginfo + + # multimatch support + if multiMatch: + + if dbpkginfo[1] != 1: # can be "0" or a string, but 1 means failure + if multiRepo: + data = set() + for q_id,q_repo in dbpkginfo[0]: + dbconn = self.__atom_match_open_db(q_repo, serverInstance) + query_data, query_rc = dbconn.atomMatch( + atom, + caseSensitive = caseSensitive, + matchSlot = matchSlot, + matchBranches = matchBranches, + matchTag = matchTag, + packagesFilter = packagesFilter, + multiMatch = True, + extendedResults = extendedResults + ) + if extendedResults: + for item in query_data: + data.add(((item[0],item[2],item[3],item[4]),q_repo)) + else: + for x in query_data: data.add((x,q_repo)) + dbpkginfo = (data,0) + else: + dbconn = self.__atom_match_open_db(dbpkginfo[1], serverInstance) + query_data, query_rc = dbconn.atomMatch( + atom, + caseSensitive = caseSensitive, + matchSlot = matchSlot, + matchBranches = matchBranches, + matchTag = matchTag, + packagesFilter = packagesFilter, + multiMatch = True, + extendedResults = extendedResults + ) + if extendedResults: + dbpkginfo = (set([((x[0],x[2],x[3],x[4]),dbpkginfo[1]) for x in query_data]),0) + else: + dbpkginfo = (set([(x,dbpkginfo[1]) for x in query_data]),0) + + if self.Client.xcache and useCache: + self.Cacher.push(c_hash,dbpkginfo) + + return dbpkginfo + + # expands package sets, and in future something more perhaps + def packages_expand(self, packages): + new_packages = [] + + for pkg_id in range(len(packages)): + package = packages[pkg_id] + + # expand package sets + if package.startswith(etpConst['packagesetprefix']): + set_pkgs = sorted(list(self.package_set_expand(package, raise_exceptions = False))) + new_packages.extend([x for x in set_pkgs if x not in packages]) # atomMatch below will filter dupies + else: + new_packages.append(package) + + return new_packages + + def package_set_expand(self, package_set, raise_exceptions = True): + + max_recursion_level = 50 + recursion_level = 0 + + def do_expand(myset, recursion_level, max_recursion_level): + recursion_level += 1 + if recursion_level > max_recursion_level: + raise InvalidPackageSet('InvalidPackageSet: corrupted, too many recursions: %s' % (myset,)) + set_data, set_rc = self.package_set_match(myset[len(etpConst['packagesetprefix']):]) + if not set_rc: + raise InvalidPackageSet('InvalidPackageSet: not found: %s' % (myset,)) + (set_from, package_set, mydata,) = set_data + + mypkgs = set() + for fset in mydata: # recursively + if fset.startswith(etpConst['packagesetprefix']): + mypkgs |= do_expand(fset, recursion_level, max_recursion_level) + else: + mypkgs.add(fset) + + return mypkgs + + if not package_set.startswith(etpConst['packagesetprefix']): + package_set = "%s%s" % (etpConst['packagesetprefix'],package_set,) + + try: + mylist = do_expand(package_set, recursion_level, max_recursion_level) + except InvalidPackageSet: + if raise_exceptions: raise + mylist = set() + + return mylist + + def package_set_list(self, server_repos = [], serverInstance = None, matchRepo = None): + return self.package_set_match('', matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] + + def package_set_search(self, package_set, server_repos = [], serverInstance = None, matchRepo = None): + # search support + if package_set == '*': package_set = '' + return self.package_set_match(package_set, matchRepo = matchRepo, server_repos = server_repos, serverInstance = serverInstance, search = True)[0] + + def __package_set_match_open_db(self, repoid, server_inst): + if server_inst != None: + dbconn = server_inst.openServerDatabase(just_reading = True, repo = repoid) + else: + dbconn = self.Client.open_repository(repoid) + return dbconn + + def package_set_match(self, package_set, multiMatch = False, matchRepo = None, server_repos = [], serverInstance = None, search = False): + + # support match in repository from shell + # set@repo1,repo2,repo3 + package_set, repos = self.entropyTools.dep_get_match_in_repos(package_set) + if (matchRepo == None) and (repos != None): + matchRepo = repos + + if server_repos: + if not serverInstance: + t = _("server_repos needs serverInstance") + raise IncorrectParameter("IncorrectParameter: %s" % (t,)) + valid_repos = server_repos[:] + else: + valid_repos = self.Client.validRepositories + + if matchRepo and (type(matchRepo) in (list,tuple,set)): + valid_repos = list(matchRepo) + + # if we search, we return all the matches available + if search: multiMatch = True + + set_data = [] + + while 1: + + # check inside SystemSettings + if not server_repos: + if search: + mysets = [x for x in self.SystemSettings['system_package_sets'].keys() if (x.find(package_set) != -1)] + for myset in mysets: + mydata = self.SystemSettings['system_package_sets'].get(myset) + set_data.append((etpConst['userpackagesetsid'], unicode(myset), mydata.copy(),)) + else: + mydata = self.SystemSettings['system_package_sets'].get(package_set) + if mydata != None: + set_data.append((etpConst['userpackagesetsid'], unicode(package_set), mydata,)) + if not multiMatch: break + + for repoid in valid_repos: + dbconn = self.__package_set_match_open_db(repoid, serverInstance) + if search: + mysets = dbconn.searchSets(package_set) + for myset in mysets: + mydata = dbconn.retrievePackageSet(myset) + set_data.append((repoid, myset, mydata.copy(),)) + else: + mydata = dbconn.retrievePackageSet(package_set) + if mydata: set_data.append((repoid, package_set, mydata,)) + if not multiMatch: break + + break + + if not set_data: return (),False + if multiMatch: return set_data,True + return set_data.pop(0),True + + def get_unsatisfied_dependencies(self, dependencies, deep_deps = False, depcache = None): + + if self.Client.xcache: + c_data = sorted(dependencies) + client_checksum = self.Client.clientDbconn.database_checksum() + c_hash = hash("%s|%s|%s" % (c_data,deep_deps,client_checksum,)) + c_hash = "%s%s" % (etpCache['filter_satisfied_deps'],c_hash,) + cached = self.dumpTools.loadobj(c_hash) + if cached != None: return cached + + if not isinstance(depcache,dict): + depcache = {} + + cdb_am = self.Client.clientDbconn.atomMatch + am = self.atom_match + open_repo = self.Client.open_repository + intf_error = self.dbapi2.InterfaceError + cdb_getversioning = self.Client.clientDbconn.getVersioningData + #cdb_retrieveneededraw = self.clientDbconn.retrieveNeededRaw + etp_cmp = self.entropyTools.entropyCompareVersions + etp_get_rev = self.entropyTools.dep_get_entropy_revision + #do_needed_check = False + + def fm_dep(dependency): + + cached = depcache.get(dependency) + if cached != None: return cached + + ### conflict + if dependency.startswith("!"): + idpackage,rc = cdb_am(dependency[1:]) + if idpackage != -1: + depcache[dependency] = dependency + return dependency + depcache[dependency] = 0 + return 0 + + c_id,c_rc = cdb_am(dependency) + if c_id == -1: + depcache[dependency] = dependency + return dependency + + #if not deep_deps and not do_needed_check: + # depcache[dependency] = 0 + # return 0 + + r_id,r_repo = am(dependency) + if r_id == -1: + depcache[dependency] = dependency + return dependency + + #if do_needed_check: + # dbconn = open_repo(r_repo) + # installed_needed = cdb_retrieveneededraw(c_id) + # repo_needed = dbconn.retrieveNeededRaw(r_id) + # if installed_needed != repo_needed: + # return dependency + # #elif not deep_deps: + # # return 0 + + dbconn = open_repo(r_repo) + try: + repo_pkgver, repo_pkgtag, repo_pkgrev = dbconn.getVersioningData(r_id) + except (intf_error,TypeError,): + # package entry is broken + return dependency + + try: + installedVer, installedTag, installedRev = cdb_getversioning(c_id) + except TypeError: # corrupted entry? + installedVer = "0" + installedTag = '' + installedRev = 0 + + # support for app-foo/foo-123~-1 + # -1 revision means, always pull the latest + do_deep = deep_deps + if not do_deep: + string_rev = etp_get_rev(dependency) + if string_rev == -1: + do_deep = True + + vcmp = etp_cmp((repo_pkgver,repo_pkgtag,repo_pkgrev,), (installedVer,installedTag,installedRev,)) + if vcmp != 0: + if not do_deep and ((repo_pkgver,repo_pkgtag,) == (installedVer,installedTag,)) and (repo_pkgrev != installedRev): + depcache[dependency] = 0 + return 0 + depcache[dependency] = dependency + return dependency + depcache[dependency] = 0 + return 0 + + unsatisfied = map(fm_dep,dependencies) + unsatisfied = set([x for x in unsatisfied if x != 0]) + + if self.Client.xcache: + self.Cacher.push(c_hash,unsatisfied) + + return unsatisfied + + def get_masked_packages_tree(self, match, atoms = False, flat = False, matchfilter = None): + + if not isinstance(matchfilter,set): + matchfilter = set() + + maskedtree = {} + mybuffer = Lifo() + depcache = set() + treelevel = -1 + + match_id, match_repo = match + + mydbconn = self.Client.open_repository(match_repo) + myatom = mydbconn.retrieveAtom(match_id) + idpackage, idreason = mydbconn.idpackageValidator(match_id) + if idpackage == -1: + treelevel += 1 + if atoms: + mydict = {myatom: idreason,} + else: + mydict = {match: idreason,} + if flat: + maskedtree.update(mydict) + else: + maskedtree[treelevel] = mydict + + mydeps = mydbconn.retrieveDependencies(match_id) + for mydep in mydeps: mybuffer.push(mydep) + mydep = mybuffer.pop() + + open_db = self.Client.open_repository + am = self.atom_match + while mydep: + + if mydep in depcache: + mydep = mybuffer.pop() + continue + depcache.add(mydep) + + idpackage, repoid = am(mydep) + if (idpackage, repoid) in matchfilter: + mydep = mybuffer.pop() + continue + + if idpackage != -1: + # doing even here because atomMatch with packagesFilter = False can pull + # something different + matchfilter.add((idpackage, repoid)) + + # collect masked + if idpackage == -1: + idpackage, repoid = am(mydep, packagesFilter = False) + if idpackage != -1: + treelevel += 1 + if not maskedtree.has_key(treelevel) and not flat: + maskedtree[treelevel] = {} + dbconn = open_db(repoid) + vidpackage, idreason = dbconn.idpackageValidator(idpackage) + if atoms: + mydict = {dbconn.retrieveAtom(idpackage): idreason} + else: + mydict = {(idpackage,repoid): idreason} + if flat: maskedtree.update(mydict) + else: maskedtree[treelevel].update(mydict) + + # push its dep into the buffer + if idpackage != -1: + matchfilter.add((idpackage, repoid)) + dbconn = open_db(repoid) + owndeps = dbconn.retrieveDependencies(idpackage) + for owndep in owndeps: + mybuffer.push(owndep) + + mydep = mybuffer.pop() + + return maskedtree + + + def generate_dependency_tree(self, + matched_atom, empty_deps = False, deep_deps = False, matchfilter = None, + flat = False, filter_unsat_cache = None, treecache = None, keyslotcache = None): + + if not isinstance(matchfilter,set): + matchfilter = set() + if not isinstance(filter_unsat_cache,dict): + filter_unsat_cache = {} + if not isinstance(treecache,set): + treecache = set() + if not isinstance(keyslotcache,set): + keyslotcache = set() + + mydbconn = self.Client.open_repository(matched_atom[1]) + myatom = mydbconn.retrieveAtom(matched_atom[0]) + + # caches + # special events + deps_not_found = set() + conflicts = set() + + mydep = (1,myatom) + mybuffer = Lifo() + deptree = set() + if matched_atom not in matchfilter: + deptree.add((1,matched_atom)) + + virgin = True + open_repo = self.Client.open_repository + atom_match = self.atom_match + cdb_atom_match = self.Client.clientDbconn.atomMatch + lookup_conflict_replacement = self._lookup_conflict_replacement + lookup_library_breakages = self._lookup_library_breakages + lookup_inverse_dependencies = self._lookup_inverse_dependencies + get_unsatisfied_deps = self.get_unsatisfied_dependencies + + def my_dep_filter(x): + if x in treecache: return False + if tuple(x.split(":")) in keyslotcache: return False + return True + + while mydep: + + dep_level, dep_atom = mydep + + # already analyzed in this call + if dep_atom in treecache: + mydep = mybuffer.pop() + continue + treecache.add(dep_atom) + + if dep_atom == None: # corrupted entry + mydep = mybuffer.pop() + continue + + # conflicts + if dep_atom[0] == "!": + c_idpackage, xst = cdb_atom_match(dep_atom[1:]) + if c_idpackage != -1: + myreplacement = lookup_conflict_replacement(dep_atom[1:], c_idpackage, deep_deps = deep_deps) + if (myreplacement != None) and (myreplacement not in treecache): + mybuffer.push((dep_level+1,myreplacement)) + else: + conflicts.add(c_idpackage) + mydep = mybuffer.pop() + continue + + # atom found? + if virgin: + virgin = False + m_idpackage, m_repo = matched_atom + dbconn = open_repo(m_repo) + myidpackage, idreason = dbconn.idpackageValidator(m_idpackage) + if myidpackage == -1: m_idpackage = -1 + else: + m_idpackage, m_repo = atom_match(dep_atom) + if m_idpackage == -1: + deps_not_found.add(dep_atom) + mydep = mybuffer.pop() + continue + + # check if atom has been already pulled in + matchdb = open_repo(m_repo) + matchatom = matchdb.retrieveAtom(m_idpackage) + matchkey, matchslot = matchdb.retrieveKeySlot(m_idpackage) + if (dep_atom != matchatom) and (matchatom in treecache): + mydep = mybuffer.pop() + continue + + treecache.add(matchatom) + + # check if key + slot has been already pulled in + if (matchslot,matchkey) in keyslotcache: + mydep = mybuffer.pop() + continue + else: + keyslotcache.add((matchslot,matchkey)) + + match = (m_idpackage, m_repo,) + # result already analyzed? + if match in matchfilter: + mydep = mybuffer.pop() + continue + + # already analyzed by the calling function + if match in matchfilter: + mydep = mybuffer.pop() + continue + matchfilter.add(match) + + treedepth = dep_level+1 + + # all checks passed, well done + matchfilter.add(match) + deptree.add((dep_level,match)) # add match + + # extra hooks + cm_idpackage, cm_result = cdb_atom_match(matchkey, matchSlot = matchslot) + if cm_idpackage != -1: + broken_atoms = lookup_library_breakages(match, (cm_idpackage, cm_result,), deep_deps = deep_deps) + inverse_deps = lookup_inverse_dependencies(match, (cm_idpackage, cm_result,)) + if inverse_deps: + deptree.remove((dep_level,match)) + for ikey,islot in inverse_deps: + iks_str = '%s:%s' % (ikey,islot,) + if ((ikey,islot) not in keyslotcache) and (iks_str not in treecache): + mybuffer.push((dep_level,iks_str)) + keyslotcache.add((ikey,islot)) + deptree.add((treedepth,match)) + treedepth += 1 + for x in broken_atoms: + if (tuple(x.split(":")) not in keyslotcache) and (x not in treecache): + mybuffer.push((treedepth,x)) + + myundeps = filter(my_dep_filter,matchdb.retrieveDependenciesList(m_idpackage)) + if not empty_deps: + myundeps = filter(my_dep_filter,get_unsatisfied_deps(myundeps, deep_deps, depcache = filter_unsat_cache)) + + # PDEPENDs support + if myundeps: + post_deps = [x for x in matchdb.retrievePostDependencies(m_idpackage) if x in myundeps] + myundeps = [x for x in myundeps if x not in post_deps] + for x in post_deps: mybuffer.push((-1,x)) # always after the package itself + + for x in myundeps: mybuffer.push((treedepth,x)) + mydep = mybuffer.pop() + + if deps_not_found: + return list(deps_not_found),-2 + + if flat: return [x[1] for x in deptree],0 + + newdeptree = {} + for key,item in deptree: + if key not in newdeptree: newdeptree[key] = set() + newdeptree[key].add(item) + # conflicts + newdeptree[0] = conflicts + + return newdeptree,0 # note: newtree[0] contains possible conflicts + + + def _lookup_system_mask_repository_deps(self): + + data = self.SystemSettings['repos_system_mask'] + if not data: return [] + mydata = [] + cached_items = set() + for atom in data: + mymatch = self.atom_match(atom) + if mymatch[0] == -1: # ignore missing ones intentionally + continue + if mymatch in cached_items: + continue + if mymatch not in mydata: + # check if not found + myaction = self.Client.get_package_action(mymatch) + # only if the package is not installed + if myaction == 1: mydata.append(mymatch) + cached_items.add(mymatch) + return mydata + + def _lookup_conflict_replacement(self, conflict_atom, client_idpackage, deep_deps): + if self.entropyTools.isjustname(conflict_atom): + return None + conflict_match = self.atom_match(conflict_atom) + mykey, myslot = self.Client.clientDbconn.retrieveKeySlot(client_idpackage) + new_match = self.atom_match(mykey, matchSlot = myslot) + if (conflict_match == new_match) or (new_match[1] == 1): + return None + action = self.Client.get_package_action(new_match) + if (action == 0) and (not deep_deps): + return None + return "%s:%s" % (mykey,myslot,) + + def _lookup_inverse_dependencies(self, match, clientmatch): + + cmpstat = self.Client.get_package_action(match) + if cmpstat == 0: return set() + + keyslots = set() + mydepends = self.Client.clientDbconn.retrieveDepends(clientmatch[0]) + am = self.atom_match + cdb_rdeps = self.Client.clientDbconn.retrieveDependencies + cdb_rks = self.Client.clientDbconn.retrieveKeySlot + gpa = self.Client.get_package_action + keyslots_cache = set() + match_cache = {} + + for idpackage in mydepends: + try: + key, slot = cdb_rks(idpackage) + except TypeError: + continue + if (key,slot) in keyslots_cache: continue + keyslots_cache.add((key,slot)) + if (key,slot) in keyslots: continue + # grab its deps + mydeps = cdb_rdeps(idpackage) + found = False + for mydep in mydeps: + mymatch = match_cache.get(mydep, 0) + if mymatch == 0: + mymatch = am(mydep) + match_cache[mydep] = mymatch + if mymatch == match: + found = True + break + if not found: + mymatch = am(key, matchSlot = slot) + if mymatch[0] == -1: continue + cmpstat = gpa(mymatch) + if cmpstat == 0: continue + keyslots.add((key,slot)) + + return keyslots + + def __get_lib_breaks_client_and_repo_side(self, match_db, match_idpackage, client_idpackage): + repo_needed = match_db.retrieveNeeded(match_idpackage, extended = True, format = True) + client_needed = self.Client.clientDbconn.retrieveNeeded(client_idpackage, extended = True, format = True) + repo_split = [x.split(".so")[0] for x in repo_needed] + client_split = [x.split(".so")[0] for x in client_needed] + client_side = [x for x in client_needed if (x not in repo_needed) and (x.split(".so")[0] in repo_split)] + repo_side = [x for x in repo_needed if (x not in client_needed) and (x.split(".so")[0] in client_split)] + return repo_needed, client_side, repo_side + + def _lookup_library_breakages(self, match, clientmatch, deep_deps = False): + + # there is no need to update this cache when "match" will be installed, because at that point + # clientmatch[0] will differ. + c_hash = "%s|%s|%s" % (hash(tuple(match)),hash(deep_deps),hash(tuple(clientmatch)),) + c_hash = "%s%s" % (etpCache['library_breakage'],hash(c_hash),) + if self.Client.xcache: + cached = self.Cacher.pop(c_hash) + if cached != None: return cached + + # these should be pulled in before + repo_atoms = set() + # these can be pulled in after + client_atoms = set() + + matchdb = self.Client.open_repository(match[1]) + reponeeded, client_side, repo_side = self.__get_lib_breaks_client_and_repo_side(matchdb, + match[0], clientmatch[0]) + + # all the packages in client_side should be pulled in and updated + client_idpackages = set() + for needed in client_side: client_idpackages |= self.Client.clientDbconn.searchNeeded(needed) + + client_keyslots = set() + def mymf(idpackage): + if idpackage == clientmatch[0]: return 0 + return self.Client.clientDbconn.retrieveKeySlot(idpackage) + client_keyslots = set([x for x in map(mymf,client_idpackages) if x != 0]) + + # all the packages in repo_side should be pulled in too + repodata = {} + for needed in repo_side: + repodata[needed] = reponeeded[needed] + del repo_side,reponeeded + + repo_dependencies = matchdb.retrieveDependencies(match[0]) + matched_deps = set() + matched_repos = set() + for dependency in repo_dependencies: + depmatch = self.atom_match(dependency) + if depmatch[0] == -1: + continue + matched_repos.add(depmatch[1]) + matched_deps.add(depmatch) + + matched_repos = [x for x in etpRepositoriesOrder if x in matched_repos] + found_matches = set() + for needed in repodata: + for myrepo in matched_repos: + mydbc = self.Client.open_repository(myrepo) + solved_needed = mydbc.resolveNeeded(needed, elfclass = repodata[needed]) + found = False + for idpackage,myfile in solved_needed: + x = (idpackage,myrepo) + if x in matched_deps: + found_matches.add(x) + found = True + break + if found: + break + + for idpackage,repo in found_matches: + if not deep_deps: + cmpstat = self.Client.get_package_action((idpackage,repo)) + if cmpstat == 0: + continue + mydbc = self.Client.open_repository(repo) + repo_atoms.add(mydbc.retrieveAtom(idpackage)) + + for key, slot in client_keyslots: + idpackage, repo = self.atom_match(key, matchSlot = slot) + if idpackage == -1: + continue + if not deep_deps: + cmpstat = self.Client.get_package_action((idpackage, repo)) + if cmpstat == 0: + continue + mydbc = self.Client.open_repository(repo) + client_atoms.add(mydbc.retrieveAtom(idpackage)) + + client_atoms |= repo_atoms + + if self.Client.xcache: + self.Cacher.push(c_hash,client_atoms) + + return client_atoms + + + def get_required_packages(self, matched_atoms, empty_deps = False, deep_deps = False, quiet = False): + + c_hash = "%s%s" % (etpCache['dep_tree'],hash("%s|%s|%s|%s" % ( + hash(frozenset(sorted(matched_atoms))),hash(empty_deps), + hash(deep_deps),self.Client.clientDbconn.database_checksum(), + )),) + if self.Client.xcache: + cached = self.Cacher.pop(c_hash) + if cached != None: return cached + + deptree = {} + deptree[0] = set() + + atomlen = len(matched_atoms); count = 0 + error_generated = 0 + error_tree = set() + + # check if there are repositories needing some mandatory packages + forced_matches = self._lookup_system_mask_repository_deps() + if forced_matches: + if isinstance(matched_atoms, list): + matched_atoms = forced_matches + [x for x in matched_atoms if x not in forced_matches] + elif isinstance(matched_atoms, set): # we cannot do anything about the order here + matched_atoms |= set(forced_matches) + + sort_dep_text = _("Sorting dependencies") + filter_unsat_cache = {} + treecache = set() + keyslotcache = set() + matchfilter = set() + for matched_atom in matched_atoms: + + if not quiet: + count += 1 + if (count%10 == 0) or (count == atomlen) or (count == 1): + self.updateProgress(sort_dep_text, importance = 0, type = "info", + back = True, header = ":: ", footer = " ::", + percent = True, count = (count,atomlen)) + + if matched_atom in matchfilter: continue + newtree, result = self.generate_dependency_tree( + matched_atom, empty_deps, deep_deps, + matchfilter = matchfilter, filter_unsat_cache = filter_unsat_cache, treecache = treecache, + keyslotcache = keyslotcache + ) + + if result == -2: # deps not found + error_generated = -2 + error_tree |= set(newtree) # it is a list, we convert it into set and update error_tree + elif (result != 0): + return newtree, result + elif newtree: + # add conflicts + max_parent_key = max(deptree) + deptree[0] |= newtree.pop(0) + levelcount = 0 + for mylevel in sorted(newtree.keys(), reverse = True): + levelcount += 1 + deptree[max_parent_key+levelcount] = newtree.get(mylevel) + + if error_generated != 0: + return error_tree,error_generated + + if self.Client.xcache: + self.Cacher.push(c_hash,(deptree,0)) + + return deptree,0 + + def _filter_depends_multimatched_atoms(self, idpackage, depends, monotree): + remove_depends = set() + for d_idpackage in depends: + mydeps = self.Client.clientDbconn.retrieveDependencies(d_idpackage) + for mydep in mydeps: + matches, rslt = self.Client.clientDbconn.atomMatch(mydep, multiMatch = True) + if rslt == 1: continue + if idpackage in matches and len(matches) > 1: + # are all in depends? + for mymatch in matches: + if mymatch not in depends and mymatch not in monotree: + remove_depends.add(d_idpackage) + break + depends -= remove_depends + return depends + + + def generate_depends_tree(self, idpackages, deep = False): + + c_hash = "%s%s" % (etpCache['depends_tree'],hash("%s|%s" % (hash(tuple(sorted(idpackages))),hash(deep),),),) + if self.Client.xcache: + cached = self.Cacher.pop(c_hash) + if cached != None: return cached + + dependscache = set() + treeview = set(idpackages) + treelevel = set(idpackages) + tree = {} + treedepth = 0 # I start from level 1 because level 0 is idpackages itself + tree[treedepth] = set(idpackages) + monotree = set(idpackages) # monodimensional tree + + # check if dependstable is sane before beginning + self.Client.clientDbconn.retrieveDepends(idpackages[0]) + count = 0 + + rem_dep_text = _("Calculating removable depends of") + while 1: + treedepth += 1 + tree[treedepth] = set() + for idpackage in treelevel: + + count += 1 + p_atom = self.Client.clientDbconn.retrieveAtom(idpackage) + self.updateProgress( + blue(rem_dep_text + " %s" % (red(p_atom),)), + importance = 0, + type = "info", + back = True, + header = '|/-\\'[count%4]+" " + ) + + systempkg = not self.Client.validate_package_removal(idpackage) + if (idpackage in dependscache) or systempkg: + if idpackage in treeview: + treeview.remove(idpackage) + continue + + # obtain its depends + depends = self.Client.clientDbconn.retrieveDepends(idpackage) + # filter already satisfied ones + depends = set([x for x in depends if x not in monotree]) + depends = set([x for x in depends if self.Client.validate_package_removal(x)]) + if depends: + depends = self._filter_depends_multimatched_atoms(idpackage, depends, monotree) + if depends: # something depends on idpackage + tree[treedepth] |= depends + monotree |= depends + treeview |= depends + elif deep: # if deep, grab its dependencies and check + + mydeps = set() + for x in self.Client.clientDbconn.retrieveDependencies(idpackage): + match = self.Client.clientDbconn.atomMatch(x) + if match[0] != -1: + mydeps.add(match[0]) + + # now filter them + mydeps = [x for x in mydeps if x not in monotree and not \ + (self.Client.clientDbconn.isSystemPackage(x) or \ + self.Client.is_installed_idpackage_in_system_mask(x) )] + for x in mydeps: + mydepends = self.Client.clientDbconn.retrieveDepends(x) + mydepends -= set([y for y in mydepends if y not in monotree]) + if not mydepends: + tree[treedepth].add(x) + monotree.add(x) + treeview.add(x) + + dependscache.add(idpackage) + if idpackage in treeview: + treeview.remove(idpackage) + + treelevel = treeview.copy() + if not treelevel: + if not tree[treedepth]: + del tree[treedepth] # probably the last one is empty then + break + + # now filter newtree + for count in sorted(tree.keys(), reverse = True): + x = 0 + while x < count: + tree[x] -= tree[count] + x += 1 + + if self.Client.xcache: + self.Cacher.push(c_hash,(tree,0)) + return tree,0 # treeview is used to show deps while tree is used to run the dependency code. + + def calculate_available_packages(self, use_cache = True): + + c_hash = self.Client.get_available_packages_chash(etpConst['branch']) + + if use_cache and self.Client.xcache: + cached = self.Client.get_available_packages_cache(myhash = c_hash) + if cached != None: + return cached + + available = [] + self.Client.setTotalCycles(len(self.Client.validRepositories)) + avail_dep_text = _("Calculating available packages for") + for repo in self.Client.validRepositories: + try: + dbconn = self.Client.open_repository(repo) + dbconn.validateDatabase() + except (RepositoryError,SystemDatabaseError): + self.Client.cycleDone() + continue + idpackages = [ x for x in dbconn.listAllIdpackages(branch = etpConst['branch'], branch_operator = "<=", order_by = 'atom') \ + if dbconn.idpackageValidator(x)[0] != -1 ] + count = 0 + maxlen = len(idpackages) + myavailable = [] + do_break = False + for idpackage in idpackages: + if do_break: + break + count += 1 + if (count % 10 == 0) or (count == 1) or (count == maxlen): + self.updateProgress( + avail_dep_text + " %s" % (repo,), + importance = 0, + type = "info", + back = True, + header = "::", + count = (count,maxlen), + percent = True, + footer = " ::" + ) + # get key + slot + try: + key, slot = dbconn.retrieveKeySlot(idpackage) + matches = self.Client.clientDbconn.searchKeySlot(key, slot) + except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,): + self.Client.cycleDone() + do_break = True + continue + if not matches: myavailable.append((idpackage,repo)) + available += myavailable[:] + self.Client.cycleDone() + + if self.Client.xcache: + self.Cacher.push("%s%s" % (etpCache['world_available'],c_hash),available) + return available + + def calculate_world_updates( + self, + empty_deps = False, + branch = etpConst['branch'], + ignore_spm_downgrades = etpConst['spm']['ignore-spm-downgrades'], + use_cache = True + ): + + db_digest = self.Client.all_repositories_checksum() + if use_cache and self.Client.xcache: + cached = self.Client.get_world_update_cache(empty_deps = empty_deps, + branch = branch, db_digest = db_digest, ignore_spm_downgrades = ignore_spm_downgrades) + if cached != None: return cached + + update = [] + remove = [] + fine = [] + + # get all the installed packages + idpackages = self.Client.clientDbconn.listAllIdpackages(order_by = 'atom') + maxlen = len(idpackages) + count = 0 + mytxt = _("Calculating world packages") + for idpackage in idpackages: + + count += 1 + if (count%10 == 0) or (count == maxlen) or (count == 1): + self.updateProgress( + mytxt, + importance = 0, + type = "info", + back = True, + header = ":: ", + count = (count,maxlen), + percent = True, + footer = " ::" + ) + + mystrictdata = self.Client.clientDbconn.getStrictData(idpackage) + # check against broken entries, or removed during iteration + if mystrictdata == None: + continue + use_match_cache = True + do_continue = False + while 1: + try: + match = self.atom_match( + mystrictdata[0], + matchSlot = mystrictdata[1], + matchBranches = (branch,), + extendedResults = True, + useCache = use_match_cache + ) + except self.dbapi2.OperationalError: + # ouch, but don't crash here + do_continue = True + break + try: + m_idpackage = match[0][0] + except TypeError: + if not use_match_cache: raise + use_match_cache = False + continue + break + if do_continue: continue + # now compare + # version: mystrictdata[2] + # tag: mystrictdata[3] + # revision: mystrictdata[4] + if (m_idpackage != -1): + repoid = match[1] + version = match[0][1] + tag = match[0][2] + revision = match[0][3] + if empty_deps: + if (m_idpackage,repoid) not in update: + update.append((m_idpackage,repoid)) + continue + elif (mystrictdata[2] != version): + # different versions + if (m_idpackage,repoid) not in update: + update.append((m_idpackage,repoid)) + continue + elif (mystrictdata[3] != tag): + # different tags + if (m_idpackage,repoid) not in update: + update.append((m_idpackage,repoid)) + continue + elif (mystrictdata[4] != revision): + # different revision + if mystrictdata[4] == 9999 and ignore_spm_downgrades: + # no difference, we're ignoring revision 9999 + fine.append(mystrictdata[5]) + continue + else: + if (m_idpackage,repoid) not in update: + update.append((m_idpackage,repoid)) + continue + else: + # no difference + fine.append(mystrictdata[5]) + continue + + # don't take action if it's just masked + maskedresults = self.atom_match(mystrictdata[0], matchSlot = mystrictdata[1], matchBranches = (branch,), packagesFilter = False) + if maskedresults[0] == -1: + remove.append(idpackage) + # look for packages that would match key with any slot (for eg: gcc, kernel updates) + matchresults = self.atom_match(mystrictdata[0], matchBranches = (branch,)) + if matchresults[0] != -1: + m_action = self.Client.get_package_action(matchresults) + if m_action > 0 and (matchresults not in update): + update.append(matchresults) + + if self.Client.xcache: + c_hash = self.Client.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades) + data = { + 'r': (update, remove, fine,), + 'empty_deps': empty_deps, + } + self.Cacher.push("%s%s" % (etpCache['world_update'],c_hash,), data, async = False) + + return update, remove, fine + + def check_package_update(self, atom, deep = False): + + c_hash = "%s%s" % (etpCache['check_package_update'],hash("%s%s" % (atom,hash(deep),)),) + if self.Client.xcache: + cached = self.Cacher.pop(c_hash) + if cached != None: + return cached + + found = False + match = self.Client.clientDbconn.atomMatch(atom) + matched = None + if match[0] != -1: + myatom = self.Client.clientDbconn.retrieveAtom(match[0]) + mytag = self.entropyTools.dep_gettag(myatom) + myatom = self.entropyTools.remove_tag(myatom) + myrev = self.Client.clientDbconn.retrieveRevision(match[0]) + pkg_match = "="+myatom+"~"+str(myrev) + if mytag != None: + pkg_match += "#%s" % (mytag,) + pkg_unsatisfied = self.get_unsatisfied_dependencies([pkg_match], deep_deps = deep) + if pkg_unsatisfied: + found = True + del pkg_unsatisfied + matched = self.atom_match(pkg_match) + del match + + if self.Client.xcache: + self.Cacher.push(c_hash,(found,matched)) + + return found, matched + + # This is the function that should be used by third party applications + # to retrieve a list of available updates, along with conflicts (removalQueue) and obsoletes + # (removed) + def get_world_queue(self, empty_deps = False, branch = etpConst['branch']): + update, remove, fine = self.calculate_world_updates(empty_deps = empty_deps, branch = branch) + del fine + data = {} + data['removed'] = list(remove) + data['runQueue'] = [] + data['removalQueue'] = [] + status = -1 + if update: + # calculate install+removal queues + install, removal, status = self.get_install_queue(update, empty_deps, deep_deps = False) + # update data['removed'] + data['removed'] = [x for x in data['removed'] if x not in removal] + data['runQueue'] += install + data['removalQueue'] += removal + return data,status + + def validate_package_removal(self, idpackage): + + pkgatom = self.Client.clientDbconn.retrieveAtom(idpackage) + pkgkey = self.entropyTools.dep_getkey(pkgatom) + + if self.Client.is_installed_idpackage_in_system_mask(idpackage): + idpackages = self.SystemSettings['repos_system_mask_installed_keys'].get(pkgkey) + if not idpackages: return False + if len(idpackages) > 1: + return True + return False # sorry! + + # did we store the bastard in the db? + system_pkg = self.Client.clientDbconn.isSystemPackage(idpackage) + if not system_pkg: return True + # check if the package is slotted and exist more than one installed first + sysresults = self.Client.clientDbconn.atomMatch(pkgkey, multiMatch = True) + if sysresults[1] == 0: + if len(sysresults[0]) < 2: return False + return True + return False + + + def get_removal_queue(self, idpackages, deep = False): + queue = [] + if not idpackages: + return queue + treeview, status = self.generate_depends_tree(idpackages, deep = deep) + if status == 0: + for x in range(len(treeview))[::-1]: queue.extend(treeview[x]) + return queue + + def get_install_queue(self, matched_atoms, empty_deps, deep_deps, quiet = False): + + install = [] + removal = [] + treepackages, result = self.get_required_packages(matched_atoms, empty_deps, deep_deps, quiet = quiet) + + if result == -2: + return treepackages,removal,result + + # format + removal = treepackages.pop(0, set()) + for x in sorted(treepackages.keys()): install.extend(treepackages[x]) + + # filter out packages that are in actionQueue comparing key + slot + if install and removal: + myremmatch = {} + for x in removal: + atom = self.Client.clientDbconn.retrieveAtom(x) + # XXX check if users removed idpackage while this whole instance is running + if atom == None: continue + myremmatch[(self.entropyTools.dep_getkey(atom),self.Client.clientDbconn.retrieveSlot(x),)] = x + for pkg_id, pkg_repo in install: + dbconn = self.Client.open_repository(pkg_repo) + testtuple = (self.entropyTools.dep_getkey(dbconn.retrieveAtom(pkg_id)),dbconn.retrieveSlot(pkg_id)) + removal.discard(myremmatch.get(testtuple)) + + return install, sorted(removal), 0 + + """ + XXX deprecated XXX + """ + + def packageSetMatch(self, *args, **kwargs): + import warning + warning.warn("deprecated, use package_set_match instead") + return self.package_set_match(*args, **kwargs) + + def packageSetSearch(self, *args, **kwargs): + import warning + warning.warn("deprecated, use package_set_search instead") + return self.package_set_search(*args, **kwargs) + + def packageSetList(self, *args, **kwargs): + import warning + warning.warn("deprecated, use package_set_list instead") + return self.package_set_list(*args, **kwargs) + + def packageSetExpand(self, *args, **kwargs): + import warning + warning.warn("deprecated, use package_set_expand instead") + return self.package_set_expand(*args, **kwargs) + + def packagesExpand(self, *args, **kwargs): + import warning + warning.warn("deprecated, use packages_expand instead") + return self.packages_expand(*args, **kwargs) + + def atomMatch(self, *args, **kwargs): + import warning + warning.warn("deprecated, use atom_match instead") + return self.atom_match(*args, **kwargs) + + def retrieveWorldQueue(self, *args, **kwargs): + import warning + warning.warn("deprecated, use get_world_queue instead") + return self.get_world_queue(*args, **kwargs) + + def validatePackageRemoval(self, *args, **kwargs): + import warning + warning.warn("deprecated, use validate_package_removal instead") + return self.validate_package_removal(*args, **kwargs) + + def retrieveInstallQueue(self, *args, **kwargs): + import warning + warning.warn("deprecated, use get_install_queue instead") + return self.get_install_queue(*args, **kwargs) + + def retrieveRemovalQueue(self, *args, **kwargs): + import warning + warning.warn("deprecated, use get_removal_queue instead") + return self.get_removal_queue(*args, **kwargs) diff --git a/libraries/entropy/client/interfaces/fetch.py b/libraries/entropy/client/interfaces/fetch.py new file mode 100644 index 000000000..ba695c0e6 --- /dev/null +++ b/libraries/entropy/client/interfaces/fetch.py @@ -0,0 +1,493 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +import os +from entropy.i18n import _ +from entropy.const import * +from entropy.exceptions import * +from entropy.output import purple, bold, red, blue, darkgreen, darkred, brown, darkblue + +class Fetchers: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.entropyTools = self.Client.entropyTools + self.updateProgress = self.Client.updateProgress + self.SystemSettings = self.Client.SystemSettings + self.Cacher = self.Client.Cacher + self.MirrorStatus = self.Client.MirrorStatus + + def check_needed_package_download(self, filepath, checksum = None): + # is the file available + if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath): + if checksum is None: + return 0 + else: + # check digest + md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum) + if (md5res): + return 0 + else: + return -2 + else: + return -1 + + def fetch_files(self, url_data_list, checksum = True, resume = True, fetch_file_abort_function = None): + """ + Fetch multiple files simultaneously on URLs. + + @param url_data_list list + [(url,dest_path [or None],checksum ['ab86fff46f6ec0f4b1e0a2a4a82bf323' or None],branch,),..] + @param digest bool, digest check (checksum) + @param resume bool enable resume support + @param fetch_file_abort_function callable method that could raise exceptions + @return general_status_code, {'url': (status_code,checksum,resumed,)}, data_transfer + """ + pkgs_bindir = etpConst['packagesbindir'] + url_path_list = [] + checksum_map = {} + count = 0 + for url, dest_path, cksum, branch in url_data_list: + count += 1 + filename = os.path.basename(url) + if dest_path == None: + dest_path = os.path.join(pkgs_bindir,branch,filename,) + + dest_dir = os.path.dirname(dest_path) + if not os.path.isdir(dest_dir): + os.makedirs(dest_dir,0755) + + url_path_list.append((url,dest_path,)) + if cksum != None: checksum_map[count] = cksum + + # load class + fetchConn = self.Client.MultipleUrlFetcher(url_path_list, resume = resume, + abort_check_func = fetch_file_abort_function, OutputInterface = self, + urlFetcherClass = self.Client.urlFetcher, checksum = checksum) + try: + data = fetchConn.download() + except KeyboardInterrupt: + return -100, {}, 0 + + diff_map = {} + if checksum_map and checksum: # verify checksums + diff_map = dict((url_path_list[x-1][0],checksum_map.get(x)) for x in checksum_map \ + if checksum_map.get(x) != data.get(x)) + + data_transfer = fetchConn.get_data_transfer() + if diff_map: + defval = -1 + for key, val in diff_map.items(): + if val == "-1": # general error + diff_map[key] = -1 + elif val == "-2": + diff_map[key] = -2 + elif val == "-4": # timeout + diff_map[key] = -4 + elif val == "-3": # not found + diff_map[key] = -3 + elif val == -100: + defval = -100 + return defval, diff_map, data_transfer + + return 0, diff_map, data_transfer + + def fetch_files_on_mirrors(self, download_list, checksum = False, fetch_abort_function = None): + """ + @param download_map list [(repository,branch,filename,checksum (digest),),..] + @param checksum bool verify checksum? + @param fetch_abort_function callable method that could raise exceptions + """ + repo_uris = dict(((x[0],etpRepositories[x[0]]['packages'][::-1],) for x in download_list)) + remaining = repo_uris.copy() + my_download_list = download_list[:] + + def get_best_mirror(repository): + try: + return remaining[repository][0] + except IndexError: + return None + + def update_download_list(down_list, failed_down): + newlist = [] + for repo,branch,fname,cksum in down_list: + myuri = get_best_mirror(repo) + myuri = os.path.join(myuri,fname) + if myuri not in failed_down: + continue + newlist.append((repo,branch,fname,cksum,)) + return newlist + + # return True: for failing, return False: for fine + def mirror_fail_check(repository, best_mirror): + # check if uri is sane + if not self.MirrorStatus.get_failing_mirror_status(best_mirror) >= 30: + return False + # set to 30 for convenience + self.MirrorStatus.set_failing_mirror_status(best_mirror, 30) + mirrorcount = repo_uris[repo].index(best_mirror)+1 + mytxt = "( mirror #%s ) " % (mirrorcount,) + mytxt += blue(" %s: ") % (_("Mirror"),) + mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) + mytxt += " - %s." % (_("maximum failure threshold reached"),) + self.updateProgress( + mytxt, + importance = 1, + type = "warning", + header = red(" ## ") + ) + + if self.MirrorStatus.get_failing_mirror_status(best_mirror) == 30: + self.MirrorStatus.add_failing_mirror(best_mirror,45) + elif self.MirrorStatus.get_failing_mirror_status(best_mirror) > 31: + self.MirrorStatus.add_failing_mirror(best_mirror,-4) + else: + self.MirrorStatus.set_failing_mirror_status(best_mirror, 0) + + remaining[repository].discard(best_mirror) + return True + + def show_download_summary(down_list): + # fetch_files_list.append((myuri,None,cksum,branch,)) + for repo, branch, fname, cksum in down_list: + best_mirror = get_best_mirror(repo) + mirrorcount = repo_uris[repo].index(best_mirror)+1 + mytxt = "( mirror #%s ) " % (mirrorcount,) + basef = os.path.basename(fname) + mytxt += "[%s] %s " % (brown(basef),blue("@"),) + mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) + # now fetch the new one + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = red(" ## ") + ) + + def show_successful_download(down_list, data_transfer): + for repo, branch, fname, cksum in down_list: + best_mirror = get_best_mirror(repo) + mirrorcount = repo_uris[repo].index(best_mirror)+1 + mytxt = "( mirror #%s ) " % (mirrorcount,) + basef = os.path.basename(fname) + mytxt += "[%s] %s %s " % (brown(basef),darkred(_("success")),blue("@"),) + mytxt += red(self.entropyTools.spliturl(best_mirror)[1]) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = red(" ## ") + ) + mytxt = " %s: %s%s%s" % ( + blue(_("Aggregated transfer rate")), + bold(self.entropyTools.bytesIntoHuman(data_transfer)), + darkred("/"), + darkblue(_("second")), + ) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = red(" ## ") + ) + + def show_download_error(down_list, rc): + for repo, branch, fname, cksum in down_list: + best_mirror = get_best_mirror(repo) + mirrorcount = repo_uris[repo].index(best_mirror)+1 + mytxt = "( mirror #%s ) " % (mirrorcount,) + mytxt += blue("%s: %s") % ( + _("Error downloading from"), + red(self.entropyTools.spliturl(best_mirror)[1]), + ) + if rc == -1: + mytxt += " - %s." % (_("files not available on this mirror"),) + elif rc == -2: + self.MirrorStatus.add_failing_mirror(best_mirror,1) + mytxt += " - %s." % (_("wrong checksum"),) + elif rc == -3: + mytxt += " - %s." % (_("not found"),) + elif rc == -4: # timeout! + mytxt += " - %s." % (_("timeout error"),) + elif rc == -100: + mytxt += " - %s." % (_("discarded download"),) + else: + self.MirrorStatus.add_failing_mirror(best_mirror, 5) + mytxt += " - %s." % (_("unknown reason"),) + self.updateProgress( + mytxt, + importance = 1, + type = "warning", + header = red(" ## ") + ) + + def remove_failing_mirrors(repos): + for repo in repos: + best_mirror = get_best_mirror(repo) + if remaining[repo]: + remaining[repo].pop(0) + + def check_remaining_mirror_failure(repos): + return [x for x in repos if not remaining.get(x)] + + while 1: + + do_resume = True + timeout_try_count = 50 + while 1: + + fetch_files_list = [] + for repo, branch, fname, cksum in my_download_list: + best_mirror = get_best_mirror(repo) + if best_mirror != None: + mirror_fail_check(repo, best_mirror) + best_mirror = get_best_mirror(repo) + if best_mirror == None: + # at least one package failed to download + # properly, give up with everything + return 3, my_download_list + myuri = os.path.join(best_mirror,fname) + fetch_files_list.append((myuri,None,cksum,branch,)) + + try: + + show_download_summary(my_download_list) + rc, failed_downloads, data_transfer = self.fetch_files( + fetch_files_list, checksum = checksum, + fetch_file_abort_function = fetch_abort_function, + resume = do_resume + ) + if rc == 0: + show_successful_download(my_download_list, data_transfer) + return 0, [] + + # update my_download_list + my_download_list = update_download_list(my_download_list,failed_downloads) + if rc not in (-3,-4,-100,) and failed_downloads and do_resume: + # disable resume + do_resume = False + continue + else: + show_download_error(my_download_list, rc) + if rc == -4: # timeout + timeout_try_count -= 1 + if timeout_try_count > 0: + continue + elif rc == -100: # user discarded fetch + return 1, [] + myrepos = set([x[0] for x in my_download_list]) + remove_failing_mirrors(myrepos) + # make sure we don't have nasty issues + remaining_failure = check_remaining_mirror_failure(myrepos) + if remaining_failure: + return 3, my_download_list + break + except KeyboardInterrupt: + return 1, [] + return 0, [] + + + def fetch_file(self, url, branch, digest = None, resume = True, fetch_file_abort_function = None, filepath = None): + + filename = os.path.basename(url) + if not filepath: + filepath = os.path.join(etpConst['packagesbindir'],branch,filename) + filepath_dir = os.path.dirname(filepath) + if not os.path.isdir(filepath_dir): + os.makedirs(filepath_dir,0755) + + # load class + fetchConn = self.Client.urlFetcher(url, filepath, resume = resume, + abort_check_func = fetch_file_abort_function, OutputInterface = self) + fetchConn.progress = self.Client.progress + + # start to download + data_transfer = 0 + resumed = False + try: + fetchChecksum = fetchConn.download() + data_transfer = fetchConn.get_transfer_rate() + resumed = fetchConn.is_resumed() + except KeyboardInterrupt: + return -100, data_transfer, resumed + except NameError: + raise + except: + if etpUi['debug']: + self.updateProgress( + "fetch_file:", + importance = 1, + type = "warning", + header = red(" ## ") + ) + self.entropyTools.printTraceback() + return -1, data_transfer, resumed + if fetchChecksum == "-3": + # not found + return -3, data_transfer, resumed + elif fetchChecksum == "-4": + # timeout + return -4, data_transfer, resumed + + del fetchConn + if digest: + if fetchChecksum != digest: + # not properly downloaded + return -2, data_transfer, resumed + else: + return 0, data_transfer, resumed + return 0, data_transfer, resumed + + + def fetch_file_on_mirrors(self, repository, branch, filename, + digest = False, fetch_abort_function = None): + + uris = etpRepositories[repository]['packages'][::-1] + remaining = set(uris) + + mirrorcount = 0 + for uri in uris: + + if not remaining: + # tried all the mirrors, quitting for error + return 3 + + mirrorcount += 1 + mirrorCountText = "( mirror #%s ) " % (mirrorcount,) + url = uri+"/"+filename + + # check if uri is sane + if self.MirrorStatus.get_failing_mirror_status(uri) >= 30: + # ohohoh! + # set to 30 for convenience + self.MirrorStatus.set_failing_mirror_status(uri, 30) + mytxt = mirrorCountText + mytxt += blue(" %s: ") % (_("Mirror"),) + mytxt += red(self.entropyTools.spliturl(uri)[1]) + mytxt += " - %s." % (_("maximum failure threshold reached"),) + self.updateProgress( + mytxt, + importance = 1, + type = "warning", + header = red(" ## ") + ) + + if self.MirrorStatus.get_failing_mirror_status(uri) == 30: + # put to 75 then decrement by 4 so we + # won't reach 30 anytime soon ahahaha + self.MirrorStatus.add_failing_mirror(uri,45) + elif self.MirrorStatus.get_failing_mirror_status(uri) > 31: + # now decrement each time this point is reached, + # if will be back < 30, then equo will try to use it again + self.MirrorStatus.add_failing_mirror(uri,-4) + else: + # put to 0 - reenable mirror, welcome back uri! + self.MirrorStatus.set_failing_mirror_status(uri, 0) + + remaining.discard(uri) + continue + + do_resume = True + timeout_try_count = 50 + while 1: + try: + mytxt = mirrorCountText + mytxt += blue("%s: ") % (_("Downloading from"),) + mytxt += red(self.entropyTools.spliturl(uri)[1]) + # now fetch the new one + self.updateProgress( + mytxt, + importance = 1, + type = "warning", + header = red(" ## ") + ) + rc, data_transfer, resumed = self.fetch_file( + url, + branch, + digest, + do_resume, + fetch_file_abort_function = fetch_abort_function + ) + if rc == 0: + mytxt = mirrorCountText + mytxt += blue("%s: ") % (_("Successfully downloaded from"),) + mytxt += red(self.entropyTools.spliturl(uri)[1]) + mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = red(" ## ") + ) + + return 0 + elif resumed and rc not in (-3,-4,-100,): + do_resume = False + continue + else: + error_message = mirrorCountText + error_message += blue("%s: %s") % ( + _("Error downloading from"), + red(self.entropyTools.spliturl(uri)[1]), + ) + # something bad happened + if rc == -1: + error_message += " - %s." % (_("file not available on this mirror"),) + elif rc == -2: + self.MirrorStatus.add_failing_mirror(uri,1) + error_message += " - %s." % (_("wrong checksum"),) + elif rc == -3: + error_message += " - %s." % (_("not found"),) + elif rc == -4: # timeout! + timeout_try_count -= 1 + if timeout_try_count > 0: + error_message += " - %s." % (_("timeout, retrying on this mirror"),) + else: + error_message += " - %s." % (_("timeout, giving up"),) + elif rc == -100: + error_message += " - %s." % (_("discarded download"),) + else: + self.MirrorStatus.add_failing_mirror(uri, 5) + error_message += " - %s." % (_("unknown reason"),) + self.updateProgress( + error_message, + importance = 1, + type = "warning", + header = red(" ## ") + ) + if rc == -4: # timeout + if timeout_try_count > 0: + continue + elif rc == -100: # user discarded fetch + return 1 + remaining.discard(uri) + # make sure we don't have nasty issues + if not remaining: + return 3 + break + except KeyboardInterrupt: + return 1 + return 0 diff --git a/libraries/entropy/client/interfaces/loaders.py b/libraries/entropy/client/interfaces/loaders.py new file mode 100644 index 000000000..16c25616d --- /dev/null +++ b/libraries/entropy/client/interfaces/loaders.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +from entropy.const import * +from entropy.exceptions import * + +class Loaders: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + from entropy.client.interfaces import Trigger + from entropy.client.interfaces import Repository + from entropy.client.interfaces import Package + self.__PackageLoader = Package + self.__RepositoryLoader = Repository + self.__TriggerLoader = Trigger + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.__QA_cache = {} + self.__security_cache = {} + self.__spm_cache = {} + + def closeAllQA(self): + self.__QA_cache.clear() + + def closeAllSecurity(self): + self.__security_cache.clear() + + def Security(self): + chroot = etpConst['systemroot'] + cached = self.Client.__security_cache.get(chroot) + if cached != None: + return cached + from entropy.security import SecurityInterface + cached = SecurityInterface(self) + self.Client.__security_cache[chroot] = cached + return cached + + def QA(self): + chroot = etpConst['systemroot'] + cached = self.__QA_cache.get(chroot) + if cached != None: + return cached + from entropy.qa import QAInterface + cached = QAInterface(self) + self.__QA_cache[chroot] = cached + return cached + + def Triggers(self, *args, **kwargs): + return self.__TriggerLoader(self, *args, **kwargs) + + def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True): + return self.__RepositoryLoader(self, reponames = reponames, + forceUpdate = forceUpdate, noEquoCheck = noEquoCheck, + fetchSecurity = fetchSecurity) + + def Spm(self): + from entropy.spm import Spm + myroot = etpConst['systemroot'] + cached = self.__spm_cache.get(myroot) + if cached != None: return cached + conn = Spm(self) + self.__spm_cache[myroot] = conn.intf + return conn.intf + + def Package(self): + return self.__PackageLoader(self) \ No newline at end of file diff --git a/libraries/entropy/client/interfaces/metadata.py b/libraries/entropy/client/interfaces/metadata.py new file mode 100644 index 000000000..ef8caa551 --- /dev/null +++ b/libraries/entropy/client/interfaces/metadata.py @@ -0,0 +1,490 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +import os +import shutil +from entropy.i18n import _ +from entropy.const import * +from entropy.exceptions import * +from entropy.output import red, bold, brown + +class Extractors: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.entropyTools = self.Client.entropyTools + self.updateProgress = self.Client.updateProgress + self.SystemSettings = self.Client.SystemSettings + + def _extract_pkg_metadata_generate_extraction_dict(self): + data = { + 'chost': { + 'path': etpConst['spm']['xpak_entries']['chost'], + 'critical': True, + }, + 'description': { + 'path': etpConst['spm']['xpak_entries']['description'], + 'critical': False, + }, + 'homepage': { + 'path': etpConst['spm']['xpak_entries']['homepage'], + 'critical': False, + }, + 'slot': { + 'path': etpConst['spm']['xpak_entries']['slot'], + 'critical': False, + }, + 'cflags': { + 'path': etpConst['spm']['xpak_entries']['cflags'], + 'critical': False, + }, + 'cxxflags': { + 'path': etpConst['spm']['xpak_entries']['cxxflags'], + 'critical': False, + }, + 'category': { + 'path': etpConst['spm']['xpak_entries']['category'], + 'critical': True, + }, + 'rdepend': { + 'path': etpConst['spm']['xpak_entries']['rdepend'], + 'critical': False, + }, + 'pdepend': { + 'path': etpConst['spm']['xpak_entries']['pdepend'], + 'critical': False, + }, + 'depend': { + 'path': etpConst['spm']['xpak_entries']['depend'], + 'critical': False, + }, + 'use': { + 'path': etpConst['spm']['xpak_entries']['use'], + 'critical': False, + }, + 'iuse': { + 'path': etpConst['spm']['xpak_entries']['iuse'], + 'critical': False, + }, + 'license': { + 'path': etpConst['spm']['xpak_entries']['license'], + 'critical': False, + }, + 'provide': { + 'path': etpConst['spm']['xpak_entries']['provide'], + 'critical': False, + }, + 'sources': { + 'path': etpConst['spm']['xpak_entries']['src_uri'], + 'critical': False, + }, + 'eclasses': { + 'path': etpConst['spm']['xpak_entries']['inherited'], + 'critical': False, + }, + 'counter': { + 'path': etpConst['spm']['xpak_entries']['counter'], + 'critical': False, + }, + 'keywords': { + 'path': etpConst['spm']['xpak_entries']['keywords'], + 'critical': False, + }, + } + return data + + def _extract_pkg_metadata_content(self, content_file, package_path): + + pkg_content = {} + + if os.path.isfile(content_file): + f = open(content_file,"r") + content = f.readlines() + f.close() + outcontent = set() + for line in content: + line = line.strip().split() + try: + datatype = line[0] + datafile = line[1:] + if datatype == 'obj': + datafile = datafile[:-2] + datafile = ' '.join(datafile) + elif datatype == 'dir': + datafile = ' '.join(datafile) + elif datatype == 'sym': + datafile = datafile[:-3] + datafile = ' '.join(datafile) + else: + myexc = "InvalidData: %s %s. %s." % ( + datafile, + _("not supported"), + _("Probably Portage API has changed"), + ) + raise InvalidData(myexc) + outcontent.add((datafile,datatype)) + except: + pass + + _outcontent = set() + for i in outcontent: + i = list(i) + datatype = i[1] + _outcontent.add((i[0],i[1])) + outcontent = sorted(_outcontent) + for i in outcontent: + pkg_content[i[0]] = i[1] + + else: + + # CONTENTS is not generated when a package is emerged with portage and the option -B + # we have to unpack the tbz2 and generate content dict + mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject" + if os.path.isdir(mytempdir): + shutil.rmtree(mytempdir) + if not os.path.isdir(mytempdir): + os.makedirs(mytempdir) + + self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True) + for currentdir, subdirs, files in os.walk(mytempdir): + pkg_content[currentdir[len(mytempdir):]] = "dir" + for item in files: + item = currentdir+"/"+item + if os.path.islink(item): + pkg_content[item[len(mytempdir):]] = "sym" + else: + pkg_content[item[len(mytempdir):]] = "obj" + + # now remove + shutil.rmtree(mytempdir,True) + try: os.rmdir(mytempdir) + except (OSError,): pass + + return pkg_content + + def _extract_pkg_metadata_needed(self, needed_file): + + pkg_needed = set() + lines = [] + + try: + f = open(needed_file,"r") + lines = [x.strip() for x in f.readlines() if x.strip()] + f.close() + except IOError: + return lines + + for line in lines: + needed = line.split() + if len(needed) == 2: + ownlib = needed[0] + ownelf = -1 + if os.access(ownlib,os.R_OK): + ownelf = self.entropyTools.read_elf_class(ownlib) + for lib in needed[1].split(","): + #if lib.find(".so") != -1: + pkg_needed.add((lib,ownelf)) + + return list(pkg_needed) + + def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False): + + pkg_messages = [] + + if os.path.isdir(log_dir): + + elogfiles = os.listdir(log_dir) + myelogfile = "%s:%s-%s" % (category, name, version,) + foundfiles = [x for x in elogfiles if x.startswith(myelogfile)] + if foundfiles: + elogfile = foundfiles[0] + if len(foundfiles) > 1: + # get the latest + mtimes = [] + for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item)) + mtimes = sorted(mtimes) + elogfile = mtimes[-1][1] + messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile)) + for message in messages: + message = message.replace("emerge","install") + pkg_messages.append(message) + + elif not silent: + + mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),) + self.updateProgress( + red(log_dir)+mytxt, + importance = 1, + type = "warning", + header = brown(" * ") + ) + + return pkg_messages + + def _extract_pkg_metadata_license_data(self, licenses_dir, license_string): + + pkg_licensedata = {} + if licenses_dir and os.path.isdir(licenses_dir): + licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())] + for mylicense in licdata: + licfile = os.path.join(licenses_dir,mylicense) + if os.access(licfile,os.R_OK): + if self.entropyTools.istextfile(licfile): + f = open(licfile) + pkg_licensedata[mylicense] = f.read() + f.close() + + return pkg_licensedata + + def _extract_pkg_metadata_mirror_links(self, Spm, sources_list): + + # =mirror://openoffice|link1|link2|link3 + pkg_links = [] + for i in sources_list: + if i.startswith("mirror://"): + # parse what mirror I need + mirrorURI = i.split("/")[2] + mirrorlist = Spm.get_third_party_mirrors(mirrorURI) + pkg_links.append([mirrorURI,mirrorlist]) + # mirrorURI = openoffice and mirrorlist = [link1, link2, link3] + + return pkg_links + + def _extract_pkg_metadata_ebuild_entropy_tag(self, ebuild): + search_tag = etpConst['spm']['ebuild_pkg_tag_var'] + ebuild_tag = '' + f = open(ebuild,"r") + tags = [x.strip() for x in f.readlines() if x.strip() and x.strip().startswith(search_tag)] + f.close() + if not tags: return ebuild_tag + tag = tags[-1] + tag = tag.split("=")[-1].strip('"').strip("'").strip() + return tag + + # This function extracts all the info from a .tbz2 file and returns them + def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False): + + data = {} + info_package = bold(os.path.basename(package))+": " + + if not silent: + self.updateProgress( + red(info_package+_("Extracting package metadata")+" ..."), + importance = 0, + type = "info", + header = brown(" * "), + back = True + ) + + filepath = package + tbz2File = package + package = package.split(etpConst['packagesext'])[0] + package = self.entropyTools.remove_entropy_revision(package) + package = self.entropyTools.remove_tag(package) + # remove entropy category + if package.find(":") != -1: + package = ':'.join(package.split(":")[1:]) + + # pkgcat is always == "null" here + pkgcat, pkgname, pkgver, pkgrev = self.entropyTools.catpkgsplit(os.path.basename(package)) + if pkgrev != "r0": pkgver += "-%s" % (pkgrev,) + + # Fill Package name and version + data['name'] = pkgname + data['version'] = pkgver + data['digest'] = self.entropyTools.md5sum(tbz2File) + data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File)) + data['size'] = str(self.entropyTools.get_file_size(tbz2File)) + + tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/" + if not os.path.isdir(tbz2TmpDir): + if os.path.lexists(tbz2TmpDir): + os.remove(tbz2TmpDir) + os.makedirs(tbz2TmpDir) + self.entropyTools.extractXpak(tbz2File,tbz2TmpDir) + + data['injected'] = False + if inject: data['injected'] = True + data['branch'] = etpBranch + + portage_entries = self._extract_pkg_metadata_generate_extraction_dict() + for item in portage_entries: + value = '' + try: + f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r") + value = f.readline().strip() + f.close() + except IOError: + if portage_entries[item]['critical']: + raise + data[item] = value + + # setup vars + data['eclasses'] = data['eclasses'].split() + try: + data['counter'] = int(data['counter']) + except ValueError: + data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database + data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()] + if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords + needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed']) + data['needed'] = self._extract_pkg_metadata_needed(needed_file) + content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents']) + data['content'] = self._extract_pkg_metadata_content(content_file, filepath) + data['disksize'] = self.entropyTools.sum_file_sizes(data['content']) + + # [][][] Kernel dependent packages hook [][][] + data['versiontag'] = '' + kernelstuff = False + kernelstuff_kernel = False + for item in data['content']: + if item.startswith("/lib/modules/"): + kernelstuff = True + # get the version of the modules + kmodver = item.split("/lib/modules/")[1] + kmodver = kmodver.split("/")[0] + + lp = kmodver.split("-")[-1] + if lp.startswith("r"): + kname = kmodver.split("-")[-2] + kver = kmodver.split("-")[0]+"-"+kmodver.split("-")[-1] + else: + kname = kmodver.split("-")[-1] + kver = kmodver.split("-")[0] + break + # validate the results above + if kernelstuff: + matchatom = "linux-%s-%s" % (kname,kver,) + if (matchatom == data['name']+"-"+data['version']): + kernelstuff_kernel = True + + data['versiontag'] = kmodver + if not kernelstuff_kernel: + data['slot'] = kmodver # if you change this behaviour, + # you must change "reagent update" + # and "equo database gentoosync" consequentially + + file_ext = etpConst['spm']['ebuild_file_extension'] + ebuilds_in_path = [x for x in os.listdir(tbz2TmpDir) if x.endswith(".%s" % (file_ext,))] + if not data['versiontag'] and ebuilds_in_path: + # has the user specified a custom package tag inside the ebuild + ebuild_path = os.path.join(tbz2TmpDir,ebuilds_in_path[0]) + data['versiontag'] = self._extract_pkg_metadata_ebuild_entropy_tag(ebuild_path) + + + data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/" + data['download'] += self.entropyTools.create_package_filename(data['category'], data['name'], data['version'], data['versiontag']) + + + data['trigger'] = "" + if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']): + f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb") + data['trigger'] = f.read() + f.close() + + Spm = self.Client.Spm() + + # Get Spm ChangeLog + pkgatom = "%s/%s-%s" % (data['category'],data['name'],data['version'],) + try: + data['changelog'] = Spm.get_package_changelog(pkgatom) + except: + data['changelog'] = None + + portage_metadata = Spm.calculate_dependencies( + data['iuse'], data['use'], data['license'], data['depend'], + data['rdepend'], data['pdepend'], data['provide'], data['sources'] + ) + + data['provide'] = portage_metadata['PROVIDE'].split() + data['license'] = portage_metadata['LICENSE'] + data['useflags'] = [] + for x in data['use'].split(): + if x.startswith("+"): + x = x[1:] + elif x.startswith("-"): + x = x[1:] + if (x in portage_metadata['USE']) or (x in portage_metadata['USE_MASK']): + data['useflags'].append(x) + else: + data['useflags'].append("-"+x) + data['sources'] = portage_metadata['SRC_URI'].split() + data['dependencies'] = {} + for x in portage_metadata['RDEPEND'].split(): + if x.startswith("!") or (x in ("(","||",")","")): + continue + data['dependencies'][x] = etpConst['spm']['(r)depend_id'] + for x in portage_metadata['PDEPEND'].split(): + if x.startswith("!") or (x in ("(","||",")","")): + continue + data['dependencies'][x] = etpConst['spm']['pdepend_id'] + data['conflicts'] = [x[1:] for x in portage_metadata['RDEPEND'].split()+portage_metadata['PDEPEND'].split() if x.startswith("!") and not x in ("(","||",")","")] + + if (kernelstuff) and (not kernelstuff_kernel): + # add kname to the dependency + data['dependencies']["=sys-kernel/linux-"+kname+"-"+kver+"~-1"] = etpConst['spm']['(r)depend_id'] + + # Conflicting tagged packages support + key = data['category']+"/"+data['name'] + confl_data = self.SystemSettings['conflicting_tagged_packages'].get(key) + if confl_data != None: + for conflict in confl_data: data['conflicts'].append(conflict) + + # Get License text if possible + licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses') + data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license']) + data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources']) + + # write only if it's a systempackage + data['systempackage'] = False + system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()] + if data['category']+"/"+data['name'] in system_packages: + data['systempackage'] = True + + # write only if it's a systempackage + protect, mask = Spm.get_config_protect_and_mask() + data['config_protect'] = protect + data['config_protect_mask'] = mask + + log_dir = etpConst['logdir']+"/elog" + if not os.path.isdir(log_dir): os.makedirs(log_dir) + data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent) + data['etpapi'] = etpConst['etpapi'] + + # removing temporary directory + shutil.rmtree(tbz2TmpDir,True) + if os.path.isdir(tbz2TmpDir): + try: os.remove(tbz2TmpDir) + except OSError: pass + + if not silent: + self.updateProgress( + red(info_package+_("Package extraction complete")), importance = 0, + type = "info", header = brown(" * "), back = True + ) + return data + + diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py new file mode 100644 index 000000000..ecaa97e6a --- /dev/null +++ b/libraries/entropy/client/interfaces/methods.py @@ -0,0 +1,1674 @@ +# -*- coding: utf-8 -*- +''' + # DESCRIPTION: + # Entropy Object Oriented Interface + + Copyright (C) 2007-2009 Fabio Erculiani + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +''' +from __future__ import with_statement +import os +import sys +import shutil +import time +import subprocess +from entropy.i18n import _ +from entropy.const import * +from entropy.exceptions import * +from entropy.db import dbapi2, LocalRepository +from entropy.output import purple, bold, red, blue, darkgreen, darkred, brown + + +class Repository: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.MirrorStatus = self.Client.MirrorStatus + self.updateProgress = self.Client.updateProgress + self.SystemSettings = self.Client.SystemSettings + self.entropyTools = self.Client.entropyTools + + self.__repo_error_messages_cache = set() + self.__repodb_cache = {} + self._memory_db_instances = {} + + def validate_repositories(self): + self.MirrorStatus.clear() + self.__repo_error_messages_cache.clear() + self.Client._package_match_validator_cache.clear() + # valid repositories + del self.Client.validRepositories[:] + for repoid in etpRepositoriesOrder: + # open database + try: + dbc = self.open_repository(repoid) + dbc.listConfigProtectDirectories() + dbc.validateDatabase() + self.Client.validRepositories.append(repoid) + except RepositoryError: + t = _("Repository") + " " + repoid + " " + _("is not available") + ". " + _("Cannot validate") + t2 = _("Please update your repositories now in order to remove this message!") + self.updateProgress( + darkred(t), + importance = 1, + type = "warning" + ) + self.updateProgress( + purple(t2), + header = bold("!!! "), + importance = 1, + type = "warning" + ) + continue # repo not available + except (self.Client.dbapi2.OperationalError,self.Client.dbapi2.DatabaseError,SystemDatabaseError,): + t = _("Repository") + " " + repoid + " " + _("is corrupted") + ". " + _("Cannot validate") + self.updateProgress( + darkred(t), + importance = 1, + type = "warning" + ) + continue + # to avoid having zillions of open files when loading a lot of EquoInterfaces + self.close_all_repositories(mask_clear = False) + + def init_generic_memory_repository(self, repoid, description, package_mirrors = []): + dbc = self.Client.open_memory_database(dbname = repoid) + self._memory_db_instances[repoid] = dbc + + # add to etpRepositories + repodata = { + 'repoid': repoid, + 'in_memory': True, + 'description': description, + 'packages': package_mirrors, + 'dbpath': ':memory:', + } + self.add_repository(repodata) + return dbc + + def close_all_repositories(self, mask_clear = True): + for item in self.__repodb_cache: + self.__repodb_cache[item].closeDB() + self.__repodb_cache.clear() + if mask_clear: self.SystemSettings.clear() + + def open_repository(self, repoid): + t_ident = 1 # thread.get_ident() disabled for now + if not self.__repodb_cache.has_key((repoid,etpConst['systemroot'],t_ident,)): + dbconn = self.load_repository_database(repoid, xcache = self.Client.xcache, + indexing = self.Client.indexing) + try: + dbconn.checkDatabaseApi() + except: + pass + self.__repodb_cache[(repoid,etpConst['systemroot'],t_ident,)] = dbconn + return dbconn + else: + return self.__repodb_cache.get((repoid,etpConst['systemroot'],t_ident,)) + + def load_repository_database(self, repoid, xcache = True, indexing = True): + + if isinstance(repoid,basestring): + if repoid.endswith(etpConst['packagesext']): + xcache = False + + if repoid not in etpRepositories: + t = _("bad repository id specified") + if repoid not in self.__repo_error_messages_cache: + self.updateProgress( + darkred(t), + importance = 2, + type = "warning" + ) + self.__repo_error_messages_cache.add(repoid) + raise RepositoryError("RepositoryError: %s" % (t,)) + + dbfile = etpRepositories[repoid]['dbpath']+"/"+etpConst['etpdatabasefile'] + if not os.path.isfile(dbfile): + t = _("Repository %s hasn't been downloaded yet.") % (repoid,) + if repoid not in self.__repo_error_messages_cache: + self.updateProgress( + darkred(t), + importance = 2, + type = "warning" + ) + self.__repo_error_messages_cache.add(repoid) + raise RepositoryError("RepositoryError: %s" % (t,)) + + conn = LocalRepository( + readOnly = True, + dbFile = dbfile, + clientDatabase = True, + dbname = etpConst['dbnamerepoprefix']+repoid, + xcache = xcache, + indexing = indexing, + OutputInterface = self, + ServiceInterface = self + ) + # initialize CONFIG_PROTECT + if (etpRepositories[repoid]['configprotect'] == None) or \ + (etpRepositories[repoid]['configprotectmask'] == None): + self.setup_repository_config(repoid, conn) + + if (repoid not in etpConst['client_treeupdatescalled']) and \ + (self.entropyTools.is_user_in_entropy_group()) and \ + (not repoid.endswith(etpConst['packagesext'])): + updated = False + try: + updated = conn.clientUpdatePackagesData(self.Client.clientDbconn) + except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): + pass + if updated: + self.Client.clear_dump_cache(etpCache['world_update']) + self.Client.clear_dump_cache(etpCache['world']) + self.Client.clear_dump_cache(etpCache['install']) + self.Client.clear_dump_cache(etpCache['remove']) + self.Client.calculate_world_updates(use_cache = False) + return conn + + def setup_repository_config(self, repoid, dbconn): + + try: + etpRepositories[repoid]['configprotect'] = dbconn.listConfigProtectDirectories() + except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): + etpRepositories[repoid]['configprotect'] = [] + try: + etpRepositories[repoid]['configprotectmask'] = dbconn.listConfigProtectDirectories(mask = True) + except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.DatabaseError): + etpRepositories[repoid]['configprotectmask'] = [] + + etpRepositories[repoid]['configprotect'] = [etpConst['systemroot']+x for \ + x in etpRepositories[repoid]['configprotect']] + etpRepositories[repoid]['configprotectmask'] = [etpConst['systemroot']+x for \ + x in etpRepositories[repoid]['configprotectmask']] + + etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for \ + x in etpConst['configprotect'] if etpConst['systemroot']+x not \ + in etpRepositories[repoid]['configprotect']] + etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for \ + x in etpConst['configprotectmask'] if etpConst['systemroot']+x not \ + in etpRepositories[repoid]['configprotectmask']] + + def get_repository_revision(self, reponame): + fname = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabaserevisionfile'] + revision = -1 + if os.path.isfile(fname) and os.access(fname,os.R_OK): + with open(fname,"r") as f: + try: + revision = int(f.readline().strip()) + except (OSError, IOError, ValueError,): + pass + return revision + + def update_repository_revision(self, reponame): + r = self.get_repository_revision(reponame) + etpRepositories[reponame]['dbrevision'] = "0" + if r != -1: + etpRepositories[reponame]['dbrevision'] = str(r) + + def get_repository_db_file_checksum(self, reponame): + fname = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasehashfile'] + mhash = "-1" + if os.path.isfile(fname) and os.access(fname,os.R_OK): + with open(fname,"r") as f: + try: + mhash = f.readline().strip().split()[0] + except (OSError, IOError, IndexError,): + pass + return mhash + + def add_repository(self, repodata): + # update etpRepositories + try: + etpRepositories[repodata['repoid']] = {} + etpRepositories[repodata['repoid']]['description'] = repodata['description'] + etpRepositories[repodata['repoid']]['configprotect'] = None + etpRepositories[repodata['repoid']]['configprotectmask'] = None + except KeyError: + t = _("repodata dictionary is corrupted") + raise InvalidData("InvalidData: %s" % (t,)) + + if repodata['repoid'].endswith(etpConst['packagesext']) or repodata.get('in_memory'): # dynamic repository + try: + # no need # etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] + etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:] + smart_package = repodata.get('smartpackage') + if smart_package != None: etpRepositories[repodata['repoid']]['smartpackage'] = smart_package + etpRepositories[repodata['repoid']]['dbpath'] = repodata.get('dbpath') + etpRepositories[repodata['repoid']]['pkgpath'] = repodata.get('pkgpath') + except KeyError: + raise InvalidData("InvalidData: repodata dictionary is corrupted") + # put at top priority, shift others + etpRepositoriesOrder.insert(0,repodata['repoid']) + else: + # XXX it's boring to keep this in sync with entropyConstants stuff, solutions? + etpRepositories[repodata['repoid']]['plain_packages'] = repodata['plain_packages'][:] + etpRepositories[repodata['repoid']]['packages'] = [x+"/"+etpConst['product'] for x in repodata['plain_packages']] + etpRepositories[repodata['repoid']]['plain_database'] = repodata['plain_database'] + etpRepositories[repodata['repoid']]['database'] = repodata['plain_database'] + \ + "/" + etpConst['product'] + "/database/" + etpConst['currentarch'] + "/" + etpConst['branch'] + if not repodata['dbcformat'] in etpConst['etpdatabasesupportedcformats']: + repodata['dbcformat'] = etpConst['etpdatabasesupportedcformats'][0] + etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat'] + etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + \ + "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch'] + "/" + etpConst['branch'] + # set dbrevision + myrev = self.get_repository_revision(repodata['repoid']) + if myrev == -1: + myrev = 0 + etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev) + if repodata.has_key("position"): + etpRepositoriesOrder.insert(repodata['position'],repodata['repoid']) + else: + etpRepositoriesOrder.append(repodata['repoid']) + if not repodata.has_key("service_port"): + repodata['service_port'] = int(etpConst['socket_service']['port']) + if not repodata.has_key("ssl_service_port"): + repodata['ssl_service_port'] = int(etpConst['socket_service']['ssl_port']) + etpRepositories[repodata['repoid']]['service_port'] = repodata['service_port'] + etpRepositories[repodata['repoid']]['ssl_service_port'] = repodata['ssl_service_port'] + self.Client.repository_move_clear_cache(repodata['repoid']) + # save new etpRepositories to file + self.entropyTools.saveRepositorySettings(repodata) + self.Client.reload_constants() + self.validate_repositories() + + def remove_repository(self, repoid, disable = False): + + # ensure that all dbs are closed + self.close_all_repositories() + + done = False + if etpRepositories.has_key(repoid): + del etpRepositories[repoid] + done = True + + if etpRepositoriesExcluded.has_key(repoid): + del etpRepositoriesExcluded[repoid] + done = True + + if done: + + if repoid in etpRepositoriesOrder: + etpRepositoriesOrder.remove(repoid) + + self.Client.repository_move_clear_cache(repoid) + # save new etpRepositories to file + repodata = {} + repodata['repoid'] = repoid + if disable: + self.entropyTools.saveRepositorySettings(repodata, disable = True) + else: + self.entropyTools.saveRepositorySettings(repodata, remove = True) + self.Client.reload_constants() + + self.validate_repositories() + + def shift_repository(self, repoid, toidx): + # update etpRepositoriesOrder + etpRepositoriesOrder.remove(repoid) + etpRepositoriesOrder.insert(toidx,repoid) + self.entropyTools.writeOrderedRepositoriesEntries() + self.Client.reload_constants() + self.Client.repository_move_clear_cache(repoid) + self.validate_repositories() + + def enable_repository(self, repoid): + self.Client.repository_move_clear_cache(repoid) + # save new etpRepositories to file + repodata = {} + repodata['repoid'] = repoid + self.entropyTools.saveRepositorySettings(repodata, enable = True) + self.Client.reload_constants() + self.validate_repositories() + + def disable_repository(self, repoid): + # update etpRepositories + done = False + try: + del etpRepositories[repoid] + done = True + except: + pass + + if done: + try: + etpRepositoriesOrder.remove(repoid) + except: + pass + # it's not vital to reset etpRepositoriesOrder counters + + self.Client.repository_move_clear_cache(repoid) + # save new etpRepositories to file + repodata = {} + repodata['repoid'] = repoid + self.entropyTools.saveRepositorySettings(repodata, disable = True) + self.Client.reload_constants() + self.validate_repositories() + + # every tbz2 file that would be installed must pass from here + def add_tbz2_to_repos(self, tbz2file): + atoms_contained = [] + basefile = os.path.basename(tbz2file) + cut_idx = -1*(len(etpConst['packagesext'])) + db_dir = etpConst['entropyunpackdir']+"/"+basefile[:cut_idx] + if os.path.isdir(db_dir): + shutil.rmtree(db_dir) + os.makedirs(db_dir) + dbfile = self.entropyTools.extractEdb(tbz2file, + dbpath = db_dir+"/packages.db") + if dbfile == None: + return -1,atoms_contained + etpSys['dirstoclean'].add(os.path.dirname(dbfile)) + # add dbfile + repodata = {} + repodata['repoid'] = basefile + repodata['description'] = "Dynamic database from "+basefile + repodata['packages'] = [] + repodata['dbpath'] = os.path.dirname(dbfile) + repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added + repodata['smartpackage'] = False # extra info added + + mydbconn = self.Client.open_generic_database(dbfile) + # read all idpackages + try: + myidpackages = mydbconn.listAllIdpackages() # all branches admitted from external files + except (AttributeError, self.Client.dbapi2.DatabaseError, \ + self.Client.dbapi2.IntegrityError, self.Client.dbapi2.OperationalError,): + return -2,atoms_contained + if len(myidpackages) > 1: + repodata[basefile]['smartpackage'] = True + for myidpackage in myidpackages: + compiled_arch = mydbconn.retrieveDownloadURL(myidpackage) + if compiled_arch.find("/"+etpSys['arch']+"/") == -1: + return -3,atoms_contained + atoms_contained.append((int(myidpackage),basefile)) + + self.add_repository(repodata) + self.validate_repositories() + if basefile not in self.Client.validRepositories: + self.remove_repository(basefile) + return -4,atoms_contained + mydbconn.closeDB() + del mydbconn + return 0,atoms_contained + + def reopen_client_repository(self): + self.clientDbconn.closeDB() + self.open_client_repository() + + def __client_repo_setup_const(self, conn): + if conn.doesTableExist('configprotect') and conn.doesTableExist('configprotectreference'): + etpConst['dbconfigprotect'] = conn.listConfigProtectDirectories() + if conn.doesTableExist('configprotectmask') and conn.doesTableExist('configprotectreference'): + etpConst['dbconfigprotectmask'] = conn.listConfigProtectDirectories(mask = True) + + etpConst['dbconfigprotect'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] + etpConst['dbconfigprotectmask'] = [etpConst['systemroot']+x for x in etpConst['dbconfigprotect']] + + etpConst['dbconfigprotect'] += [etpConst['systemroot']+x for \ + x in etpConst['configprotect'] if etpConst['systemroot']+x not \ + in etpConst['dbconfigprotect']] + etpConst['dbconfigprotectmask'] += [etpConst['systemroot']+x for \ + x in etpConst['configprotectmask'] if etpConst['systemroot']+x not \ + in etpConst['dbconfigprotectmask']] + + def open_client_repository(self): + + def load_db_from_ram(): + self.Client.safe_mode = etpConst['safemodeerrors']['clientdb'] + mytxt = "%s, %s" % (_("System database not found or corrupted"), + _("running in safe mode using empty database from RAM"),) + self.updateProgress( + darkred(mytxt), + importance = 1, + type = "warning", + header = bold("!!!"), + ) + conn = self.open_memory_database(dbname = etpConst['clientdbid']) + return conn + + db_dir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) + if not os.path.isdir(db_dir): os.makedirs(db_dir) + + db_path = etpConst['etpdatabaseclientfilepath'] + if (not self.Client.noclientdb) and (not os.path.isfile(db_path)): + conn = load_db_from_ram() + self.entropyTools.printTraceback(f = self.Client.clientLog) + else: + conn = LocalRepository(readOnly = False, dbFile = db_path, + clientDatabase = True, dbname = etpConst['clientdbid'], + xcache = self.Client.xcache, indexing = self.Client.indexing, + OutputInterface = self, ServiceInterface = self + ) + # validate database + if not self.Client.noclientdb: + try: + conn.validateDatabase() + except SystemDatabaseError: + try: + conn.closeDB() + except: + pass + self.entropyTools.printTraceback(f = self.Client.clientLog) + conn = load_db_from_ram() + + if not etpConst['dbconfigprotect']: + self.__client_repo_setup_const(conn) + + self.Client.clientDbconn = conn + return self.Client.clientDbconn + + def client_repository_sanity_check(self): + self.updateProgress( + darkred(_("Sanity Check") + ": " + _("system database")), + importance = 2, + type = "warning" + ) + idpkgs = self.clientDbconn.listAllIdpackages() + length = len(idpkgs) + count = 0 + errors = False + scanning_txt = _("Scanning...") + for x in idpkgs: + count += 1 + self.updateProgress( + darkgreen(scanning_txt), + importance = 0, + type = "info", + back = True, + count = (count,length), + percent = True + ) + try: + self.clientDbconn.getPackageData(x) + except Exception ,e: + self.entropyTools.printTraceback() + errors = True + self.updateProgress( + darkred(_("Errors on idpackage %s, error: %s")) % (x,str(e)), + importance = 0, + type = "warning" + ) + + if not errors: + t = _("Sanity Check") + ": %s" % (bold(_("PASSED")),) + self.updateProgress( + darkred(t), + importance = 2, + type = "warning" + ) + return 0 + else: + t = _("Sanity Check") + ": %s" % (bold(_("CORRUPTED")),) + self.updateProgress( + darkred(t), + importance = 2, + type = "warning" + ) + return -1 + + def open_generic_database(self, dbfile, dbname = None, xcache = None, + readOnly = False, indexing_override = None, skipChecks = False): + if xcache == None: + xcache = self.Client.xcache + if indexing_override != None: + indexing = indexing_override + else: + indexing = self.Client.indexing + if dbname == None: + dbname = etpConst['genericdbid'] + return LocalRepository( + readOnly = readOnly, + dbFile = dbfile, + clientDatabase = True, + dbname = dbname, + xcache = xcache, + indexing = indexing, + OutputInterface = self, + skipChecks = skipChecks + ) + + def open_memory_database(self, dbname = None): + if dbname == None: + dbname = etpConst['genericdbid'] + dbc = LocalRepository( + readOnly = False, + dbFile = ':memory:', + clientDatabase = True, + dbname = dbname, + xcache = False, + indexing = False, + OutputInterface = self, + skipChecks = True + ) + dbc.initializeDatabase() + return dbc + + def backup_database(self, dbpath, backup_dir = None, silent = False, compress_level = 9): + + if compress_level not in range(1,10): + compress_level = 9 + + backup_dir = os.path.dirname(dbpath) + if not backup_dir: backup_dir = os.path.dirname(dbpath) + dbname = os.path.basename(dbpath) + bytes_required = 1024000*300 + if not (os.access(backup_dir,os.W_OK) and \ + os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \ + os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)): + if not silent: + mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),) + self.updateProgress( + mytxt, + importance = 1, + type = "error", + header = red(" @@ ") + ) + return False, mytxt + + def get_ts(): + from datetime import datetime + ts = datetime.fromtimestamp(time.time()) + return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second) + + comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),) + comp_dbpath = os.path.join(backup_dir,comp_dbname) + if not silent: + mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + import bz2 + try: + self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) + except: + if not silent: + self.entropyTools.printTraceback() + return False, _("Unable to compress") + + if not silent: + mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + return True, _("All fine") + + def restore_database(self, backup_path, db_destination, silent = False): + + bytes_required = 1024000*300 + db_dir = os.path.dirname(db_destination) + if not (os.access(db_dir,os.W_OK) and os.path.isdir(db_dir) and \ + os.path.isfile(backup_path) and os.access(backup_path,os.R_OK) and \ + self.entropyTools.check_required_space(db_dir, bytes_required)): + + if not silent: + mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")), + blue(backup_path),darkred(_("permission denied")),) + self.updateProgress( + mytxt, + importance = 1, + type = "error", + header = red(" @@ ") + ) + return False, mytxt + + if not silent: + mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")), + blue(os.path.basename(backup_path)),blue(db_destination),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + + import bz2 + try: + self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File) + except: + if not silent: + self.entropyTools.printTraceback() + return False, _("Unable to unpack") + + if not silent: + mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")), + blue(db_destination),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + self.Client.purge_cache() + return True, _("All fine") + + def list_backedup_client_databases(self, client_dbdir = None): + if not client_dbdir: + client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) + return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \ + if x.startswith(etpConst['dbbackupprefix']) and \ + os.access(os.path.join(client_dbdir,x),os.R_OK) + ] + + """ + XXX deprecated XXX + """ + + def backupDatabase(self, *args, **kwargs): + import warning + warning.warn("deprecated, use backup_database instead") + return self.backup_database(*args, **kwargs) + + def restoreDatabase(self, *args, **kwargs): + import warning + warning.warn("deprecated, use restore_database instead") + return self.restore_database(*args, **kwargs) + + def openGenericDatabase(self, *args, **kwargs): + import warning + warning.warn("deprecated, use open_generic_database instead") + return self.open_generic_database(*args, **kwargs) + + def openMemoryDatabase(self, *args, **kwargs): + import warning + warning.warn("deprecated, use open_memory_database instead") + return self.open_memory_database(*args, **kwargs) + + def clientDatabaseSanityCheck(self): + import warning + warning.warn("deprecated, use client_repository_sanity_check instead") + return self.client_repository_sanity_check() + + def openClientDatabase(self): + import warning + warning.warn("deprecated, use open_client_repository instead") + return self.open_client_repository() + + def reopenClientDbconn(self): + import warning + warning.warn("deprecated, use reopen_client_repository instead") + return self.reopen_client_repository() + + def openRepositoryDatabase(self, repoid): + import warning + warning.warn("deprecated, use open_repository instead") + return self.open_repository(repoid) + + def closeAllRepositoryDatabases(self, mask_clear = True): + import warning + warning.warn("deprecated, use close_all_repositories instead") + return self.close_all_repositories(mask_clear = mask_clear) + + def addRepository(self, repodata): + import warning + warning.warn("deprecated, use add_repository instead") + return self.add_repository(repodata) + + def removeRepository(self, repoid, disable = False): + import warning + warning.warn("deprecated, use remove_repository instead") + return self.remove_repository(repoid, disable = disable) + + def shiftRepository(self, repoid, toidx): + import warning + warning.warn("deprecated, use shift_repository instead") + return self.shift_repository(repoid, toidx) + + def enableRepository(self, repoid): + import warning + warning.warn("deprecated, use enable_repository instead") + return self.enable_repository(repoid) + + def disableRepository(self, repoid): + import warning + warning.warn("deprecated, use disable_repository instead") + return self.disable_repository(repoid) + + +class Misc: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.entropyTools = self.Client.entropyTools + self.updateProgress = self.Client.updateProgress + self.SystemSettings = self.Client.SystemSettings + + def reload_constants(self): + initConfig_entropyConstants(etpSys['rootdir']) + initConfig_clientConstants() + + def setup_default_file_perms(self, filepath): + # setup file permissions + os.chmod(filepath,0664) + if etpConst['entropygid'] != None: + os.chown(filepath,-1,etpConst['entropygid']) + + def _resources_run_create_lock(self): + self.create_pid_file_lock(etpConst['locks']['using_resources']) + + def _resources_run_remove_lock(self): + if os.path.isfile(etpConst['locks']['using_resources']): + os.remove(etpConst['locks']['using_resources']) + + def _resources_run_check_lock(self): + rc = self.check_pid_file_lock(etpConst['locks']['using_resources']) + return rc + + def check_pid_file_lock(self, pidfile): + if not os.path.isfile(pidfile): + return False # not locked + f = open(pidfile) + s_pid = f.readline().strip() + f.close() + try: + s_pid = int(s_pid) + except ValueError: + return False # not locked + # is it our pid? + mypid = os.getpid() + if (s_pid != mypid) and os.path.isdir("%s/proc/%s" % (etpConst['systemroot'],s_pid,)): + # is it running + return True # locked + return False + + def create_pid_file_lock(self, pidfile, mypid = None): + lockdir = os.path.dirname(pidfile) + if not os.path.isdir(lockdir): + os.makedirs(lockdir,0775) + const_setup_perms(lockdir,etpConst['entropygid']) + if mypid == None: + mypid = os.getpid() + f = open(pidfile,"w") + f.write(str(mypid)) + f.flush() + f.close() + + def application_lock_check(self, silent = False): + # check if another instance is running + etpConst['applicationlock'] = False + const_setupEntropyPid(just_read = True) + locked = self.entropyTools.applicationLockCheck(option = None, gentle = True, silent = True) + if locked: + if not silent: + self.updateProgress( + red(_("Another Entropy instance is currently active, cannot satisfy your request.")), + importance = 1, + type = "error", + header = darkred(" @@ ") + ) + return True + return False + + def lock_check(self, check_function): + + lock_count = 0 + max_lock_count = 600 + sleep_seconds = 0.5 + + # check lock file + while 1: + locked = check_function() + if not locked: + if lock_count > 0: + self.updateProgress( + blue(_("Resources unlocked, let's go!")), + importance = 1, + type = "info", + header = darkred(" @@ ") + ) + break + if lock_count >= max_lock_count: + mycalc = max_lock_count*sleep_seconds/60 + self.updateProgress( + blue(_("Resources still locked after %s minutes, giving up!")) % (mycalc,), + importance = 1, + type = "warning", + header = darkred(" @@ ") + ) + return True # gave up + lock_count += 1 + self.updateProgress( + blue(_("Resources locked, sleeping %s seconds, check #%s/%s")) % ( + sleep_seconds, + lock_count, + max_lock_count, + ), + importance = 1, + type = "warning", + header = darkred(" @@ "), + back = True + ) + time.sleep(sleep_seconds) + return False # yay! + + def backup_setting(self, setting_name): + if etpConst.has_key(setting_name): + myinst = etpConst[setting_name] + if type(etpConst[setting_name]) in (list,tuple): + myinst = etpConst[setting_name][:] + elif type(etpConst[setting_name]) in (dict,set): + myinst = etpConst[setting_name].copy() + else: + myinst = etpConst[setting_name] + etpConst['backed_up'].update({setting_name: myinst}) + else: + t = _("Nothing to backup in etpConst with %s key") % (setting_name,) + raise InvalidData("InvalidData: %s" % (t,)) + + def set_priority(self, low = 0): + return const_setNiceLevel(low) + + def reload_repositories_config(self, repositories = None): + if repositories == None: + repositories = self.Client.validRepositories + for repoid in repositories: + dbconn = self.Client.open_repository(repoid) + self.Client.setup_repository_config(repoid, dbconn) + + def switch_chroot(self, chroot = ""): + # clean caches + self.Client.purge_cache() + self.Client.close_all_repositories() + if chroot.endswith("/"): + chroot = chroot[:-1] + etpSys['rootdir'] = chroot + self.reload_constants() + self.Client.validate_repositories() + self.Client.reopen_client_repository() + if chroot: + try: self.Client.clientDbconn.resetTreeupdatesDigests() + except: pass + # I don't think it's safe to keep them open + # isn't it? + self.Client.closeAllSecurity() + self.Client.closeAllQA() + + def get_file_viewer(self): + viewer = None + if os.access("/usr/bin/less",os.X_OK): + viewer = "/usr/bin/less" + elif os.access("/bin/more",os.X_OK): + viewer = "/bin/more" + if not viewer: + viewer = self.get_file_editor() + return viewer + + def get_file_editor(self): + editor = None + if os.getenv("EDITOR"): + editor = "$EDITOR" + elif os.access("/bin/nano",os.X_OK): + editor = "/bin/nano" + elif os.access("/bin/vi",os.X_OK): + editor = "/bin/vi" + elif os.access("/usr/bin/vi",os.X_OK): + editor = "/usr/bin/vi" + elif os.access("/usr/bin/emacs",os.X_OK): + editor = "/usr/bin/emacs" + elif os.access("/bin/emacs",os.X_OK): + editor = "/bin/emacs" + return editor + + def add_user_package_set(self, set_name, set_atoms): + + def _ensure_package_sets_dir(): + sets_dir = etpConst['confsetsdir'] + if not os.path.isdir(sets_dir): + if os.path.lexists(sets_dir): + os.remove(sets_dir) + os.makedirs(sets_dir,0775) + const_setup_perms(sets_dir, etpConst['entropygid']) + + try: + set_name = str(set_name) + except (UnicodeEncodeError,UnicodeDecodeError,): + raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) + + if set_name.startswith(etpConst['packagesetprefix']): + raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) + set_match, rc = self.Client.package_set_match(set_name) + if rc: return -1,_("Name already taken") + + _ensure_package_sets_dir() + set_file = os.path.join(etpConst['confsetsdir'],set_name) + if os.path.isfile(set_file) and os.access(set_file,os.W_OK): + try: + os.remove(set_file) + except OSError: + return -2,_("Cannot remove the old element") + if not os.access(os.path.dirname(set_file),os.W_OK): + return -3,_("Cannot create the element") + + f = open(set_file,"w") + for x in set_atoms: f.write("%s\n" % (x,)) + f.flush() + f.close() + self.SystemSettings['system_package_sets'][set_name] = set(set_atoms) + return 0,_("All fine") + + def remove_user_package_set(self, set_name): + + try: + set_name = str(set_name) + except (UnicodeEncodeError,UnicodeDecodeError,): + raise InvalidPackageSet("InvalidPackageSet: %s %s" % (set_name,_("must be an ASCII string"),)) + + if set_name.startswith(etpConst['packagesetprefix']): + raise InvalidPackageSet("InvalidPackageSet: %s %s '%s'" % (set_name,_("cannot start with"),etpConst['packagesetprefix'],)) + + set_match, rc = self.Client.package_set_match(set_name) + if not rc: return -1,_("Already removed") + set_id, set_x, set_y = set_match + + if set_id != etpConst['userpackagesetsid']: + return -2,_("Not defined by user") + set_file = os.path.join(etpConst['confsetsdir'],set_name) + if os.path.isfile(set_file) and os.access(set_file,os.W_OK): + os.remove(set_file) + if set_name in self.SystemSettings['system_package_sets']: + del self.SystemSettings['system_package_sets'][set_name] + return 0,_("All fine") + return -3,_("Set not found or unable to remove") + + def is_installed_idpackage_in_system_mask(self, idpackage): + if idpackage in self.SystemSettings['repos_system_mask_installed']: + return True + return False + + def get_branch_from_download_relative_uri(self, db_download_uri): + return db_download_uri.split("/")[2] + + def unused_packages_test(self, dbconn = None): + if dbconn == None: dbconn = self.Client.clientDbconn + return [x for x in dbconn.retrieveUnusedIdpackages() if self.Client.validate_package_removal(x)] + + def get_licenses_to_accept(self, install_queue): + if not install_queue: + return {} + licenses = {} + for match in install_queue: + repoid = match[1] + dbconn = self.Client.open_repository(repoid) + wl = self.SystemSettings['repos_license_whitelist'].get(repoid) + if not wl: + continue + keys = dbconn.retrieveLicensedataKeys(match[0]) + for key in keys: + if key not in wl: + found = self.Client.clientDbconn.isLicenseAccepted(key) + if found: + continue + if not licenses.has_key(key): + licenses[key] = set() + licenses[key].add(match) + return licenses + + def get_text_license(self, license_name, repoid): + dbconn = self.Client.open_repository(repoid) + text = dbconn.retrieveLicenseText(license_name) + tempfile = self.entropyTools.getRandomTempFile() + f = open(tempfile,"w") + f.write(text) + f.flush() + f.close() + return tempfile + + def libraries_test(self, dbconn = None, broken_symbols = False, task_bombing_func = None): + + if dbconn == None: + dbconn = self.Client.clientDbconn + + self.updateProgress( + blue(_("Libraries test")), + importance = 2, + type = "info", + header = red(" @@ ") + ) + + if not etpConst['systemroot']: + myroot = "/" + else: + myroot = etpConst['systemroot']+"/" + # run ldconfig first + subprocess.call("ldconfig -r %s &> /dev/null" % (myroot,), shell = True) + # open /etc/ld.so.conf + if not os.path.isfile(etpConst['systemroot']+"/etc/ld.so.conf"): + self.updateProgress( + blue(_("Cannot find "))+red(etpConst['systemroot']+"/etc/ld.so.conf"), + importance = 1, + type = "error", + header = red(" @@ ") + ) + return {},set(),-1 + + ldpaths = set(self.entropyTools.collectLinkerPaths()) + ldpaths |= self.entropyTools.collectPaths() + # speed up when /usr/lib is a /usr/lib64 symlink + if "/usr/lib64" in ldpaths and "/usr/lib" in ldpaths: + if os.path.realpath("/usr/lib64") == "/usr/lib": + ldpaths.discard("/usr/lib") + # some crappy packages put shit here too + ldpaths.add("/usr/share") + # always force /usr/libexec too + ldpaths.add("/usr/libexec") + + executables = set() + total = len(ldpaths) + count = 0 + sys_root_len = len(etpConst['systemroot']) + for ldpath in ldpaths: + if callable(task_bombing_func): task_bombing_func() + count += 1 + self.updateProgress( + blue("Tree: ")+red(etpConst['systemroot']+ldpath), + importance = 0, + type = "info", + count = (count,total), + back = True, + percent = True, + header = " " + ) + ldpath = ldpath.encode(sys.getfilesystemencoding()) + mywalk_iter = os.walk(etpConst['systemroot']+ldpath) + + def mywimf(dt): + + currentdir, subdirs, files = dt + + def mymf(item): + filepath = os.path.join(currentdir,item) + if filepath in etpConst['libtest_files_blacklist']: + return 0 + if not os.access(filepath,os.R_OK): + return 0 + if not os.path.isfile(filepath): + return 0 + if not self.entropyTools.is_elf_file(filepath): + return 0 + return filepath[sys_root_len:] + + return set([x for x in map(mymf,files) if type(x) != int]) + + for x in map(mywimf,mywalk_iter): executables |= x + + self.updateProgress( + blue(_("Collecting broken executables")), + importance = 2, + type = "info", + header = red(" @@ ") + ) + t = red(_("Attention")) + ": " + \ + blue(_("don't worry about libraries that are shown here but not later.")) + self.updateProgress( + t, + importance = 1, + type = "info", + header = red(" @@ ") + ) + + myQA = self.Client.QA() + + plain_brokenexecs = set() + total = len(executables) + count = 0 + scan_txt = blue("%s ..." % (_("Scanning libraries"),)) + for executable in executables: + if callable(task_bombing_func): task_bombing_func() + count += 1 + if (count%10 == 0) or (count == total) or (count == 1): + self.updateProgress( + scan_txt, + importance = 0, + type = "info", + count = (count,total), + back = True, + percent = True, + header = " " + ) + myelfs = self.entropyTools.read_elf_dynamic_libraries(etpConst['systemroot']+executable) + def mymf2(mylib): + return not myQA.resolve_dynamic_library(mylib, executable) + mylibs = set(filter(mymf2,myelfs)) + broken_sym_found = set() + if broken_symbols and not mylibs: broken_sym_found |= self.entropyTools.read_elf_broken_symbols(etpConst['systemroot']+executable) + if not (mylibs or broken_sym_found): + continue + + if mylibs: + alllibs = blue(' :: ').join(list(mylibs)) + self.updateProgress( + red(etpConst['systemroot']+executable)+" [ "+alllibs+" ]", + importance = 1, + type = "info", + percent = True, + count = (count,total), + header = " " + ) + elif broken_sym_found: + allsyms = darkred(' :: ').join([brown(x) for x in list(broken_sym_found)]) + if len(allsyms) > 50: allsyms = brown(_('various broken symbols')) + self.updateProgress( + red(etpConst['systemroot']+executable)+" { "+allsyms+" }", + importance = 1, + type = "info", + percent = True, + count = (count,total), + header = " " + ) + plain_brokenexecs.add(executable) + + del executables + packagesMatched = {} + + if not etpSys['serverside']: + + self.updateProgress( + blue(_("Matching broken libraries/executables")), + importance = 1, + type = "info", + header = red(" @@ ") + ) + matched = set() + for brokenlib in plain_brokenexecs: + idpackages = self.Client.clientDbconn.searchBelongs(brokenlib) + for idpackage in idpackages: + key, slot = self.Client.clientDbconn.retrieveKeySlot(idpackage) + mymatch = self.Client.atom_match(key, matchSlot = slot) + if mymatch[0] == -1: + matched.add(brokenlib) + continue + cmpstat = self.Client.get_package_action(mymatch) + if cmpstat == 0: + continue + if not packagesMatched.has_key(brokenlib): + packagesMatched[brokenlib] = set() + packagesMatched[brokenlib].add(mymatch) + matched.add(brokenlib) + plain_brokenexecs -= matched + + return packagesMatched,plain_brokenexecs,0 + + def move_to_branch(self, branch, pretend = False): + if pretend: return 0 + if branch != etpConst['branch']: + # update configuration + self.entropyTools.writeNewBranch(branch) + # reset treeupdatesactions + self.Client.clientDbconn.resetTreeupdatesDigests() + # clean cache + self.Client.purge_cache(showProgress = False) + # reopen Client Database, this will make treeupdates to be re-read + self.Client.close_all_repositories() + etpConst['branch'] = branch + self.Client.reload_constants() + etpConst['branch'] = branch + self.Client.validate_repositories() + self.Client.reopen_client_repository() + return 0 + + def get_meant_packages(self, search_term, from_installed = False, valid_repos = []): + + pkg_data = [] + atom_srch = False + if "/" in search_term: atom_srch = True + + if not valid_repos: valid_repos = self.Client.validRepositories + if from_installed: valid_repos = [1] + for repo in valid_repos: + if isinstance(repo,basestring): + dbconn = self.Client.open_repository(repo) + elif isinstance(repo,LocalRepository): + dbconn = repo + elif hasattr(self,'clientDbconn'): + dbconn = self.Client.clientDbconn + else: + continue + pkg_data.extend([(x,repo,) for x in dbconn.searchSimilarPackages(search_term, atom = atom_srch)]) + + return pkg_data + + def list_repo_categories(self): + categories = set() + for repo in self.Client.validRepositories: + dbconn = self.Client.open_repository(repo) + catsdata = dbconn.listAllCategories() + categories.update(set([x[1] for x in catsdata])) + return categories + + def list_repo_packages_in_category(self, category): + pkg_matches = [] + for repo in self.Client.validRepositories: + dbconn = self.Client.open_repository(repo) + catsdata = dbconn.searchPackagesByCategory(category, branch = etpConst['branch']) + pkg_matches.extend([(x[1],repo,) for x in catsdata if (x[1],repo,) not in pkg_matches]) + return pkg_matches + + def get_category_description_data(self, category): + + data = {} + for repo in self.Client.validRepositories: + try: + dbconn = self.Client.open_repository(repo) + except RepositoryError: + continue + try: + data = dbconn.retrieveCategoryDescription(category) + except (self.Client.dbapi2.OperationalError, self.Client.dbapi2.IntegrityError,): + continue + if data: break + + return data + + def list_installed_packages_in_category(self, category): + pkg_matches = set([x[1] for x in self.Client.clientDbconn.searchPackagesByCategory(category)]) + return pkg_matches + + def inject_entropy_database_into_package(self, package_filename, data, treeupdates_actions = None): + dbpath = self.get_tmp_dbpath() + dbconn = self.Client.open_generic_database(dbpath) + dbconn.initializeDatabase() + dbconn.addPackage(data, revision = data['revision']) + if treeupdates_actions != None: + dbconn.bumpTreeUpdatesActions(treeupdates_actions) + dbconn.commitChanges() + dbconn.closeDB() + self.entropyTools.aggregateEdb(tbz2file = package_filename, dbfile = dbpath) + return dbpath + + def get_tmp_dbpath(self): + dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) + while os.path.isfile(dbpath): + dbpath = etpConst['packagestmpdir']+"/"+str(self.entropyTools.getRandomNumber()) + return dbpath + + def quickpkg(self, atomstring, savedir = None): + if savedir == None: + savedir = etpConst['packagestmpdir'] + if not os.path.isdir(etpConst['packagestmpdir']): + os.makedirs(etpConst['packagestmpdir']) + # match package + match = self.Client.clientDbconn.atomMatch(atomstring) + if match[0] == -1: + return -1,None,None + atom = self.Client.clientDbconn.atomMatch(match[0]) + pkgdata = self.Client.clientDbconn.getPackageData(match[0]) + resultfile = self.quickpkg_handler(pkgdata = pkgdata, dirpath = savedir) + if resultfile == None: + return -1,atom,None + else: + return 0,atom,resultfile + + def quickpkg_handler(self, pkgdata, dirpath, edb = True, + portdbPath = None, fake = False, compression = "bz2", shiftpath = ""): + + import stat + import tarfile + + if compression not in ("bz2","","gz"): + compression = "bz2" + + # getting package info + pkgtag = '' + pkgrev = "~"+str(pkgdata['revision']) + if pkgdata['versiontag']: pkgtag = "#"+pkgdata['versiontag'] + pkgname = pkgdata['name']+"-"+pkgdata['version']+pkgrev+pkgtag # + version + tag + pkgcat = pkgdata['category'] + #pkgfile = pkgname+etpConst['packagesext'] + dirpath += "/"+pkgname+etpConst['packagesext'] + if os.path.isfile(dirpath): + os.remove(dirpath) + tar = tarfile.open(dirpath,"w:"+compression) + + if not fake: + + contents = sorted([x for x in pkgdata['content']]) + id_strings = {} + + # collect files + for path in contents: + # convert back to filesystem str + encoded_path = path + path = path.encode('raw_unicode_escape') + path = shiftpath+path + try: + exist = os.lstat(path) + except OSError: + continue # skip file + arcname = path[len(shiftpath):] # remove shiftpath + if arcname.startswith("/"): + arcname = arcname[1:] # remove trailing / + ftype = pkgdata['content'][encoded_path] + if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype + if 'dir' == ftype and \ + not stat.S_ISDIR(exist.st_mode) and \ + os.path.isdir(path): # workaround for directory symlink issues + path = os.path.realpath(path) + + tarinfo = tar.gettarinfo(path, arcname) + tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid)) + tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid)) + + if stat.S_ISREG(exist.st_mode): + tarinfo.type = tarfile.REGTYPE + f = open(path) + try: + tar.addfile(tarinfo, f) + finally: + f.close() + else: + tar.addfile(tarinfo) + + tar.close() + + # appending xpak metadata + if etpConst['gentoo-compat']: + import entropy.xpak as xpak + Spm = self.Client.Spm() + + gentoo_name = self.entropyTools.remove_tag(pkgname) + gentoo_name = self.entropyTools.remove_entropy_revision(gentoo_name) + if portdbPath == None: + dbdir = Spm.get_vdb_path()+"/"+pkgcat+"/"+gentoo_name+"/" + else: + dbdir = portdbPath+"/"+pkgcat+"/"+gentoo_name+"/" + if os.path.isdir(dbdir): + tbz2 = xpak.tbz2(dirpath) + tbz2.recompose(dbdir) + + if edb: + self.inject_entropy_database_into_package(dirpath, pkgdata) + + if os.path.isfile(dirpath): + return dirpath + return None + + """ + XXX deprecated XXX + """ + + def switchChroot(self, *args, **kwargs): + import warning + warning.warn("deprecated, use switch_chroot instead") + return self.switch_chroot(*args, **kwargs) + + +class Match: + + def __init__(self, ClientInterface): + from entropy.client.interfaces import Client + if not isinstance(ClientInterface,Client): + mytxt = _("A valid Client instance or subclass is needed") + raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,)) + self.Client = ClientInterface + self.entropyTools = self.Client.entropyTools + self.updateProgress = self.Client.updateProgress + self.SystemSettings = self.Client.SystemSettings + self.Cacher = self.Client.Cacher + + def get_package_action(self, match): + """ + @input: matched atom (idpackage,repoid) + @output: + upgrade: int(2) + install: int(1) + reinstall: int(0) + downgrade: int(-1) + """ + dbconn = self.Client.open_repository(match[1]) + pkgkey, pkgslot = dbconn.retrieveKeySlot(match[0]) + results = self.Client.clientDbconn.searchKeySlot(pkgkey, pkgslot) + if not results: return 1 + + installed_idpackage = results[0][0] + pkgver, pkgtag, pkgrev = dbconn.getVersioningData(match[0]) + installedVer, installedTag, installedRev = self.Client.clientDbconn.getVersioningData(installed_idpackage) + pkgcmp = self.entropyTools.entropyCompareVersions((pkgver,pkgtag,pkgrev),(installedVer,installedTag,installedRev)) + if pkgcmp == 0: + return 0 + elif pkgcmp > 0: + return 2 + return -1 + + def get_masked_package_reason(self, match): + idpackage, repoid = match + dbconn = self.Client.open_repository(repoid) + idpackage, idreason = dbconn.idpackageValidator(idpackage) + masked = False + if idpackage == -1: masked = True + return masked, idreason, self.SystemSettings['pkg_masking_reasons'].get(idreason) + + def get_match_conflicts(self, match): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + conflicts = dbconn.retrieveConflicts(m_id) + found_conflicts = set() + for conflict in conflicts: + my_m_id, my_m_rc = self.Client.clientDbconn.atomMatch(conflict) + if my_m_id != -1: + # check if the package shares the same slot + match_data = dbconn.retrieveKeySlot(m_id) + installed_match_data = self.Client.clientDbconn.retrieveKeySlot(my_m_id) + if match_data != installed_match_data: + found_conflicts.add(my_m_id) + return found_conflicts + + def is_match_masked(self, match, live_check = True): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) + if idpackage != -1: + return False + return True + + def is_match_masked_by_user(self, match, live_check = True): + # (query_status,masked?,) + m_id, m_repo = match + if m_repo not in self.Client.validRepositories: return False + dbconn = self.Client.open_repository(m_repo) + idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) + if idpackage != -1: return False #,False + myr = self.SystemSettings['pkg_masking_reference'] + user_masks = [myr['user_package_mask'],myr['user_license_mask'],myr['user_live_mask']] + if idreason in user_masks: + return True #,True + return False #,True + + def is_match_unmasked_by_user(self, match, live_check = True): + # (query_status,unmasked?,) + m_id, m_repo = match + if m_repo not in self.Client.validRepositories: return False + dbconn = self.Client.open_repository(m_repo) + idpackage, idreason = dbconn.idpackageValidator(m_id, live = live_check) + if idpackage == -1: return False #,False + myr = self.SystemSettings['pkg_masking_reference'] + user_masks = [ + myr['user_package_unmask'],myr['user_live_unmask'],myr['user_package_keywords'], + myr['user_repo_package_keywords_all'], myr['user_repo_package_keywords'] + ] + if idreason in user_masks: + return True #,True + return False #,True + + def mask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): + if self.is_match_masked(match, live_check = False): return True + methods = { + 'atom': self.mask_match_by_atom, + 'keyslot': self.mask_match_by_keyslot, + } + rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) + if dry_run: # inject if done "live" + self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) + self.SystemSettings['live_packagemasking']['mask_matches'].add(match) + return rc + + def unmask_match(self, match, method = 'atom', dry_run = False, clean_all_cache = False): + if not self.is_match_masked(match, live_check = False): return True + methods = { + 'atom': self.unmask_match_by_atom, + 'keyslot': self.unmask_match_by_keyslot, + } + rc = self._mask_unmask_match(match, method, methods, dry_run = dry_run, clean_all_cache = clean_all_cache) + if dry_run: # inject if done "live" + self.SystemSettings['live_packagemasking']['unmask_matches'].add(match) + self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) + return rc + + def _mask_unmask_match(self, match, method, methods_reference, dry_run = False, clean_all_cache = False): + + f = methods_reference.get(method) + if not callable(f): + raise IncorrectParameter('IncorrectParameter: %s: %s' % (_("not a valid method"),method,) ) + + self.Cacher.sync(wait = True) + done = f(match, dry_run) + if done: self.SystemSettings.clear() + + # clear atomMatch cache anyway + if clean_all_cache and not dry_run: + self.Client.clear_dump_cache(etpCache['world_available']) + self.Client.clear_dump_cache(etpCache['world_update']) + + self.Client.clear_dump_cache(etpCache['check_package_update']) + self.Client.clear_dump_cache(etpCache['filter_satisfied_deps']) + self.Client.clear_dump_cache(self.Client.atomMatchCacheKey) + self.Client.clear_dump_cache(etpCache['dep_tree']) + self.Client.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],match[1],)) + self.Client.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],match[1],)) + + self.Client._package_match_validator_cache.clear() + return done + + def unmask_match_by_atom(self, match, dry_run = False): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + atom = dbconn.retrieveAtom(m_id) + return self.unmask_match_generic(match, atom, dry_run = dry_run) + + def unmask_match_by_keyslot(self, match, dry_run = False): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) + return self.unmask_match_generic(match, keyslot, dry_run = dry_run) + + def mask_match_by_atom(self, match, dry_run = False): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + atom = dbconn.retrieveAtom(m_id) + return self.mask_match_generic(match, atom, dry_run = dry_run) + + def mask_match_by_keyslot(self, match, dry_run = False): + m_id, m_repo = match + dbconn = self.Client.open_repository(m_repo) + keyslot = "%s:%s" % dbconn.retrieveKeySlot(m_id) + return self.mask_match_generic(match, keyslot, dry_run = dry_run) + + def unmask_match_generic(self, match, keyword, dry_run = False): + self.clear_match_mask(match, dry_run) + m_file = self.SystemSettings.etpSettingFiles['unmask'] + return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) + + def mask_match_generic(self, match, keyword, dry_run = False): + self.clear_match_mask(match, dry_run) + m_file = self.SystemSettings.etpSettingFiles['mask'] + return self._mask_unmask_match_generic(keyword, m_file, dry_run = dry_run) + + def _mask_unmask_match_generic(self, keyword, m_file, dry_run = False): + exist = False + if not os.path.isfile(m_file): + if not os.access(os.path.dirname(m_file),os.W_OK): + return False # cannot write + elif not os.access(m_file, os.W_OK): + return False + elif not dry_run: + exist = True + + if dry_run: + return True + + content = [] + if exist: + f = open(m_file,"r") + content = [x.strip() for x in f.readlines()] + f.close() + content.append(keyword) + m_file_tmp = m_file+".tmp" + f = open(m_file_tmp,"w") + for line in content: + f.write(line+"\n") + f.flush() + f.close() + shutil.move(m_file_tmp,m_file) + return True + + def clear_match_mask(self, match, dry_run = False): + masking_list = [self.SystemSettings.etpSettingFiles['mask'],self.SystemSettings.etpSettingFiles['unmask']] + return self._clear_match_generic(match, masking_list = masking_list, dry_run = dry_run) + + def _clear_match_generic(self, match, masking_list = [], dry_run = False): + + self.SystemSettings['live_packagemasking']['unmask_matches'].discard(match) + self.SystemSettings['live_packagemasking']['mask_matches'].discard(match) + + if dry_run: return + + for mask_file in masking_list: + if not (os.path.isfile(mask_file) and os.access(mask_file,os.W_OK)): continue + f = open(mask_file,"r") + newf = self.entropyTools.open_buffer() + line = f.readline() + while line: + line = line.strip() + if line.startswith("#"): + newf.write(line+"\n") + line = f.readline() + continue + elif not line: + newf.write("\n") + line = f.readline() + continue + mymatch = self.Client.atom_match(line, packagesFilter = False) + if mymatch == match: + line = f.readline() + continue + newf.write(line+"\n") + line = f.readline() + f.close() + tmpfile = mask_file+".w_tmp" + f = open(tmpfile,"w") + f.write(newf.getvalue()) + f.flush() + f.close() + newf.close() + shutil.move(tmpfile,mask_file)