Entropy/Client/Interface:
- became a subpackage containing split Client (EquoInterface) classes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@3155 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
+20
-4337
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,226 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
# DESCRIPTION:
|
||||
# Entropy Object Oriented Interface
|
||||
|
||||
Copyright (C) 2007-2009 Fabio Erculiani
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
'''
|
||||
from __future__ import with_statement
|
||||
import os
|
||||
from entropy.const import *
|
||||
from entropy.exceptions import *
|
||||
from entropy.output import red, darkred, darkgreen
|
||||
|
||||
class Cache:
|
||||
|
||||
def __init__(self, ClientInterface):
|
||||
from entropy.client.interfaces import Client
|
||||
if not isinstance(ClientInterface,Client):
|
||||
mytxt = _("A valid Client instance or subclass is needed")
|
||||
raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
|
||||
self.Client = ClientInterface
|
||||
self.Cacher = self.Client.Cacher
|
||||
self.updateProgress = self.Client.updateProgress
|
||||
self.FileUpdates = self.Client.FileUpdates
|
||||
self.entropyTools = self.Client.entropyTools
|
||||
|
||||
def validate_repositories_cache(self):
|
||||
# is the list of repos changed?
|
||||
cached = self.Cacher.pop(etpCache['repolist'])
|
||||
if cached == None:
|
||||
# invalidate matching cache
|
||||
try: self.repository_move_clear_cache()
|
||||
except IOError: pass
|
||||
elif isinstance(cached,tuple):
|
||||
difflist = [x for x in cached if x not in etpRepositoriesOrder]
|
||||
for repoid in difflist:
|
||||
try: self.repository_move_clear_cache(repoid)
|
||||
except IOError: pass
|
||||
self.store_repository_list_cache()
|
||||
|
||||
def store_repository_list_cache(self):
|
||||
self.Cacher.push(etpCache['repolist'],tuple(etpRepositoriesOrder), async = False)
|
||||
|
||||
def generate_cache(self, depcache = True, configcache = True, client_purge = True, install_queue = True):
|
||||
self.Cacher.stop()
|
||||
# clean first of all
|
||||
self.purge_cache(client_purge = client_purge)
|
||||
if depcache:
|
||||
self.do_depcache(do_install_queue = install_queue)
|
||||
if configcache:
|
||||
self.do_configcache()
|
||||
self.Cacher.start()
|
||||
|
||||
def do_configcache(self):
|
||||
self.updateProgress(
|
||||
darkred(_("Configuration files")),
|
||||
importance = 2,
|
||||
type = "warning"
|
||||
)
|
||||
self.updateProgress(
|
||||
red(_("Scanning hard disk")),
|
||||
importance = 1,
|
||||
type = "warning"
|
||||
)
|
||||
self.FileUpdates.scanfs(dcache = False, quiet = True)
|
||||
self.updateProgress(
|
||||
darkred(_("Cache generation complete.")),
|
||||
importance = 2,
|
||||
type = "info"
|
||||
)
|
||||
|
||||
def do_depcache(self, do_install_queue = True):
|
||||
|
||||
self.updateProgress(
|
||||
darkgreen(_("Resolving metadata")),
|
||||
importance = 1,
|
||||
type = "warning"
|
||||
)
|
||||
# we can barely ignore any exception from here
|
||||
# especially cases where client db does not exist
|
||||
try:
|
||||
update, remove, fine = self.Client.calculate_world_updates()
|
||||
del fine, remove
|
||||
if do_install_queue:
|
||||
self.Client.get_install_queue(update, False, False)
|
||||
self.Client.calculate_available_packages()
|
||||
except:
|
||||
pass
|
||||
|
||||
self.updateProgress(
|
||||
darkred(_("Dependencies cache filled.")),
|
||||
importance = 2,
|
||||
type = "warning"
|
||||
)
|
||||
|
||||
def purge_cache(self, showProgress = True, client_purge = True):
|
||||
if self.entropyTools.is_user_in_entropy_group():
|
||||
skip = set()
|
||||
if not client_purge:
|
||||
skip.add("/"+etpCache['dbMatch']+"/"+etpConst['clientdbid']) # it's ok this way
|
||||
skip.add("/"+etpCache['dbSearch']+"/"+etpConst['clientdbid']) # it's ok this way
|
||||
for key in etpCache:
|
||||
if showProgress:
|
||||
self.Client.updateProgress(
|
||||
darkred(_("Cleaning %s => dumps...")) % (etpCache[key],),
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
back = True
|
||||
)
|
||||
self.clear_dump_cache(etpCache[key], skip = skip)
|
||||
|
||||
if showProgress:
|
||||
self.updateProgress(
|
||||
darkgreen(_("Cache is now empty.")),
|
||||
importance = 2,
|
||||
type = "info"
|
||||
)
|
||||
|
||||
def clear_dump_cache(self, dump_name, skip = []):
|
||||
self.Cacher.sync(wait = True)
|
||||
dump_path = os.path.join(etpConst['dumpstoragedir'],dump_name)
|
||||
dump_dir = os.path.dirname(dump_path)
|
||||
#dump_file = os.path.basename(dump_path)
|
||||
for currentdir, subdirs, files in os.walk(dump_dir):
|
||||
path = os.path.join(dump_dir,currentdir)
|
||||
if skip:
|
||||
found = False
|
||||
for myskip in skip:
|
||||
if path.find(myskip) != -1:
|
||||
found = True
|
||||
break
|
||||
if found: continue
|
||||
for item in files:
|
||||
if item.endswith(etpConst['cachedumpext']):
|
||||
item = os.path.join(path,item)
|
||||
try: os.remove(item)
|
||||
except OSError: pass
|
||||
try:
|
||||
if not os.listdir(path):
|
||||
os.rmdir(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def update_ugc_cache(self, repository):
|
||||
if not self.Client.UGC.is_repository_eapi3_aware(repository):
|
||||
return None
|
||||
status = True
|
||||
|
||||
votes_dict, err_msg = self.Client.UGC.get_all_votes(repository)
|
||||
if isinstance(votes_dict,dict):
|
||||
self.Client.UGC.UGCCache.save_vote_cache(repository, votes_dict)
|
||||
else:
|
||||
status = False
|
||||
|
||||
downloads_dict, err_msg = self.Client.UGC.get_all_downloads(repository)
|
||||
if isinstance(downloads_dict,dict):
|
||||
self.Client.UGC.UGCCache.save_downloads_cache(repository, downloads_dict)
|
||||
else:
|
||||
status = False
|
||||
return status
|
||||
|
||||
def repository_move_clear_cache(self, repoid = None):
|
||||
self.clear_dump_cache(etpCache['world_available'])
|
||||
self.clear_dump_cache(etpCache['world_update'])
|
||||
self.clear_dump_cache(etpCache['check_package_update'])
|
||||
self.clear_dump_cache(etpCache['filter_satisfied_deps'])
|
||||
self.clear_dump_cache(self.Client.atomMatchCacheKey)
|
||||
self.clear_dump_cache(etpCache['dep_tree'])
|
||||
if repoid != None:
|
||||
self.clear_dump_cache("%s/%s%s/" % (etpCache['dbMatch'],etpConst['dbnamerepoprefix'],repoid,))
|
||||
self.clear_dump_cache("%s/%s%s/" % (etpCache['dbSearch'],etpConst['dbnamerepoprefix'],repoid,))
|
||||
|
||||
def get_available_packages_chash(self, branch):
|
||||
# client digest not needed, cache is kept updated
|
||||
return str(hash("%s%s%s" % (
|
||||
self.all_repositories_checksum(),
|
||||
branch,self.Client.validRepositories,)))
|
||||
|
||||
def all_repositories_checksum(self):
|
||||
sum_hashes = ''
|
||||
for repo in self.Client.validRepositories:
|
||||
try:
|
||||
dbconn = self.Client.open_repository(repo)
|
||||
except (RepositoryError):
|
||||
continue # repo not available
|
||||
try:
|
||||
sum_hashes += dbconn.database_checksum()
|
||||
except self.Client.dbapi2.OperationalError:
|
||||
pass
|
||||
return sum_hashes
|
||||
|
||||
def get_available_packages_cache(self, branch = etpConst['branch'], myhash = None):
|
||||
if myhash == None: myhash = self.get_available_packages_chash(branch)
|
||||
return self.Cacher.pop("%s%s" % (etpCache['world_available'],myhash))
|
||||
|
||||
def get_world_update_cache(self, empty_deps, branch = etpConst['branch'], db_digest = None, ignore_spm_downgrades = False):
|
||||
if self.Client.xcache:
|
||||
if db_digest == None: db_digest = self.all_repositories_checksum()
|
||||
c_hash = "%s%s" % (etpCache['world_update'],
|
||||
self.get_world_update_cache_hash(db_digest, empty_deps, branch, ignore_spm_downgrades),)
|
||||
disk_cache = self.Cacher.pop(c_hash)
|
||||
if disk_cache != None:
|
||||
try:
|
||||
return disk_cache['r']
|
||||
except (KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def get_world_update_cache_hash(self, db_digest, empty_deps, branch, ignore_spm_downgrades):
|
||||
return str(hash("%s|%s|%s|%s|%s|%s" % (
|
||||
db_digest,empty_deps,self.Client.validRepositories,
|
||||
etpRepositoriesOrder, branch, ignore_spm_downgrades,
|
||||
)))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,493 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
# DESCRIPTION:
|
||||
# Entropy Object Oriented Interface
|
||||
|
||||
Copyright (C) 2007-2009 Fabio Erculiani
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
'''
|
||||
from __future__ import with_statement
|
||||
import os
|
||||
from entropy.i18n import _
|
||||
from entropy.const import *
|
||||
from entropy.exceptions import *
|
||||
from entropy.output import purple, bold, red, blue, darkgreen, darkred, brown, darkblue
|
||||
|
||||
class Fetchers:
|
||||
|
||||
def __init__(self, ClientInterface):
|
||||
from entropy.client.interfaces import Client
|
||||
if not isinstance(ClientInterface,Client):
|
||||
mytxt = _("A valid Client instance or subclass is needed")
|
||||
raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
|
||||
self.Client = ClientInterface
|
||||
self.entropyTools = self.Client.entropyTools
|
||||
self.updateProgress = self.Client.updateProgress
|
||||
self.SystemSettings = self.Client.SystemSettings
|
||||
self.Cacher = self.Client.Cacher
|
||||
self.MirrorStatus = self.Client.MirrorStatus
|
||||
|
||||
def check_needed_package_download(self, filepath, checksum = None):
|
||||
# is the file available
|
||||
if os.path.isfile(etpConst['entropyworkdir']+"/"+filepath):
|
||||
if checksum is None:
|
||||
return 0
|
||||
else:
|
||||
# check digest
|
||||
md5res = self.entropyTools.compareMd5(etpConst['entropyworkdir']+"/"+filepath,checksum)
|
||||
if (md5res):
|
||||
return 0
|
||||
else:
|
||||
return -2
|
||||
else:
|
||||
return -1
|
||||
|
||||
def fetch_files(self, url_data_list, checksum = True, resume = True, fetch_file_abort_function = None):
|
||||
"""
|
||||
Fetch multiple files simultaneously on URLs.
|
||||
|
||||
@param url_data_list list
|
||||
[(url,dest_path [or None],checksum ['ab86fff46f6ec0f4b1e0a2a4a82bf323' or None],branch,),..]
|
||||
@param digest bool, digest check (checksum)
|
||||
@param resume bool enable resume support
|
||||
@param fetch_file_abort_function callable method that could raise exceptions
|
||||
@return general_status_code, {'url': (status_code,checksum,resumed,)}, data_transfer
|
||||
"""
|
||||
pkgs_bindir = etpConst['packagesbindir']
|
||||
url_path_list = []
|
||||
checksum_map = {}
|
||||
count = 0
|
||||
for url, dest_path, cksum, branch in url_data_list:
|
||||
count += 1
|
||||
filename = os.path.basename(url)
|
||||
if dest_path == None:
|
||||
dest_path = os.path.join(pkgs_bindir,branch,filename,)
|
||||
|
||||
dest_dir = os.path.dirname(dest_path)
|
||||
if not os.path.isdir(dest_dir):
|
||||
os.makedirs(dest_dir,0755)
|
||||
|
||||
url_path_list.append((url,dest_path,))
|
||||
if cksum != None: checksum_map[count] = cksum
|
||||
|
||||
# load class
|
||||
fetchConn = self.Client.MultipleUrlFetcher(url_path_list, resume = resume,
|
||||
abort_check_func = fetch_file_abort_function, OutputInterface = self,
|
||||
urlFetcherClass = self.Client.urlFetcher, checksum = checksum)
|
||||
try:
|
||||
data = fetchConn.download()
|
||||
except KeyboardInterrupt:
|
||||
return -100, {}, 0
|
||||
|
||||
diff_map = {}
|
||||
if checksum_map and checksum: # verify checksums
|
||||
diff_map = dict((url_path_list[x-1][0],checksum_map.get(x)) for x in checksum_map \
|
||||
if checksum_map.get(x) != data.get(x))
|
||||
|
||||
data_transfer = fetchConn.get_data_transfer()
|
||||
if diff_map:
|
||||
defval = -1
|
||||
for key, val in diff_map.items():
|
||||
if val == "-1": # general error
|
||||
diff_map[key] = -1
|
||||
elif val == "-2":
|
||||
diff_map[key] = -2
|
||||
elif val == "-4": # timeout
|
||||
diff_map[key] = -4
|
||||
elif val == "-3": # not found
|
||||
diff_map[key] = -3
|
||||
elif val == -100:
|
||||
defval = -100
|
||||
return defval, diff_map, data_transfer
|
||||
|
||||
return 0, diff_map, data_transfer
|
||||
|
||||
def fetch_files_on_mirrors(self, download_list, checksum = False, fetch_abort_function = None):
|
||||
"""
|
||||
@param download_map list [(repository,branch,filename,checksum (digest),),..]
|
||||
@param checksum bool verify checksum?
|
||||
@param fetch_abort_function callable method that could raise exceptions
|
||||
"""
|
||||
repo_uris = dict(((x[0],etpRepositories[x[0]]['packages'][::-1],) for x in download_list))
|
||||
remaining = repo_uris.copy()
|
||||
my_download_list = download_list[:]
|
||||
|
||||
def get_best_mirror(repository):
|
||||
try:
|
||||
return remaining[repository][0]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def update_download_list(down_list, failed_down):
|
||||
newlist = []
|
||||
for repo,branch,fname,cksum in down_list:
|
||||
myuri = get_best_mirror(repo)
|
||||
myuri = os.path.join(myuri,fname)
|
||||
if myuri not in failed_down:
|
||||
continue
|
||||
newlist.append((repo,branch,fname,cksum,))
|
||||
return newlist
|
||||
|
||||
# return True: for failing, return False: for fine
|
||||
def mirror_fail_check(repository, best_mirror):
|
||||
# check if uri is sane
|
||||
if not self.MirrorStatus.get_failing_mirror_status(best_mirror) >= 30:
|
||||
return False
|
||||
# set to 30 for convenience
|
||||
self.MirrorStatus.set_failing_mirror_status(best_mirror, 30)
|
||||
mirrorcount = repo_uris[repo].index(best_mirror)+1
|
||||
mytxt = "( mirror #%s ) " % (mirrorcount,)
|
||||
mytxt += blue(" %s: ") % (_("Mirror"),)
|
||||
mytxt += red(self.entropyTools.spliturl(best_mirror)[1])
|
||||
mytxt += " - %s." % (_("maximum failure threshold reached"),)
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
if self.MirrorStatus.get_failing_mirror_status(best_mirror) == 30:
|
||||
self.MirrorStatus.add_failing_mirror(best_mirror,45)
|
||||
elif self.MirrorStatus.get_failing_mirror_status(best_mirror) > 31:
|
||||
self.MirrorStatus.add_failing_mirror(best_mirror,-4)
|
||||
else:
|
||||
self.MirrorStatus.set_failing_mirror_status(best_mirror, 0)
|
||||
|
||||
remaining[repository].discard(best_mirror)
|
||||
return True
|
||||
|
||||
def show_download_summary(down_list):
|
||||
# fetch_files_list.append((myuri,None,cksum,branch,))
|
||||
for repo, branch, fname, cksum in down_list:
|
||||
best_mirror = get_best_mirror(repo)
|
||||
mirrorcount = repo_uris[repo].index(best_mirror)+1
|
||||
mytxt = "( mirror #%s ) " % (mirrorcount,)
|
||||
basef = os.path.basename(fname)
|
||||
mytxt += "[%s] %s " % (brown(basef),blue("@"),)
|
||||
mytxt += red(self.entropyTools.spliturl(best_mirror)[1])
|
||||
# now fetch the new one
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
def show_successful_download(down_list, data_transfer):
|
||||
for repo, branch, fname, cksum in down_list:
|
||||
best_mirror = get_best_mirror(repo)
|
||||
mirrorcount = repo_uris[repo].index(best_mirror)+1
|
||||
mytxt = "( mirror #%s ) " % (mirrorcount,)
|
||||
basef = os.path.basename(fname)
|
||||
mytxt += "[%s] %s %s " % (brown(basef),darkred(_("success")),blue("@"),)
|
||||
mytxt += red(self.entropyTools.spliturl(best_mirror)[1])
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = red(" ## ")
|
||||
)
|
||||
mytxt = " %s: %s%s%s" % (
|
||||
blue(_("Aggregated transfer rate")),
|
||||
bold(self.entropyTools.bytesIntoHuman(data_transfer)),
|
||||
darkred("/"),
|
||||
darkblue(_("second")),
|
||||
)
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
def show_download_error(down_list, rc):
|
||||
for repo, branch, fname, cksum in down_list:
|
||||
best_mirror = get_best_mirror(repo)
|
||||
mirrorcount = repo_uris[repo].index(best_mirror)+1
|
||||
mytxt = "( mirror #%s ) " % (mirrorcount,)
|
||||
mytxt += blue("%s: %s") % (
|
||||
_("Error downloading from"),
|
||||
red(self.entropyTools.spliturl(best_mirror)[1]),
|
||||
)
|
||||
if rc == -1:
|
||||
mytxt += " - %s." % (_("files not available on this mirror"),)
|
||||
elif rc == -2:
|
||||
self.MirrorStatus.add_failing_mirror(best_mirror,1)
|
||||
mytxt += " - %s." % (_("wrong checksum"),)
|
||||
elif rc == -3:
|
||||
mytxt += " - %s." % (_("not found"),)
|
||||
elif rc == -4: # timeout!
|
||||
mytxt += " - %s." % (_("timeout error"),)
|
||||
elif rc == -100:
|
||||
mytxt += " - %s." % (_("discarded download"),)
|
||||
else:
|
||||
self.MirrorStatus.add_failing_mirror(best_mirror, 5)
|
||||
mytxt += " - %s." % (_("unknown reason"),)
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
def remove_failing_mirrors(repos):
|
||||
for repo in repos:
|
||||
best_mirror = get_best_mirror(repo)
|
||||
if remaining[repo]:
|
||||
remaining[repo].pop(0)
|
||||
|
||||
def check_remaining_mirror_failure(repos):
|
||||
return [x for x in repos if not remaining.get(x)]
|
||||
|
||||
while 1:
|
||||
|
||||
do_resume = True
|
||||
timeout_try_count = 50
|
||||
while 1:
|
||||
|
||||
fetch_files_list = []
|
||||
for repo, branch, fname, cksum in my_download_list:
|
||||
best_mirror = get_best_mirror(repo)
|
||||
if best_mirror != None:
|
||||
mirror_fail_check(repo, best_mirror)
|
||||
best_mirror = get_best_mirror(repo)
|
||||
if best_mirror == None:
|
||||
# at least one package failed to download
|
||||
# properly, give up with everything
|
||||
return 3, my_download_list
|
||||
myuri = os.path.join(best_mirror,fname)
|
||||
fetch_files_list.append((myuri,None,cksum,branch,))
|
||||
|
||||
try:
|
||||
|
||||
show_download_summary(my_download_list)
|
||||
rc, failed_downloads, data_transfer = self.fetch_files(
|
||||
fetch_files_list, checksum = checksum,
|
||||
fetch_file_abort_function = fetch_abort_function,
|
||||
resume = do_resume
|
||||
)
|
||||
if rc == 0:
|
||||
show_successful_download(my_download_list, data_transfer)
|
||||
return 0, []
|
||||
|
||||
# update my_download_list
|
||||
my_download_list = update_download_list(my_download_list,failed_downloads)
|
||||
if rc not in (-3,-4,-100,) and failed_downloads and do_resume:
|
||||
# disable resume
|
||||
do_resume = False
|
||||
continue
|
||||
else:
|
||||
show_download_error(my_download_list, rc)
|
||||
if rc == -4: # timeout
|
||||
timeout_try_count -= 1
|
||||
if timeout_try_count > 0:
|
||||
continue
|
||||
elif rc == -100: # user discarded fetch
|
||||
return 1, []
|
||||
myrepos = set([x[0] for x in my_download_list])
|
||||
remove_failing_mirrors(myrepos)
|
||||
# make sure we don't have nasty issues
|
||||
remaining_failure = check_remaining_mirror_failure(myrepos)
|
||||
if remaining_failure:
|
||||
return 3, my_download_list
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
return 1, []
|
||||
return 0, []
|
||||
|
||||
|
||||
def fetch_file(self, url, branch, digest = None, resume = True, fetch_file_abort_function = None, filepath = None):
|
||||
|
||||
filename = os.path.basename(url)
|
||||
if not filepath:
|
||||
filepath = os.path.join(etpConst['packagesbindir'],branch,filename)
|
||||
filepath_dir = os.path.dirname(filepath)
|
||||
if not os.path.isdir(filepath_dir):
|
||||
os.makedirs(filepath_dir,0755)
|
||||
|
||||
# load class
|
||||
fetchConn = self.Client.urlFetcher(url, filepath, resume = resume,
|
||||
abort_check_func = fetch_file_abort_function, OutputInterface = self)
|
||||
fetchConn.progress = self.Client.progress
|
||||
|
||||
# start to download
|
||||
data_transfer = 0
|
||||
resumed = False
|
||||
try:
|
||||
fetchChecksum = fetchConn.download()
|
||||
data_transfer = fetchConn.get_transfer_rate()
|
||||
resumed = fetchConn.is_resumed()
|
||||
except KeyboardInterrupt:
|
||||
return -100, data_transfer, resumed
|
||||
except NameError:
|
||||
raise
|
||||
except:
|
||||
if etpUi['debug']:
|
||||
self.updateProgress(
|
||||
"fetch_file:",
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
self.entropyTools.printTraceback()
|
||||
return -1, data_transfer, resumed
|
||||
if fetchChecksum == "-3":
|
||||
# not found
|
||||
return -3, data_transfer, resumed
|
||||
elif fetchChecksum == "-4":
|
||||
# timeout
|
||||
return -4, data_transfer, resumed
|
||||
|
||||
del fetchConn
|
||||
if digest:
|
||||
if fetchChecksum != digest:
|
||||
# not properly downloaded
|
||||
return -2, data_transfer, resumed
|
||||
else:
|
||||
return 0, data_transfer, resumed
|
||||
return 0, data_transfer, resumed
|
||||
|
||||
|
||||
def fetch_file_on_mirrors(self, repository, branch, filename,
|
||||
digest = False, fetch_abort_function = None):
|
||||
|
||||
uris = etpRepositories[repository]['packages'][::-1]
|
||||
remaining = set(uris)
|
||||
|
||||
mirrorcount = 0
|
||||
for uri in uris:
|
||||
|
||||
if not remaining:
|
||||
# tried all the mirrors, quitting for error
|
||||
return 3
|
||||
|
||||
mirrorcount += 1
|
||||
mirrorCountText = "( mirror #%s ) " % (mirrorcount,)
|
||||
url = uri+"/"+filename
|
||||
|
||||
# check if uri is sane
|
||||
if self.MirrorStatus.get_failing_mirror_status(uri) >= 30:
|
||||
# ohohoh!
|
||||
# set to 30 for convenience
|
||||
self.MirrorStatus.set_failing_mirror_status(uri, 30)
|
||||
mytxt = mirrorCountText
|
||||
mytxt += blue(" %s: ") % (_("Mirror"),)
|
||||
mytxt += red(self.entropyTools.spliturl(uri)[1])
|
||||
mytxt += " - %s." % (_("maximum failure threshold reached"),)
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
if self.MirrorStatus.get_failing_mirror_status(uri) == 30:
|
||||
# put to 75 then decrement by 4 so we
|
||||
# won't reach 30 anytime soon ahahaha
|
||||
self.MirrorStatus.add_failing_mirror(uri,45)
|
||||
elif self.MirrorStatus.get_failing_mirror_status(uri) > 31:
|
||||
# now decrement each time this point is reached,
|
||||
# if will be back < 30, then equo will try to use it again
|
||||
self.MirrorStatus.add_failing_mirror(uri,-4)
|
||||
else:
|
||||
# put to 0 - reenable mirror, welcome back uri!
|
||||
self.MirrorStatus.set_failing_mirror_status(uri, 0)
|
||||
|
||||
remaining.discard(uri)
|
||||
continue
|
||||
|
||||
do_resume = True
|
||||
timeout_try_count = 50
|
||||
while 1:
|
||||
try:
|
||||
mytxt = mirrorCountText
|
||||
mytxt += blue("%s: ") % (_("Downloading from"),)
|
||||
mytxt += red(self.entropyTools.spliturl(uri)[1])
|
||||
# now fetch the new one
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
rc, data_transfer, resumed = self.fetch_file(
|
||||
url,
|
||||
branch,
|
||||
digest,
|
||||
do_resume,
|
||||
fetch_file_abort_function = fetch_abort_function
|
||||
)
|
||||
if rc == 0:
|
||||
mytxt = mirrorCountText
|
||||
mytxt += blue("%s: ") % (_("Successfully downloaded from"),)
|
||||
mytxt += red(self.entropyTools.spliturl(uri)[1])
|
||||
mytxt += " %s %s/%s" % (_("at"),self.entropyTools.bytesIntoHuman(data_transfer),_("second"),)
|
||||
self.updateProgress(
|
||||
mytxt,
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = red(" ## ")
|
||||
)
|
||||
|
||||
return 0
|
||||
elif resumed and rc not in (-3,-4,-100,):
|
||||
do_resume = False
|
||||
continue
|
||||
else:
|
||||
error_message = mirrorCountText
|
||||
error_message += blue("%s: %s") % (
|
||||
_("Error downloading from"),
|
||||
red(self.entropyTools.spliturl(uri)[1]),
|
||||
)
|
||||
# something bad happened
|
||||
if rc == -1:
|
||||
error_message += " - %s." % (_("file not available on this mirror"),)
|
||||
elif rc == -2:
|
||||
self.MirrorStatus.add_failing_mirror(uri,1)
|
||||
error_message += " - %s." % (_("wrong checksum"),)
|
||||
elif rc == -3:
|
||||
error_message += " - %s." % (_("not found"),)
|
||||
elif rc == -4: # timeout!
|
||||
timeout_try_count -= 1
|
||||
if timeout_try_count > 0:
|
||||
error_message += " - %s." % (_("timeout, retrying on this mirror"),)
|
||||
else:
|
||||
error_message += " - %s." % (_("timeout, giving up"),)
|
||||
elif rc == -100:
|
||||
error_message += " - %s." % (_("discarded download"),)
|
||||
else:
|
||||
self.MirrorStatus.add_failing_mirror(uri, 5)
|
||||
error_message += " - %s." % (_("unknown reason"),)
|
||||
self.updateProgress(
|
||||
error_message,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = red(" ## ")
|
||||
)
|
||||
if rc == -4: # timeout
|
||||
if timeout_try_count > 0:
|
||||
continue
|
||||
elif rc == -100: # user discarded fetch
|
||||
return 1
|
||||
remaining.discard(uri)
|
||||
# make sure we don't have nasty issues
|
||||
if not remaining:
|
||||
return 3
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
return 1
|
||||
return 0
|
||||
@@ -0,0 +1,88 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
# DESCRIPTION:
|
||||
# Entropy Object Oriented Interface
|
||||
|
||||
Copyright (C) 2007-2009 Fabio Erculiani
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
'''
|
||||
from __future__ import with_statement
|
||||
from entropy.const import *
|
||||
from entropy.exceptions import *
|
||||
|
||||
class Loaders:
|
||||
|
||||
def __init__(self, ClientInterface):
|
||||
from entropy.client.interfaces import Client
|
||||
from entropy.client.interfaces import Trigger
|
||||
from entropy.client.interfaces import Repository
|
||||
from entropy.client.interfaces import Package
|
||||
self.__PackageLoader = Package
|
||||
self.__RepositoryLoader = Repository
|
||||
self.__TriggerLoader = Trigger
|
||||
if not isinstance(ClientInterface,Client):
|
||||
mytxt = _("A valid Client instance or subclass is needed")
|
||||
raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
|
||||
self.Client = ClientInterface
|
||||
self.__QA_cache = {}
|
||||
self.__security_cache = {}
|
||||
self.__spm_cache = {}
|
||||
|
||||
def closeAllQA(self):
|
||||
self.__QA_cache.clear()
|
||||
|
||||
def closeAllSecurity(self):
|
||||
self.__security_cache.clear()
|
||||
|
||||
def Security(self):
|
||||
chroot = etpConst['systemroot']
|
||||
cached = self.Client.__security_cache.get(chroot)
|
||||
if cached != None:
|
||||
return cached
|
||||
from entropy.security import SecurityInterface
|
||||
cached = SecurityInterface(self)
|
||||
self.Client.__security_cache[chroot] = cached
|
||||
return cached
|
||||
|
||||
def QA(self):
|
||||
chroot = etpConst['systemroot']
|
||||
cached = self.__QA_cache.get(chroot)
|
||||
if cached != None:
|
||||
return cached
|
||||
from entropy.qa import QAInterface
|
||||
cached = QAInterface(self)
|
||||
self.__QA_cache[chroot] = cached
|
||||
return cached
|
||||
|
||||
def Triggers(self, *args, **kwargs):
|
||||
return self.__TriggerLoader(self, *args, **kwargs)
|
||||
|
||||
def Repositories(self, reponames = [], forceUpdate = False, noEquoCheck = False, fetchSecurity = True):
|
||||
return self.__RepositoryLoader(self, reponames = reponames,
|
||||
forceUpdate = forceUpdate, noEquoCheck = noEquoCheck,
|
||||
fetchSecurity = fetchSecurity)
|
||||
|
||||
def Spm(self):
|
||||
from entropy.spm import Spm
|
||||
myroot = etpConst['systemroot']
|
||||
cached = self.__spm_cache.get(myroot)
|
||||
if cached != None: return cached
|
||||
conn = Spm(self)
|
||||
self.__spm_cache[myroot] = conn.intf
|
||||
return conn.intf
|
||||
|
||||
def Package(self):
|
||||
return self.__PackageLoader(self)
|
||||
@@ -0,0 +1,490 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
'''
|
||||
# DESCRIPTION:
|
||||
# Entropy Object Oriented Interface
|
||||
|
||||
Copyright (C) 2007-2009 Fabio Erculiani
|
||||
|
||||
This program is free software; you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation; either version 2 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program; if not, write to the Free Software
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
'''
|
||||
from __future__ import with_statement
|
||||
import os
|
||||
import shutil
|
||||
from entropy.i18n import _
|
||||
from entropy.const import *
|
||||
from entropy.exceptions import *
|
||||
from entropy.output import red, bold, brown
|
||||
|
||||
class Extractors:
|
||||
|
||||
def __init__(self, ClientInterface):
|
||||
from entropy.client.interfaces import Client
|
||||
if not isinstance(ClientInterface,Client):
|
||||
mytxt = _("A valid Client instance or subclass is needed")
|
||||
raise IncorrectParameter("IncorrectParameter: %s" % (mytxt,))
|
||||
self.Client = ClientInterface
|
||||
self.entropyTools = self.Client.entropyTools
|
||||
self.updateProgress = self.Client.updateProgress
|
||||
self.SystemSettings = self.Client.SystemSettings
|
||||
|
||||
def _extract_pkg_metadata_generate_extraction_dict(self):
|
||||
data = {
|
||||
'chost': {
|
||||
'path': etpConst['spm']['xpak_entries']['chost'],
|
||||
'critical': True,
|
||||
},
|
||||
'description': {
|
||||
'path': etpConst['spm']['xpak_entries']['description'],
|
||||
'critical': False,
|
||||
},
|
||||
'homepage': {
|
||||
'path': etpConst['spm']['xpak_entries']['homepage'],
|
||||
'critical': False,
|
||||
},
|
||||
'slot': {
|
||||
'path': etpConst['spm']['xpak_entries']['slot'],
|
||||
'critical': False,
|
||||
},
|
||||
'cflags': {
|
||||
'path': etpConst['spm']['xpak_entries']['cflags'],
|
||||
'critical': False,
|
||||
},
|
||||
'cxxflags': {
|
||||
'path': etpConst['spm']['xpak_entries']['cxxflags'],
|
||||
'critical': False,
|
||||
},
|
||||
'category': {
|
||||
'path': etpConst['spm']['xpak_entries']['category'],
|
||||
'critical': True,
|
||||
},
|
||||
'rdepend': {
|
||||
'path': etpConst['spm']['xpak_entries']['rdepend'],
|
||||
'critical': False,
|
||||
},
|
||||
'pdepend': {
|
||||
'path': etpConst['spm']['xpak_entries']['pdepend'],
|
||||
'critical': False,
|
||||
},
|
||||
'depend': {
|
||||
'path': etpConst['spm']['xpak_entries']['depend'],
|
||||
'critical': False,
|
||||
},
|
||||
'use': {
|
||||
'path': etpConst['spm']['xpak_entries']['use'],
|
||||
'critical': False,
|
||||
},
|
||||
'iuse': {
|
||||
'path': etpConst['spm']['xpak_entries']['iuse'],
|
||||
'critical': False,
|
||||
},
|
||||
'license': {
|
||||
'path': etpConst['spm']['xpak_entries']['license'],
|
||||
'critical': False,
|
||||
},
|
||||
'provide': {
|
||||
'path': etpConst['spm']['xpak_entries']['provide'],
|
||||
'critical': False,
|
||||
},
|
||||
'sources': {
|
||||
'path': etpConst['spm']['xpak_entries']['src_uri'],
|
||||
'critical': False,
|
||||
},
|
||||
'eclasses': {
|
||||
'path': etpConst['spm']['xpak_entries']['inherited'],
|
||||
'critical': False,
|
||||
},
|
||||
'counter': {
|
||||
'path': etpConst['spm']['xpak_entries']['counter'],
|
||||
'critical': False,
|
||||
},
|
||||
'keywords': {
|
||||
'path': etpConst['spm']['xpak_entries']['keywords'],
|
||||
'critical': False,
|
||||
},
|
||||
}
|
||||
return data
|
||||
|
||||
def _extract_pkg_metadata_content(self, content_file, package_path):
|
||||
|
||||
pkg_content = {}
|
||||
|
||||
if os.path.isfile(content_file):
|
||||
f = open(content_file,"r")
|
||||
content = f.readlines()
|
||||
f.close()
|
||||
outcontent = set()
|
||||
for line in content:
|
||||
line = line.strip().split()
|
||||
try:
|
||||
datatype = line[0]
|
||||
datafile = line[1:]
|
||||
if datatype == 'obj':
|
||||
datafile = datafile[:-2]
|
||||
datafile = ' '.join(datafile)
|
||||
elif datatype == 'dir':
|
||||
datafile = ' '.join(datafile)
|
||||
elif datatype == 'sym':
|
||||
datafile = datafile[:-3]
|
||||
datafile = ' '.join(datafile)
|
||||
else:
|
||||
myexc = "InvalidData: %s %s. %s." % (
|
||||
datafile,
|
||||
_("not supported"),
|
||||
_("Probably Portage API has changed"),
|
||||
)
|
||||
raise InvalidData(myexc)
|
||||
outcontent.add((datafile,datatype))
|
||||
except:
|
||||
pass
|
||||
|
||||
_outcontent = set()
|
||||
for i in outcontent:
|
||||
i = list(i)
|
||||
datatype = i[1]
|
||||
_outcontent.add((i[0],i[1]))
|
||||
outcontent = sorted(_outcontent)
|
||||
for i in outcontent:
|
||||
pkg_content[i[0]] = i[1]
|
||||
|
||||
else:
|
||||
|
||||
# CONTENTS is not generated when a package is emerged with portage and the option -B
|
||||
# we have to unpack the tbz2 and generate content dict
|
||||
mytempdir = etpConst['packagestmpdir']+"/"+os.path.basename(package_path)+".inject"
|
||||
if os.path.isdir(mytempdir):
|
||||
shutil.rmtree(mytempdir)
|
||||
if not os.path.isdir(mytempdir):
|
||||
os.makedirs(mytempdir)
|
||||
|
||||
self.entropyTools.uncompressTarBz2(package_path, extractPath = mytempdir, catchEmpty = True)
|
||||
for currentdir, subdirs, files in os.walk(mytempdir):
|
||||
pkg_content[currentdir[len(mytempdir):]] = "dir"
|
||||
for item in files:
|
||||
item = currentdir+"/"+item
|
||||
if os.path.islink(item):
|
||||
pkg_content[item[len(mytempdir):]] = "sym"
|
||||
else:
|
||||
pkg_content[item[len(mytempdir):]] = "obj"
|
||||
|
||||
# now remove
|
||||
shutil.rmtree(mytempdir,True)
|
||||
try: os.rmdir(mytempdir)
|
||||
except (OSError,): pass
|
||||
|
||||
return pkg_content
|
||||
|
||||
def _extract_pkg_metadata_needed(self, needed_file):
|
||||
|
||||
pkg_needed = set()
|
||||
lines = []
|
||||
|
||||
try:
|
||||
f = open(needed_file,"r")
|
||||
lines = [x.strip() for x in f.readlines() if x.strip()]
|
||||
f.close()
|
||||
except IOError:
|
||||
return lines
|
||||
|
||||
for line in lines:
|
||||
needed = line.split()
|
||||
if len(needed) == 2:
|
||||
ownlib = needed[0]
|
||||
ownelf = -1
|
||||
if os.access(ownlib,os.R_OK):
|
||||
ownelf = self.entropyTools.read_elf_class(ownlib)
|
||||
for lib in needed[1].split(","):
|
||||
#if lib.find(".so") != -1:
|
||||
pkg_needed.add((lib,ownelf))
|
||||
|
||||
return list(pkg_needed)
|
||||
|
||||
def _extract_pkg_metadata_messages(self, log_dir, category, name, version, silent = False):
|
||||
|
||||
pkg_messages = []
|
||||
|
||||
if os.path.isdir(log_dir):
|
||||
|
||||
elogfiles = os.listdir(log_dir)
|
||||
myelogfile = "%s:%s-%s" % (category, name, version,)
|
||||
foundfiles = [x for x in elogfiles if x.startswith(myelogfile)]
|
||||
if foundfiles:
|
||||
elogfile = foundfiles[0]
|
||||
if len(foundfiles) > 1:
|
||||
# get the latest
|
||||
mtimes = []
|
||||
for item in foundfiles: mtimes.append((self.entropyTools.getFileUnixMtime(os.path.join(log_dir,item)),item))
|
||||
mtimes = sorted(mtimes)
|
||||
elogfile = mtimes[-1][1]
|
||||
messages = self.entropyTools.extractElog(os.path.join(log_dir,elogfile))
|
||||
for message in messages:
|
||||
message = message.replace("emerge","install")
|
||||
pkg_messages.append(message)
|
||||
|
||||
elif not silent:
|
||||
|
||||
mytxt = " %s, %s" % (_("not set"),_("have you configured make.conf properly?"),)
|
||||
self.updateProgress(
|
||||
red(log_dir)+mytxt,
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = brown(" * ")
|
||||
)
|
||||
|
||||
return pkg_messages
|
||||
|
||||
def _extract_pkg_metadata_license_data(self, licenses_dir, license_string):
|
||||
|
||||
pkg_licensedata = {}
|
||||
if licenses_dir and os.path.isdir(licenses_dir):
|
||||
licdata = [x.strip() for x in license_string.split() if x.strip() and self.entropyTools.is_valid_string(x.strip())]
|
||||
for mylicense in licdata:
|
||||
licfile = os.path.join(licenses_dir,mylicense)
|
||||
if os.access(licfile,os.R_OK):
|
||||
if self.entropyTools.istextfile(licfile):
|
||||
f = open(licfile)
|
||||
pkg_licensedata[mylicense] = f.read()
|
||||
f.close()
|
||||
|
||||
return pkg_licensedata
|
||||
|
||||
def _extract_pkg_metadata_mirror_links(self, Spm, sources_list):
|
||||
|
||||
# =mirror://openoffice|link1|link2|link3
|
||||
pkg_links = []
|
||||
for i in sources_list:
|
||||
if i.startswith("mirror://"):
|
||||
# parse what mirror I need
|
||||
mirrorURI = i.split("/")[2]
|
||||
mirrorlist = Spm.get_third_party_mirrors(mirrorURI)
|
||||
pkg_links.append([mirrorURI,mirrorlist])
|
||||
# mirrorURI = openoffice and mirrorlist = [link1, link2, link3]
|
||||
|
||||
return pkg_links
|
||||
|
||||
def _extract_pkg_metadata_ebuild_entropy_tag(self, ebuild):
|
||||
search_tag = etpConst['spm']['ebuild_pkg_tag_var']
|
||||
ebuild_tag = ''
|
||||
f = open(ebuild,"r")
|
||||
tags = [x.strip() for x in f.readlines() if x.strip() and x.strip().startswith(search_tag)]
|
||||
f.close()
|
||||
if not tags: return ebuild_tag
|
||||
tag = tags[-1]
|
||||
tag = tag.split("=")[-1].strip('"').strip("'").strip()
|
||||
return tag
|
||||
|
||||
# This function extracts all the info from a .tbz2 file and returns them
|
||||
def extract_pkg_metadata(self, package, etpBranch = etpConst['branch'], silent = False, inject = False):
|
||||
|
||||
data = {}
|
||||
info_package = bold(os.path.basename(package))+": "
|
||||
|
||||
if not silent:
|
||||
self.updateProgress(
|
||||
red(info_package+_("Extracting package metadata")+" ..."),
|
||||
importance = 0,
|
||||
type = "info",
|
||||
header = brown(" * "),
|
||||
back = True
|
||||
)
|
||||
|
||||
filepath = package
|
||||
tbz2File = package
|
||||
package = package.split(etpConst['packagesext'])[0]
|
||||
package = self.entropyTools.remove_entropy_revision(package)
|
||||
package = self.entropyTools.remove_tag(package)
|
||||
# remove entropy category
|
||||
if package.find(":") != -1:
|
||||
package = ':'.join(package.split(":")[1:])
|
||||
|
||||
# pkgcat is always == "null" here
|
||||
pkgcat, pkgname, pkgver, pkgrev = self.entropyTools.catpkgsplit(os.path.basename(package))
|
||||
if pkgrev != "r0": pkgver += "-%s" % (pkgrev,)
|
||||
|
||||
# Fill Package name and version
|
||||
data['name'] = pkgname
|
||||
data['version'] = pkgver
|
||||
data['digest'] = self.entropyTools.md5sum(tbz2File)
|
||||
data['datecreation'] = str(self.entropyTools.getFileUnixMtime(tbz2File))
|
||||
data['size'] = str(self.entropyTools.get_file_size(tbz2File))
|
||||
|
||||
tbz2TmpDir = etpConst['packagestmpdir']+"/"+data['name']+"-"+data['version']+"/"
|
||||
if not os.path.isdir(tbz2TmpDir):
|
||||
if os.path.lexists(tbz2TmpDir):
|
||||
os.remove(tbz2TmpDir)
|
||||
os.makedirs(tbz2TmpDir)
|
||||
self.entropyTools.extractXpak(tbz2File,tbz2TmpDir)
|
||||
|
||||
data['injected'] = False
|
||||
if inject: data['injected'] = True
|
||||
data['branch'] = etpBranch
|
||||
|
||||
portage_entries = self._extract_pkg_metadata_generate_extraction_dict()
|
||||
for item in portage_entries:
|
||||
value = ''
|
||||
try:
|
||||
f = open(os.path.join(tbz2TmpDir,portage_entries[item]['path']),"r")
|
||||
value = f.readline().strip()
|
||||
f.close()
|
||||
except IOError:
|
||||
if portage_entries[item]['critical']:
|
||||
raise
|
||||
data[item] = value
|
||||
|
||||
# setup vars
|
||||
data['eclasses'] = data['eclasses'].split()
|
||||
try:
|
||||
data['counter'] = int(data['counter'])
|
||||
except ValueError:
|
||||
data['counter'] = -2 # -2 values will be insterted as incremental negative values into the database
|
||||
data['keywords'] = [x.strip() for x in data['keywords'].split() if x.strip()]
|
||||
if not data['keywords']: data['keywords'].insert(0,"") # support for packages with no keywords
|
||||
needed_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['needed'])
|
||||
data['needed'] = self._extract_pkg_metadata_needed(needed_file)
|
||||
content_file = os.path.join(tbz2TmpDir,etpConst['spm']['xpak_entries']['contents'])
|
||||
data['content'] = self._extract_pkg_metadata_content(content_file, filepath)
|
||||
data['disksize'] = self.entropyTools.sum_file_sizes(data['content'])
|
||||
|
||||
# [][][] Kernel dependent packages hook [][][]
|
||||
data['versiontag'] = ''
|
||||
kernelstuff = False
|
||||
kernelstuff_kernel = False
|
||||
for item in data['content']:
|
||||
if item.startswith("/lib/modules/"):
|
||||
kernelstuff = True
|
||||
# get the version of the modules
|
||||
kmodver = item.split("/lib/modules/")[1]
|
||||
kmodver = kmodver.split("/")[0]
|
||||
|
||||
lp = kmodver.split("-")[-1]
|
||||
if lp.startswith("r"):
|
||||
kname = kmodver.split("-")[-2]
|
||||
kver = kmodver.split("-")[0]+"-"+kmodver.split("-")[-1]
|
||||
else:
|
||||
kname = kmodver.split("-")[-1]
|
||||
kver = kmodver.split("-")[0]
|
||||
break
|
||||
# validate the results above
|
||||
if kernelstuff:
|
||||
matchatom = "linux-%s-%s" % (kname,kver,)
|
||||
if (matchatom == data['name']+"-"+data['version']):
|
||||
kernelstuff_kernel = True
|
||||
|
||||
data['versiontag'] = kmodver
|
||||
if not kernelstuff_kernel:
|
||||
data['slot'] = kmodver # if you change this behaviour,
|
||||
# you must change "reagent update"
|
||||
# and "equo database gentoosync" consequentially
|
||||
|
||||
file_ext = etpConst['spm']['ebuild_file_extension']
|
||||
ebuilds_in_path = [x for x in os.listdir(tbz2TmpDir) if x.endswith(".%s" % (file_ext,))]
|
||||
if not data['versiontag'] and ebuilds_in_path:
|
||||
# has the user specified a custom package tag inside the ebuild
|
||||
ebuild_path = os.path.join(tbz2TmpDir,ebuilds_in_path[0])
|
||||
data['versiontag'] = self._extract_pkg_metadata_ebuild_entropy_tag(ebuild_path)
|
||||
|
||||
|
||||
data['download'] = etpConst['packagesrelativepath'] + data['branch'] + "/"
|
||||
data['download'] += self.entropyTools.create_package_filename(data['category'], data['name'], data['version'], data['versiontag'])
|
||||
|
||||
|
||||
data['trigger'] = ""
|
||||
if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']):
|
||||
f = open(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername'],"rb")
|
||||
data['trigger'] = f.read()
|
||||
f.close()
|
||||
|
||||
Spm = self.Client.Spm()
|
||||
|
||||
# Get Spm ChangeLog
|
||||
pkgatom = "%s/%s-%s" % (data['category'],data['name'],data['version'],)
|
||||
try:
|
||||
data['changelog'] = Spm.get_package_changelog(pkgatom)
|
||||
except:
|
||||
data['changelog'] = None
|
||||
|
||||
portage_metadata = Spm.calculate_dependencies(
|
||||
data['iuse'], data['use'], data['license'], data['depend'],
|
||||
data['rdepend'], data['pdepend'], data['provide'], data['sources']
|
||||
)
|
||||
|
||||
data['provide'] = portage_metadata['PROVIDE'].split()
|
||||
data['license'] = portage_metadata['LICENSE']
|
||||
data['useflags'] = []
|
||||
for x in data['use'].split():
|
||||
if x.startswith("+"):
|
||||
x = x[1:]
|
||||
elif x.startswith("-"):
|
||||
x = x[1:]
|
||||
if (x in portage_metadata['USE']) or (x in portage_metadata['USE_MASK']):
|
||||
data['useflags'].append(x)
|
||||
else:
|
||||
data['useflags'].append("-"+x)
|
||||
data['sources'] = portage_metadata['SRC_URI'].split()
|
||||
data['dependencies'] = {}
|
||||
for x in portage_metadata['RDEPEND'].split():
|
||||
if x.startswith("!") or (x in ("(","||",")","")):
|
||||
continue
|
||||
data['dependencies'][x] = etpConst['spm']['(r)depend_id']
|
||||
for x in portage_metadata['PDEPEND'].split():
|
||||
if x.startswith("!") or (x in ("(","||",")","")):
|
||||
continue
|
||||
data['dependencies'][x] = etpConst['spm']['pdepend_id']
|
||||
data['conflicts'] = [x[1:] for x in portage_metadata['RDEPEND'].split()+portage_metadata['PDEPEND'].split() if x.startswith("!") and not x in ("(","||",")","")]
|
||||
|
||||
if (kernelstuff) and (not kernelstuff_kernel):
|
||||
# add kname to the dependency
|
||||
data['dependencies']["=sys-kernel/linux-"+kname+"-"+kver+"~-1"] = etpConst['spm']['(r)depend_id']
|
||||
|
||||
# Conflicting tagged packages support
|
||||
key = data['category']+"/"+data['name']
|
||||
confl_data = self.SystemSettings['conflicting_tagged_packages'].get(key)
|
||||
if confl_data != None:
|
||||
for conflict in confl_data: data['conflicts'].append(conflict)
|
||||
|
||||
# Get License text if possible
|
||||
licenses_dir = os.path.join(Spm.get_spm_setting('PORTDIR'),'licenses')
|
||||
data['licensedata'] = self._extract_pkg_metadata_license_data(licenses_dir, data['license'])
|
||||
data['mirrorlinks'] = self._extract_pkg_metadata_mirror_links(Spm, data['sources'])
|
||||
|
||||
# write only if it's a systempackage
|
||||
data['systempackage'] = False
|
||||
system_packages = [self.entropyTools.dep_getkey(x) for x in Spm.get_atoms_in_system()]
|
||||
if data['category']+"/"+data['name'] in system_packages:
|
||||
data['systempackage'] = True
|
||||
|
||||
# write only if it's a systempackage
|
||||
protect, mask = Spm.get_config_protect_and_mask()
|
||||
data['config_protect'] = protect
|
||||
data['config_protect_mask'] = mask
|
||||
|
||||
log_dir = etpConst['logdir']+"/elog"
|
||||
if not os.path.isdir(log_dir): os.makedirs(log_dir)
|
||||
data['messages'] = self._extract_pkg_metadata_messages(log_dir, data['category'], data['name'], data['version'], silent = silent)
|
||||
data['etpapi'] = etpConst['etpapi']
|
||||
|
||||
# removing temporary directory
|
||||
shutil.rmtree(tbz2TmpDir,True)
|
||||
if os.path.isdir(tbz2TmpDir):
|
||||
try: os.remove(tbz2TmpDir)
|
||||
except OSError: pass
|
||||
|
||||
if not silent:
|
||||
self.updateProgress(
|
||||
red(info_package+_("Package extraction complete")), importance = 0,
|
||||
type = "info", header = brown(" * "), back = True
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user