sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
Files
69fb71f9578c6bfaeaddb5e16597edb77f9ec696
entropy/lib/entropy/client/interfaces/package.py
T
Fabio Erculiani 69fb71f957 [entropy.client.package] code style updates
2013-11-22 14:02:49 +01:00

5305 lines
195 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client Package Interface}.
"""
import sys
import os
import errno
import stat
import shutil
import time
import codecs
from entropy.const import etpConst, const_setup_perms, const_mkstemp, \
const_isunicode, const_convert_to_unicode, const_debug_write, \
const_debug_enabled, const_convert_to_rawstring, const_is_python3
from entropy.exceptions import PermissionDenied, SPMError
from entropy.i18n import _, ngettext
from entropy.output import brown, blue, bold, darkgreen, \
darkblue, red, purple, darkred, teal
from entropy.client.mirrors import StatusInterface
from entropy.core.settings.base import SystemSettings
from entropy.security import Repository as RepositorySecurity
from entropy.fetchers import UrlFetcher
import entropy.dep
import entropy.tools
class Package:
class FileContentReader:
def __init__(self, path, enc=None):
if enc is None:
self._enc = etpConst['conf_encoding']
else:
self._enc = enc
self._cpath = path
self._f = None
self._eof = False
def _open_f(self):
# opening the file in universal newline mode
# fixes the readline() issues wrt
# truncated lines.
if isinstance(self._cpath, int):
self._f = entropy.tools.codecs_fdopen(
self._cpath, "rU", self._enc)
else:
self._f = codecs.open(
self._cpath, "rU", self._enc)
def __iter__(self):
# reset object status, this makes possible
# to reuse the iterator more than once
# restarting from the beginning. It is really
# important for scenarios where transactions
# have to be rolled back and replayed.
self.close()
self._open_f()
# reset EOF status on each new iteration
self._eof = False
return self
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, tb):
self.close()
def __next__(self):
return self.next()
def next(self):
if self._eof:
raise StopIteration()
if self._f is None:
self._open_f()
line = self._f.readline()
if not line:
self.close()
self._eof = True
raise StopIteration()
# non-deterministic BUG with
# ca-certificates and Python crappy
# API causes readline() to return
# partial lines when non ASCII cruft
# is on the line. This is probably a
# Python bug.
# Example of partial readline():
# 0|obj|/usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_F\xc3\x85
# and the next call:
# \xc2\x91tan\xc3\x83\xc2\xbas\xc3\x83\xc2\xadtv\xc3\x83\xc2\xa1ny.crt\n
# Try to workaround it by reading ahead
# if line does not end with \n
# HOWEVER: opening the file in
# Universal Newline mode fixes it.
# But let's keep the check for QA.
# 2012-08-14: is has been observed that
# Universal Newline mode is not enough
# to avoid this issue.
while not line.endswith("\n"):
part_line = self._f.readline()
line += part_line
sys.stderr.write(
"FileContentReader, broken readline()"
", executing fixup code\n")
sys.stderr.write("%s\n" % (repr(part_line),))
# break infinite loops
# and let it crash
if not part_line: # EOF
break
_package_id, _ftype, _path = line[:-1].split("|", 2)
# must be legal or die!
_package_id = int(_package_id)
return _package_id, _path, _ftype
def close(self):
if self._f is not None:
self._f.close()
self._f = None
class FileContentWriter:
TMP_SUFFIX = "__filter_tmp"
def __init__(self, path, enc=None):
self._f = None
if enc is None:
self._enc = etpConst['conf_encoding']
else:
self._enc = enc
self._cpath = path
# callers expect that file is created
# on open object instantiation, don't
# remove this or things like os.rename()
# will fail
self._open_f()
def _open_f(self):
if isinstance(self._cpath, int):
self._f = entropy.tools.codecs_fdopen(
self._cpath, "w", self._enc)
else:
self._f = codecs.open(
self._cpath, "w", self._enc)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, tb):
self.close()
def write(self, package_id, path, ftype):
if self._f is None:
self._open_f()
if package_id is not None:
self._f.write(str(package_id))
else:
self._f.write("0")
self._f.write("|")
self._f.write(ftype)
self._f.write("|")
self._f.write(path)
self._f.write("\n")
def close(self):
if self._f is not None:
self._f.flush()
self._f.close()
self._f = None
class FileContentSafetyWriter:
def __init__(self, path, enc=None):
self._f = None
if enc is None:
self._enc = etpConst['conf_encoding']
else:
self._enc = enc
self._cpath = path
# callers expect that file is created
# on open object instantiation, don't
# remove this or things like os.rename()
# will fail
self._open_f()
def _open_f(self):
if isinstance(self._cpath, int):
self._f = entropy.tools.codecs_fdopen(
self._cpath, "w", self._enc)
else:
self._f = codecs.open(
self._cpath, "w", self._enc)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, tb):
self.close()
def write(self, path, sha256, mtime):
if self._f is None:
self._open_f()
self._f.write("%f" % (mtime,))
self._f.write("|")
self._f.write(sha256)
self._f.write("|")
self._f.write(path)
self._f.write("\n")
def close(self):
if self._f is not None:
self._f.flush()
self._f.close()
self._f = None
class FileContentSafetyReader:
def __init__(self, path, enc=None):
if enc is None:
self._enc = etpConst['conf_encoding']
else:
self._enc = enc
self._cpath = path
self._f = None
self._eof = False
def _open_f(self):
# opening the file in universal newline mode
# fixes the readline() issues wrt
# truncated lines.
if isinstance(self._cpath, int):
self._f = entropy.tools.codecs_fdopen(
self._cpath, "rU", self._enc)
else:
self._f = codecs.open(
self._cpath, "rU", self._enc)
def __iter__(self):
# reset object status, this makes possible
# to reuse the iterator more than once
# restarting from the beginning. It is really
# important for scenarios where transactions
# have to be rolled back and replayed.
self.close()
self._open_f()
# reset EOF status on each new iteration
self._eof = False
return self
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, tb):
self.close()
def __next__(self):
return self.next()
def next(self):
if self._eof:
raise StopIteration()
if self._f is None:
self._open_f()
line = self._f.readline()
if not line:
self.close()
self._eof = True
raise StopIteration()
# non-deterministic BUG with
# ca-certificates and Python crappy
# API causes readline() to return
# partial lines when non ASCII cruft
# is on the line. This is probably a
# Python bug.
# Example of partial readline():
# 0|obj|/usr/share/ca-certificates/mozilla/NetLock_Arany_=Class_Gold=_F\xc3\x85
# and the next call:
# \xc2\x91tan\xc3\x83\xc2\xbas\xc3\x83\xc2\xadtv\xc3\x83\xc2\xa1ny.crt\n
# Try to workaround it by reading ahead
# if line does not end with \n
# HOWEVER: opening the file in
# Universal Newline mode fixes it.
# But let's keep the check for QA.
# 2012-08-14: is has been observed that
# Universal Newline mode is not enough
# to avoid this issue.
while not line.endswith("\n"):
part_line = self._f.readline()
line += part_line
sys.stderr.write(
"FileContentReader, broken readline()"
", executing fixup code\n")
sys.stderr.write("%s\n" % (repr(part_line),))
# break infinite loops
# and let it crash
if not part_line: # EOF
break
_mtime, _sha256, _path = line[:-1].split("|", 2)
# must be legal or die!
_mtime = float(_mtime)
return _path, _sha256, _mtime
def close(self):
if self._f is not None:
self._f.close()
self._f = None
def __init__(self, entropy_client):
self._entropy = entropy_client
self._settings = SystemSettings()
self.pkgmeta = {}
self.__prepared = False
self._package_match = ()
self._valid_actions = ("source", "fetch", "multi_fetch", "remove",
"remove_conflict", "install", "config"
)
self._action = None
self._xterm_title = ''
def __repr__(self):
return "<%s.Package at %s | metadata: %s | action: %s, prepared: %s>" \
% (__name__, hex(id(self)), self.pkgmeta, self._action,
self.__prepared)
def __str__(self):
return repr(self)
def __unicode__(self):
return unicode(repr(self))
def kill(self):
# remove temporary content files
# created by __generate_content_file()
content_files = self.pkgmeta.get(
'__content_files__', [])
for content_file in content_files:
try:
os.remove(content_file)
except (OSError, IOError):
pass
self.pkgmeta.clear()
self._package_match = ()
self._valid_actions = ()
self._action = None
self.__prepared = False
def _error_on_prepared(self):
if self.__prepared:
mytxt = _("Already prepared")
raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
def _error_on_not_prepared(self):
if not self.__prepared:
mytxt = _("Not yet prepared")
raise PermissionDenied("PermissionDenied: %s" % (mytxt,))
def _check_action_validity(self, action):
if action not in self._valid_actions:
raise AttributeError("Action must be in %s" % (
self._valid_actions,))
_INFO_EXTS = (
const_convert_to_unicode(".gz"),
const_convert_to_unicode(".bz2")
)
def _get_info_directories(self):
"""
Return a list of `info` directories as declared in the
INFOPATH and INFODIR environment variable.
"""
info_dirs = os.getenv("INFOPATH", "").split(":")
info_dirs += os.getenv("INFODIR", "").split(":")
info_dirs = [const_convert_to_unicode(
os.path.normpath(x)) for x in info_dirs]
info_dirs.sort()
return info_dirs
@staticmethod
def get_standard_fetch_disk_path(download):
"""
Return standard path where package is going to be downloaded.
"download" argument passed must come from
EntropyRepository.retrieveDownloadURL()
"""
return os.path.join(etpConst['entropypackagesworkdir'], download)
def __get_fetch_disk_path(self, download):
"""
Return proper Entropy package store path
"""
if 'fetch_path' in self.pkgmeta:
# only supported by fetch action, multifetch also unsupported
pkg_disk_path = os.path.join(self.pkgmeta['fetch_path'],
os.path.basename(download))
else:
pkg_disk_path = Package.get_standard_fetch_disk_path(download)
return pkg_disk_path
def __escape_path(self, path):
"""
Some applications (like ld) don't like ":" in path, others just don't
escape paths at all. So, it's better to avoid to use field separators
in path.
"""
path = path.replace(":", "_")
path = path.replace("~", "_")
return path
def __check_pkg_path_download(self, download, checksum = None):
# is the file available
pkg_path = self.__get_fetch_disk_path(download)
if os.path.isfile(pkg_path):
if checksum is None:
return 0
# check digest
md5res = entropy.tools.compare_md5(pkg_path, checksum)
if md5res:
return 0
return -2
return -1
def __fetch_files(self, url_data_list, checksum = True, resume = True):
def _generate_checksum_map(url_data):
if not checksum:
return {}
ck_map = {}
ck_map_id = 0
for _pkg_id, _repo, _url, _dest_path, cksum in url_data:
ck_map_id += 1
if cksum is not None:
ck_map[ck_map_id] = cksum
return ck_map
fetch_abort_function = self.pkgmeta.get('fetch_abort_function')
# avoid tainting data pointed by url_data_list
url_data = url_data_list[:]
diff_map = {}
# setup directories
for pkg_id, repo, url, dest_path, _cksum in url_data:
dest_dir = os.path.dirname(dest_path)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir, 0o775)
const_setup_perms(dest_dir, etpConst['entropygid'])
checksum_map = _generate_checksum_map(url_data)
fetched_url_data, data_transfer, abort = self.__try_edelta_multifetch(
url_data, resume)
if abort:
return -100, {}, 0
for url_data_item in fetched_url_data:
url_data.remove(url_data_item)
# some packages haven't been downloaded using edelta
if url_data:
url_path_list = []
for pkg_id, repo, url, dest_path, _cksum in url_data:
url_path_list.append((url, dest_path,))
self._setup_differential_download(
self._entropy._multiple_url_fetcher, url, resume, dest_path,
repo, pkg_id)
# load class
fetch_intf = self._entropy._multiple_url_fetcher(url_path_list,
resume = resume, abort_check_func = fetch_abort_function,
url_fetcher_class = self._entropy._url_fetcher,
checksum = checksum)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
data = fetch_intf.download()
except KeyboardInterrupt:
return -100, {}, 0
# update transfer rate information
data_transfer = fetch_intf.get_transfer_rate()
checksum_map = _generate_checksum_map(url_data)
# if checksum_map is empty, it means that checksum == False
for ck_id in checksum_map:
orig_checksum = checksum_map.get(ck_id)
if orig_checksum != data.get(ck_id):
diff_map[url_path_list[ck_id-1][0]] = orig_checksum
if diff_map:
defval = -1
for key, val in tuple(diff_map.items()):
if val == UrlFetcher.GENERIC_FETCH_WARN:
diff_map[key] = -2
elif val == UrlFetcher.TIMEOUT_FETCH_ERROR:
diff_map[key] = -4
elif val == UrlFetcher.GENERIC_FETCH_ERROR:
diff_map[key] = -3
elif val == -100:
defval = -100
return defval, diff_map, data_transfer
return 0, diff_map, data_transfer
def _get_url_name(self, url):
"""
Given a mirror URL, returns a smaller string representing the URL name.
@param url: URL string
@type url: string
@return: representative URL string
@rtype: string
"""
url_data = entropy.tools.spliturl(url)
url_name = url_data.netloc
url_scheme = url_data.scheme
if not url_scheme:
url_scheme = "unknown"
return "%s://%s" % (url_scheme, url_name,)
def _download_packages(self, download_list, checksum = False):
avail_data = self._settings['repositories']['available']
excluded_data = self._settings['repositories']['excluded']
repo_uris = {}
for pkg_id, repo, fname, cksum, _signatures in download_list:
repo_db = self._entropy.open_repository(repo)
# grab original repo, if any and use it if available
# this is done in order to support "equo repo merge" feature
# allowing client-side repository package metadata moves.
original_repo = repo_db.getInstalledPackageRepository(pkg_id)
if (original_repo != repo) and (original_repo not in avail_data) \
and (original_repo is not None):
# build up a new uris list, at least try, hoping that
# repository is just shadowing original_repo
# for example: original_repo got copied to repository, without
# copying packages, which would be useless. like it happens
# with sabayon-weekly
uris = self.__build_uris_list(original_repo, repo)
else:
if original_repo in avail_data:
uris = avail_data[original_repo]['packages'][::-1]
uris += avail_data[repo]['packages'][::-1]
elif original_repo in excluded_data:
uris = excluded_data[original_repo]['packages'][::-1]
uris += avail_data[repo]['packages'][::-1]
else:
uris = avail_data[repo]['packages'][::-1]
obj = repo_uris.setdefault(repo, [])
# append at the beginning
new_ones = [x for x in uris if x not in obj][::-1]
for new_obj in new_ones:
obj.insert(0, new_obj)
remaining = repo_uris.copy()
my_download_list = download_list[:]
mirror_status = StatusInterface()
def get_best_mirror(repository):
try:
return remaining[repository][0]
except IndexError:
return None
def update_download_list(down_list, failed_down):
newlist = []
for pkg_id, repo, fname, cksum, signatures in down_list:
myuri = get_best_mirror(repo)
myuri = os.path.join(myuri, fname)
if myuri not in failed_down:
continue
newlist.append((pkg_id, repo, fname, cksum, signatures,))
return newlist
# return True: for failing, return False: for fine
def mirror_fail_check(repository, best_mirror):
# check if uri is sane
if not mirror_status.get_failing_mirror_status(best_mirror) >= 30:
return False
# set to 30 for convenience
mirror_status.set_failing_mirror_status(best_mirror, 30)
mirrorcount = repo_uris[repository].index(best_mirror)+1
mytxt = "( mirror #%s ) " % (mirrorcount,)
mytxt += blue(" %s: ") % (_("Mirror"),)
mytxt += red(self._get_url_name(best_mirror))
mytxt += " - %s." % (_("maximum failure threshold reached"),)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
if mirror_status.get_failing_mirror_status(best_mirror) == 30:
mirror_status.add_failing_mirror(best_mirror, 45)
elif mirror_status.get_failing_mirror_status(best_mirror) > 31:
mirror_status.add_failing_mirror(best_mirror, -4)
else:
mirror_status.set_failing_mirror_status(best_mirror, 0)
try:
remaining[repository].remove(best_mirror)
except ValueError:
# ignore
pass
return True
def show_download_summary(down_list):
for _pkg_id, repo, fname, _cksum, _signatures in down_list:
best_mirror = get_best_mirror(repo)
mirrorcount = repo_uris[repo].index(best_mirror)+1
mytxt = "( mirror #%s ) " % (mirrorcount,)
basef = os.path.basename(fname)
mytxt += "[%s] %s " % (brown(basef), blue("@"),)
mytxt += red(self._get_url_name(best_mirror))
# now fetch the new one
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
def show_successful_download(down_list, data_transfer):
for _pkg_id, repo, fname, _cksum, _signatures in down_list:
best_mirror = get_best_mirror(repo)
mirrorcount = repo_uris[repo].index(best_mirror)+1
mytxt = "( mirror #%s ) " % (mirrorcount,)
basef = os.path.basename(fname)
mytxt += "[%s] %s %s " % (brown(basef),
darkred(_("success")), blue("@"),)
mytxt += red(self._get_url_name(best_mirror))
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
mytxt = " %s: %s%s%s" % (
blue(_("Aggregated transfer rate")),
bold(entropy.tools.bytes_into_human(data_transfer)),
darkred("/"),
darkblue(_("second")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
def show_download_error(down_list, rc):
for _pkg_id, repo, _fname, _cksum, _signatures in down_list:
best_mirror = get_best_mirror(repo)
mirrorcount = repo_uris[repo].index(best_mirror)+1
mytxt = "( mirror #%s ) " % (mirrorcount,)
mytxt += blue("%s: %s") % (
_("Error downloading from"),
red(self._get_url_name(best_mirror)),
)
if rc == -1:
mytxt += " - %s." % (
_("data not available on this mirror"),)
elif rc == -2:
mirror_status.add_failing_mirror(best_mirror, 1)
mytxt += " - %s." % (_("wrong checksum"),)
elif rc == -3:
mytxt += " - %s." % (_("not found"),)
elif rc == -4: # timeout!
mytxt += " - %s." % (_("timeout error"),)
elif rc == -100:
mytxt += " - %s." % (_("discarded download"),)
else:
mirror_status.add_failing_mirror(best_mirror, 5)
mytxt += " - %s." % (_("unknown reason"),)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
def remove_failing_mirrors(repos):
for repo in repos:
get_best_mirror(repo)
if remaining[repo]:
remaining[repo].pop(0)
def check_remaining_mirror_failure(repos):
return [x for x in repos if not remaining.get(x)]
while True:
do_resume = True
timeout_try_count = 50
while True:
fetch_files_list = []
for pkg_id, repo, fname, cksum, _signatures in my_download_list:
best_mirror = get_best_mirror(repo)
# set working mirror, dont care if its None
mirror_status.set_working_mirror(best_mirror)
if best_mirror is not None:
mirror_fail_check(repo, best_mirror)
best_mirror = get_best_mirror(repo)
if best_mirror is None:
# at least one package failed to download
# properly, give up with everything
return 3, my_download_list
myuri = os.path.join(best_mirror, fname)
pkg_path = Package.get_standard_fetch_disk_path(fname)
fetch_files_list.append(
(pkg_id, repo, myuri, pkg_path, cksum,))
try:
show_download_summary(my_download_list)
rc, failed_downloads, data_transfer = self.__fetch_files(
fetch_files_list, checksum = checksum,
resume = do_resume
)
if rc == 0:
show_successful_download(my_download_list,
data_transfer)
return 0, []
# update my_download_list
my_download_list = update_download_list(my_download_list,
failed_downloads)
if rc not in (-3, -4, -100,) and failed_downloads and \
do_resume:
# disable resume
do_resume = False
continue
else:
show_download_error(my_download_list, rc)
if rc == -4: # timeout
timeout_try_count -= 1
if timeout_try_count > 0:
continue
elif rc == -100: # user discarded fetch
return 1, []
myrepos = set([x[1] for x in my_download_list])
remove_failing_mirrors(myrepos)
# make sure we don't have nasty issues
remaining_failure = check_remaining_mirror_failure(
myrepos)
if remaining_failure:
return 3, my_download_list
break
except KeyboardInterrupt:
return 1, []
return 0, []
def _setup_differential_download(self, fetcher, url, resume,
fetch_path, repository, package_id):
"""
Setup differential download in case of URL supporting it.
Internal function.
@param fetcher: UrlFetcher or MultipleUrlFetcher class
@param url: URL to check differential download against
@type url: string
@param resume: resume support
@type resume: bool
@param fetch_path: path where package file will be saved
@type fetch_path: string
@param repository: repository identifier belonging to package file
@type repository: string
@param package_id: package identifier belonging to repository identifier
@type package_id: int
"""
# no resume? no party?
if not resume:
const_debug_write(__name__,
"_setup_differential_download(%s) %s" % (
url, "resume disabled"))
return
if not fetcher.supports_differential_download(url):
# no differential download support
const_debug_write(__name__,
"_setup_differential_download(%s) %s" % (
url, "no differential download support"))
return
# this is the fetch path of the file that is going to be downloaded
# not going to overwrite it if a file is located there because
# user would want a resume for sure in first place
if os.path.isfile(fetch_path):
const_debug_write(__name__,
"_setup_differential_download(%s) %s" % (
url, "fetch path already exists, not overwriting"))
return
pkg_repo = self._entropy.open_repository(repository)
keyslot = pkg_repo.retrieveKeySlot(package_id)
if keyslot is None:
# fucked up entry, not dealing with it here
const_debug_write(__name__,
"_setup_differential_download(%s) %s" % (
url, "key, slot fucked up"))
return
key, slot = keyslot
inst_repo = self._entropy.installed_repository()
pkg_ids = inst_repo.searchKeySlot(key, slot)
if not pkg_ids:
# not installed, nothing to use as diff download
const_debug_write(__name__,
"_setup_differential_download(%s) %s" % (
url, "no installed packages"))
return
# grab the highest, we don't know if user was able to mess up
# its installed packages repository
pkg_id = max(pkg_ids)
download_url = inst_repo.retrieveDownloadURL(pkg_id)
installed_fetch_path = self.__get_fetch_disk_path(download_url)
try:
shutil.copyfile(installed_fetch_path, fetch_path)
except (OSError, IOError, shutil.Error) as err:
const_debug_write(
__name__,
"_setup_differential_download(%s), copyfile error: %s" % (
url, err))
try:
os.remove(fetch_path)
except OSError:
pass
return
try:
user = os.stat(installed_fetch_path)[stat.ST_UID]
group = os.stat(installed_fetch_path)[stat.ST_GID]
os.chown(fetch_path, user, group)
except (OSError, IOError) as err:
const_debug_write(
__name__,
"_setup_differential_download(%s), chown error: %s" % (
url, err))
return
try:
shutil.copystat(installed_fetch_path, fetch_path)
except (OSError, IOError, shutil.Error) as err:
const_debug_write(
__name__,
"_setup_differential_download(%s), copystat error: %s" % (
url, err))
return
const_debug_write(
__name__,
"_setup_differential_download(%s) copied to %s" % (
url, fetch_path))
def __approve_edelta(self, url, installed_package_id, package_digest):
"""
Approve Entropy package delta support for given url, checking if
a previously fetched package is available.
@return: edelta URL to download and previously downloaded package path
or None if edelta is not available
@rtype: tuple of strings or None
"""
inst_repo = self._entropy.installed_repository()
download_url = inst_repo.retrieveDownloadURL(installed_package_id)
installed_digest = inst_repo.retrieveDigest(installed_package_id)
installed_fetch_path = self.__get_fetch_disk_path(download_url)
edelta_local_approved = False
try:
edelta_local_approved = entropy.tools.compare_md5(
installed_fetch_path, installed_digest)
except (OSError, IOError) as err:
const_debug_write(
__name__, "__approve_edelta, error: %s" % (err,))
return
if edelta_local_approved:
hash_tag = installed_digest + package_digest
edelta_file_name = entropy.tools.generate_entropy_delta_file_name(
os.path.basename(download_url),
os.path.basename(url),
hash_tag)
edelta_url = os.path.join(
os.path.dirname(url),
etpConst['packagesdeltasubdir'],
edelta_file_name)
return edelta_url, installed_fetch_path
def __try_edelta_multifetch(self, url_data, resume):
# no edelta support enabled
if not self.pkgmeta.get('edelta_support'):
return [], 0.0, False
# edelta enabled?
if not entropy.tools.is_entropy_delta_available():
return [], 0.0, False
url_path_list = []
url_data_map = {}
url_data_map_idx = 0
for pkg_id, repo, url, dest_path, cksum in url_data:
repo_db = self._entropy.open_repository(repo)
if cksum is None:
# cannot setup edelta without checksum, get from repository
cksum = repo_db.retrieveDigest(pkg_id)
if cksum is None:
# still nothing
continue
key_slot = repo_db.retrieveKeySlot(pkg_id)
if key_slot is None:
# wtf corrupted entry, skip
continue
pkg_key, pkg_slot = key_slot
installed_package_id = self.__setup_package_to_remove(pkg_key,
pkg_slot)
if installed_package_id == -1:
# package is not installed
continue
edelta_approve = self.__approve_edelta(url, installed_package_id,
cksum)
if edelta_approve is None:
# no edelta support
continue
edelta_url, installed_fetch_path = edelta_approve
edelta_save_path = dest_path + etpConst['packagesdeltaext']
key = (edelta_url, edelta_save_path)
url_path_list.append(key)
url_data_map_idx += 1
url_data_map[url_data_map_idx] = (pkg_id, repo, url, dest_path,
cksum, edelta_url, edelta_save_path, installed_fetch_path)
if not url_path_list:
# no martini, no party!
return [], 0.0, False
fetch_abort_function = self.pkgmeta.get('fetch_abort_function')
fetch_intf = self._entropy._multiple_url_fetcher(url_path_list,
resume = resume, abort_check_func = fetch_abort_function,
url_fetcher_class = self._entropy._url_fetcher)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
data = fetch_intf.download()
except KeyboardInterrupt:
return [], 0.0, True
data_transfer = fetch_intf.get_transfer_rate()
fetch_errors = (
UrlFetcher.TIMEOUT_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_WARN,
)
valid_idxs = []
for url_data_map_idx, cksum in tuple(data.items()):
if cksum in fetch_errors:
# download failed
continue
pkg_id, repo, url, dest_path, orig_cksum, edelta_url, \
edelta_save_path, installed_fetch_path = \
url_data_map[url_data_map_idx]
# now check
tmp_dest_path = dest_path + ".edelta_pkg_tmp"
# yay, we can apply the delta and cook the new package file!
try:
entropy.tools.apply_entropy_delta(installed_fetch_path,
edelta_save_path, tmp_dest_path)
except IOError:
# give up with this edelta
try:
os.remove(tmp_dest_path)
except (OSError, IOError):
pass
continue
os.rename(tmp_dest_path, dest_path)
valid_idxs.append(url_data_map_idx)
# now check md5
fetched_url_data = []
for url_data_map_idx in valid_idxs:
pkg_id, repo, url, dest_path, orig_cksum, edelta_url, \
edelta_save_path, installed_fetch_path = \
url_data_map[url_data_map_idx]
try:
valid = entropy.tools.compare_md5(dest_path, orig_cksum)
except (IOError, OSError):
valid = False
if valid:
url_data_item = (pkg_id, repo, url, dest_path, orig_cksum)
fetched_url_data.append(url_data_item)
return fetched_url_data, data_transfer, False
def __try_edelta_fetch(self, url, save_path, resume):
# no edelta support enabled
if not self.pkgmeta.get('edelta_support'):
return 1, 0.0
# edelta enabled?
if not entropy.tools.is_entropy_delta_available():
return 1, 0.0
# when called by __fetch_file, which is called by _download_package
# which is called by _match_checksum, which is called by
# multi_match_checksum, removepackage_id metadatum is not available
# So, be fault tolerant.
installed_package_id = self.pkgmeta.get('removepackage_id', -1)
# fresh install, cannot fetch edelta, edelta only works for installed
# packages, by design.
if installed_package_id == -1:
return 1, 0.0
edelta_approve = self.__approve_edelta(url, installed_package_id,
self.pkgmeta['checksum'])
if edelta_approve is None:
# edelta not available, give up
return 1, 0.0
edelta_url, installed_fetch_path = edelta_approve
# check if edelta file is available online
edelta_save_path = save_path + etpConst['packagesdeltaext']
max_tries = 2
edelta_approved = False
data_transfer = 0
download_plan = [(edelta_url, edelta_save_path) for _x in \
range(max_tries)]
delta_resume = resume
fetch_abort_function = self.pkgmeta.get('fetch_abort_function')
fetch_errors = (
UrlFetcher.TIMEOUT_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_WARN,
)
for delta_url, delta_save in download_plan:
delta_fetcher = self._entropy._url_fetcher(delta_url,
delta_save, resume = delta_resume,
abort_check_func = fetch_abort_function)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
delta_checksum = delta_fetcher.download()
data_transfer = delta_fetcher.get_transfer_rate()
del delta_fetcher
except KeyboardInterrupt:
return -100, data_transfer
except NameError:
raise
except Exception:
return -1, data_transfer
if delta_checksum in fetch_errors:
# make sure this points to hell
delta_resume = False
# retry
continue
# now check
tmp_save_path = save_path + ".edelta_pkg_tmp"
# yay, we can apply the delta and cook the new package file!
try:
entropy.tools.apply_entropy_delta(installed_fetch_path,
delta_save, tmp_save_path)
except IOError:
# make sure this points to the hell
delta_resume = False
# retry
try:
os.remove(tmp_save_path)
except (OSError, IOError):
pass
continue
os.rename(tmp_save_path, save_path)
edelta_approved = True
break
if edelta_approved:
# we can happily return
return 0, data_transfer
# error, give up with the edelta stuff
return 1, data_transfer
def __fetch_file(self, url, save_path, digest = None, resume = True,
download = None, package_id = None, repository = None):
def do_stfu_rm(xpath):
try:
os.remove(xpath)
except OSError:
pass
def do_get_md5sum(path):
try:
return entropy.tools.md5sum(path)
except IOError:
return None
except OSError:
return None
fetch_abort_function = self.pkgmeta.get('fetch_abort_function')
filepath_dir = os.path.dirname(save_path)
# symlink support
if not os.path.isdir(os.path.realpath(filepath_dir)):
try:
os.remove(filepath_dir)
except OSError as err:
const_debug_write(__name__,
"__fetch_file.remove, %s, error: %s" % (
filepath_dir, err))
try:
os.makedirs(filepath_dir, 0o755)
except OSError as err:
const_debug_write(__name__,
"__fetch_file.makedirs, %s, error: %s" % (
filepath_dir, err))
return -1, 0, False
rc, data_transfer = self.__try_edelta_fetch(url, save_path, resume)
if rc == 0:
return rc, data_transfer, False
elif rc < 0: # < 0 errors are unrecoverable
return rc, data_transfer, False
# otherwise, just fallback to package download
existed_before = False
if os.path.isfile(save_path) and os.path.exists(save_path):
existed_before = True
fetch_intf = self._entropy._url_fetcher(
url, save_path, resume = resume,
abort_check_func = fetch_abort_function)
if (download is not None) and (package_id is not None) and \
(repository is not None) and (rc == 0):
fetch_path = self.__get_fetch_disk_path(download)
self._setup_differential_download(
self._entropy._url_fetcher, url,
resume, fetch_path, repository, package_id)
# start to download
data_transfer = 0
resumed = False
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
if fetch_abort_function != None:
fetch_abort_function()
fetch_checksum = fetch_intf.download()
data_transfer = fetch_intf.get_transfer_rate()
resumed = fetch_intf.is_resumed()
except KeyboardInterrupt:
return -100, data_transfer, resumed
except NameError:
raise
except:
if const_debug_enabled():
self._entropy.output(
"fetch_file:",
importance = 1,
level = "warning",
header = red(" ## ")
)
entropy.tools.print_traceback()
if (not existed_before) or (not resume):
do_stfu_rm(save_path)
return -1, data_transfer, resumed
if fetch_checksum == UrlFetcher.GENERIC_FETCH_ERROR:
# !! not found
# maybe we already have it?
# this handles the case where network is unavailable
# but file is already downloaded
fetch_checksum = do_get_md5sum(save_path)
if (fetch_checksum != digest) or fetch_checksum is None:
return -3, data_transfer, resumed
elif fetch_checksum == UrlFetcher.TIMEOUT_FETCH_ERROR:
# maybe we already have it?
# this handles the case where network is unavailable
# but file is already downloaded
fetch_checksum = do_get_md5sum(save_path)
if (fetch_checksum != digest) or fetch_checksum is None:
return -4, data_transfer, resumed
del fetch_intf
if digest and (fetch_checksum != digest):
# not properly downloaded
if (not existed_before) or (not resume):
do_stfu_rm(save_path)
return -2, data_transfer, resumed
return 0, data_transfer, resumed
def __build_uris_list(self, original_repo, repository):
avail_data = self._settings['repositories']['available']
product = self._settings['repositories']['product']
uris = []
plain_packages = avail_data[repository]['plain_packages']
for uri in plain_packages:
expanded_uri = entropy.tools.expand_plain_package_mirror(
uri, product, original_repo)
uris.append(expanded_uri)
uris.reverse()
uris.extend(avail_data[repository]['packages'][::-1])
return uris
def _download_package(self, package_id, repository, download, save_path,
digest = False, resume = True):
avail_data = self._settings['repositories']['available']
excluded_data = self._settings['repositories']['excluded']
repo_db = self._entropy.open_repository(repository)
# grab original repo, if any and use it if available
# this is done in order to support "equo repo merge" feature
# allowing client-side repository package metadata moves.
original_repo = repo_db.getInstalledPackageRepository(package_id)
if (original_repo != repository) and (original_repo not in avail_data) \
and (original_repo is not None):
# build up a new uris list, at least try, hoping that
# repository is just shadowing original_repo
# for example: original_repo got copied to repository, without
# copying packages, which would be useless. like it happens
# with sabayon-weekly
uris = self.__build_uris_list(original_repo, repository)
else:
if original_repo in avail_data:
uris = avail_data[original_repo]['packages'][::-1]
if repository in avail_data:
uris += avail_data[repository]['packages'][::-1]
elif original_repo in excluded_data:
uris = excluded_data[original_repo]['packages'][::-1]
if repository in avail_data:
uris += avail_data[repository]['packages'][::-1]
else:
uris = avail_data[repository]['packages'][::-1]
remaining = set(uris)
mirror_status = StatusInterface()
mirrorcount = 0
for uri in uris:
if not remaining:
# tried all the mirrors, quitting for error
mirror_status.set_working_mirror(None)
return 3
mirror_status.set_working_mirror(uri)
mirrorcount += 1
mirror_count_txt = "( mirror #%s ) " % (mirrorcount,)
url = uri + "/" + download
# check if uri is sane
if mirror_status.get_failing_mirror_status(uri) >= 30:
# ohohoh!
# set to 30 for convenience
mirror_status.set_failing_mirror_status(uri, 30)
mytxt = mirror_count_txt
mytxt += blue(" %s: ") % (_("Mirror"),)
mytxt += red(self._get_url_name(uri))
mytxt += " - %s." % (_("maximum failure threshold reached"),)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
if mirror_status.get_failing_mirror_status(uri) == 30:
# put to 75 then decrement by 4 so we
# won't reach 30 anytime soon ahahaha
mirror_status.add_failing_mirror(uri, 45)
elif mirror_status.get_failing_mirror_status(uri) > 31:
# now decrement each time this point is reached,
# if will be back < 30, then equo will try to use it again
mirror_status.add_failing_mirror(uri, -4)
else:
# put to 0 - reenable mirror, welcome back uri!
mirror_status.set_failing_mirror_status(uri, 0)
remaining.discard(uri)
continue
do_resume = resume
timeout_try_count = 50
while True:
try:
mytxt = mirror_count_txt
mytxt += blue("%s: ") % (_("Downloading from"),)
mytxt += red(self._get_url_name(uri))
# now fetch the new one
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
rc, data_transfer, resumed = \
self.__fetch_file(
url,
save_path,
download = download,
package_id = package_id,
repository = repository,
digest = digest,
resume = do_resume
)
if rc == 0:
mytxt = mirror_count_txt
mytxt += "%s: " % (
blue(_("Successfully downloaded from")),
)
mytxt += red(self._get_url_name(uri))
human_bytes = entropy.tools.bytes_into_human(
data_transfer)
mytxt += " %s %s/%s" % (_("at"),
human_bytes, _("second"),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
mirror_status.set_working_mirror(None)
return 0
elif resumed and (rc not in (-3, -4, -100,)):
do_resume = False
continue
else:
error_message = mirror_count_txt
error_message += blue("%s: %s") % (
_("Error downloading from"),
red(self._get_url_name(uri)),
)
# something bad happened
if rc == -1:
error_message += " - %s." % (
_("file not available on this mirror"),)
elif rc == -2:
mirror_status.add_failing_mirror(uri, 1)
error_message += " - %s." % (_("wrong checksum"),)
# If file is fetched (with no resume) and its
# complete better to enforce resume to False.
if (data_transfer < 1) and do_resume:
error_message += " %s." % (
_("Disabling resume"),)
do_resume = False
continue
elif rc == -3:
mirror_status.add_failing_mirror(uri, 3)
error_message += " - %s." % (_("not found"),)
elif rc == -4: # timeout!
timeout_try_count -= 1
if timeout_try_count > 0:
error_message += " - %s." % (
_("timeout, retrying on this mirror"),)
else:
error_message += " - %s." % (
_("timeout, giving up"),)
elif rc == -100:
error_message += " - %s." % (
_("discarded download"),)
else:
mirror_status.add_failing_mirror(uri, 5)
error_message += " - %s." % (_("unknown reason"),)
self._entropy.output(
error_message,
importance = 1,
level = "warning",
header = red(" ## ")
)
if rc == -4: # timeout
if timeout_try_count > 0:
continue
elif rc == -100: # user discarded fetch
mirror_status.set_working_mirror(None)
return 1
remaining.discard(uri)
# make sure we don't have nasty issues
if not remaining:
mirror_status.set_working_mirror(None)
return 3
break
except KeyboardInterrupt:
mirror_status.set_working_mirror(None)
return 1
mirror_status.set_working_mirror(None)
return 0
def _match_checksum(self, package_id, repository, checksum, download,
signatures):
sys_settings = self._settings
sys_set_plg_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
enabled_hashes = sys_settings[sys_set_plg_id]['misc']['packagehashes']
pkg_disk_path = self.__get_fetch_disk_path(download)
pkg_disk_path_mtime = pkg_disk_path + etpConst['packagemtimefileext']
def do_mtime_validation():
enc = etpConst['conf_encoding']
try:
with codecs.open(pkg_disk_path_mtime,
"r", encoding=enc) as mt_f:
stored_mtime = mt_f.read().strip()
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
return 1
# get pkg mtime
try:
cur_mtime = str(os.path.getmtime(pkg_disk_path))
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
return 2
if cur_mtime == stored_mtime:
return 0
return 1
def do_store_mtime():
enc = etpConst['conf_encoding']
try:
with codecs.open(pkg_disk_path_mtime,
"w", encoding=enc) as mt_f:
cur_mtime = str(os.path.getmtime(pkg_disk_path))
mt_f.write(cur_mtime)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
def do_compare_gpg(pkg_path, hash_val):
try:
repo_sec = self._entropy.RepositorySecurity()
except RepositorySecurity.GPGServiceNotAvailable:
return None
# check if we have repository pubkey
try:
if not repo_sec.is_pubkey_available(repository):
return None
except repo_sec.KeyExpired:
# key is expired
return None
# write gpg signature to disk for verification
tmp_fd, tmp_path = const_mkstemp(prefix="do_compare_gpg")
with os.fdopen(tmp_fd, "w") as tmp_f:
tmp_f.write(hash_val)
try:
# actually verify
valid, err_msg = repo_sec.verify_file(repository, pkg_path,
tmp_path)
finally:
os.remove(tmp_path)
if valid:
return True
if err_msg:
self._entropy.output(
"%s: %s, %s" % (
darkred(_("Package signature verification error for")),
purple("GPG"),
err_msg,
),
importance = 0,
level = "error",
header = darkred(" ## ")
)
return False
signature_vry_map = {
'sha1': entropy.tools.compare_sha1,
'sha256': entropy.tools.compare_sha256,
'sha512': entropy.tools.compare_sha512,
'gpg': do_compare_gpg,
}
def do_signatures_validation(signatures):
# check signatures, if available
if isinstance(signatures, dict):
for hash_type in sorted(signatures):
hash_val = signatures[hash_type]
# NOTE: workaround bug on unreleased
# entropy versions
if hash_val in signatures:
continue
if hash_val is None:
continue
if hash_type not in enabled_hashes:
self._entropy.output(
"%s %s" % (
purple(hash_type.upper()),
darkgreen(_("disabled")),
),
importance = 0,
level = "info",
header = " : "
)
continue
cmp_func = signature_vry_map.get(hash_type)
if cmp_func is None:
continue
self._entropy.output(
"%s: %s" % (blue(_("Checking package signature")),
purple(hash_type.upper()),),
importance = 0,
level = "info",
header = red(" ## "),
back = True
)
valid = cmp_func(pkg_disk_path, hash_val)
if valid is None:
self._entropy.output(
"%s '%s' %s" % (
darkred(_("Package signature verification")),
purple(hash_type.upper()),
darkred(_("temporarily unavailable")),
),
importance = 0,
level = "warning",
header = darkred(" ## ")
)
continue
if not valid:
self._entropy.output(
"%s: %s %s" % (
darkred(_("Package signature")),
purple(hash_type.upper()),
darkred(_("does not match the recorded one")),
),
importance = 0,
level = "error",
header = darkred(" ## ")
)
return 1
self._entropy.output(
"%s %s" % (
purple(hash_type.upper()),
darkgreen(_("matches")),
),
importance = 0,
level = "info",
header = " : "
)
return 0
dlcount = 0
match = False
max_dlcount = 5
while dlcount <= max_dlcount:
self._entropy.output(
blue(_("Checking package checksum...")),
importance = 0,
level = "info",
header = red(" ## "),
back = True
)
dlcheck = self.__check_pkg_path_download(download,
checksum = checksum)
if dlcheck == 0:
basef = os.path.basename(download)
self._entropy.output(
"%s: %s" % (
blue(_("Package checksum matches")),
darkgreen(basef),
),
importance = 0,
level = "info",
header = red(" ## ")
)
# check if package has been already checked
dlcheck = do_mtime_validation()
if dlcheck != 0:
dlcheck = do_signatures_validation(signatures)
if dlcheck == 0:
do_store_mtime()
match = True
break # file downloaded successfully
if dlcheck != 0:
dlcount += 1
mytxt = _("Checksum does not match. Download attempt #%s") % (
dlcount,
)
self._entropy.output(
darkred(mytxt),
importance = 0,
level = "warning",
header = darkred(" ## ")
)
# Unfortunately, disabling resume makes possible to recover
# from bad download data. trying to resume would do more harm
# than good in the majority of cases.
fetch = self._download_package(
package_id,
repository,
download,
pkg_disk_path,
checksum,
resume = False
)
if fetch != 0:
self._entropy.output(
blue(_("Cannot properly fetch package! Quitting.")),
importance = 0,
level = "error",
header = darkred(" ## ")
)
return fetch
# package is fetched, let's loop one more time
# to make sure to run all the checksum checks
continue
if not match:
mytxt = _("Cannot fetch package or checksum does not match")
mytxt2 = _("Try to download latest repositories")
for txt in (mytxt, mytxt2,):
self._entropy.output(
"%s." % (blue(txt),),
importance = 0,
level = "info",
header = red(" ## ")
)
return 1
return 0
def multi_match_checksum(self):
rc = 0
for pkg_id, repository, download, digest, signatures in \
self.pkgmeta['multi_checksum_list']:
rc = self._match_checksum(pkg_id, repository, digest, download,
signatures)
if rc != 0:
break
if rc == 0:
self.pkgmeta['verified'] = True
return rc
def __unpack_package(self, download, package_path, image_dir, pkg_dbpath):
mytxt = "%s: %s" % (
blue(_("Unpacking")),
red(os.path.basename(download)),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Unpacking package: %s" % (download,)
)
# removed in the meantime? at least try to cope
if not os.path.isfile(package_path):
# must be fault-tolerant
if os.path.isdir(package_path):
shutil.rmtree(package_path)
if os.path.islink(package_path):
os.remove(package_path)
self.pkgmeta['verified'] = False
rc = self._fetch_step()
if rc != 0:
return rc
# make sure image_dir always exists
# pkgs not providing any file would cause image_dir
# to not be created by uncompress_tarball
try:
os.makedirs(image_dir, 0o755)
except OSError as err:
if err.errno != errno.EEXIST:
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"Unable to mkdir: %s, error: %s" % (
image_dir, repr(err),)
)
self._entropy.output(
"%s: %s" % (brown(_("Unpack error")), err.errno,),
importance = 1,
level = "error",
header = red(" ## ")
)
return 1
# pkg_dbpath is only non-None for the base package file
# extra package files don't carry any other edb information
if pkg_dbpath is not None:
# extract entropy database from package file
# in order to avoid having to read content data
# from the repository database, which, in future
# is allowed to not provide such info.
pkg_dbdir = os.path.dirname(pkg_dbpath)
if not os.path.isdir(pkg_dbdir):
os.makedirs(pkg_dbdir, 0o755)
# extract edb
dump_rc = entropy.tools.dump_entropy_metadata(
package_path, pkg_dbpath)
if not dump_rc:
# error during entropy db extraction from package file
# might be because edb entry point is not found or
# because there is not enough space for it
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"Unable to dump edb for: " + pkg_dbpath
)
self._entropy.output(
brown(_("Unable to find Entropy metadata in package")),
importance = 1,
level = "error",
header = red(" ## ")
)
return 1
unpack_tries = 3
while True:
if unpack_tries <= 0:
return 1
unpack_tries -= 1
try:
rc = entropy.tools.uncompress_tarball(
package_path,
extract_path = image_dir,
catch_empty = True
)
except EOFError as err:
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"EOFError on " + package_path + " " + \
repr(err)
)
entropy.tools.print_traceback()
# try again until unpack_tries goes to 0
rc = 1
except Exception as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Ouch! error while unpacking " + \
package_path + " " + repr(err)
)
entropy.tools.print_traceback()
# try again until unpack_tries goes to 0
rc = 1
if rc == 0:
break
# try to download it again
self.pkgmeta['verified'] = False
f_rc = self._fetch_step()
if f_rc != 0:
return f_rc
return 0
def __configure_package(self):
try:
spm = self._entropy.Spm()
except Exception as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager not available: %s | %s" % (
type(Exception), err,
)
)
return 1
self._entropy.output(
"SPM: %s" % (brown(_("configuration phase")),),
importance = 0,
header = red(" ## ")
)
try:
spm.execute_package_phase(self.pkgmeta, self.pkgmeta,
self._action, "configure")
except spm.PhaseFailure as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Phase execution failed with %s, %d" % (
err.message, err.code))
return err.code
except spm.OutdatedPhaseError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager is too old: %s" % (
err))
err_msg = "%s: %s" % (
brown(_("Source Package Manager is too old, please update it")),
err)
self._entropy.output(
err_msg,
importance = 1,
header = darkred(" ## "),
level = "error"
)
return 1
except spm.PhaseError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Phase execution error: %s" % (
err))
return 1
return 0
def _setup_config_protect_and_mask(self, installed_repository,
installed_package_id):
"""
Setup the config_protect+mask metadata object.
Make sure to call this before the package goes away from the
repository.
"""
protect = self._get_config_protect(
installed_repository, installed_package_id)
mask = self._get_config_protect(
installed_repository, installed_package_id, mask = True)
self.pkgmeta['config_protect+mask'] = (protect, mask)
def __remove_package(self):
self._entropy.clear_cache()
self._entropy.logger.log("[Package]",
etpConst['logging']['normal_loglevel_id'],
"Removing package: %s" % (self.pkgmeta['removeatom'],))
mytxt = "%s: %s" % (
blue(_("Removing from Entropy")),
red(self.pkgmeta['removeatom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
inst_repo = self._entropy.installed_repository()
automerge_metadata = inst_repo.retrieveAutomergefiles(
self.pkgmeta['removepackage_id'], get_dict = True)
# setup config_protect and config_protect_mask metadata before it's
# too late.
self._setup_config_protect_and_mask(
inst_repo, self.pkgmeta['removepackage_id'])
inst_repo.removePackage(self.pkgmeta['removepackage_id'])
# commit changes, to avoid users pressing CTRL+C and still having
# all the db entries in, so we need to commit at every iteration
inst_repo.commit()
self._remove_content_from_system(
automerge_metadata = automerge_metadata)
return 0
def _remove_content_from_system_loop(
self, remove_content, directories, directories_cache,
not_removed_due_to_collisions, colliding_path_messages,
automerge_metadata, col_protect, protect, mask, protectskip,
sys_root):
"""
Body of the _remove_content_from_system() method.
"""
inst_repo = self._entropy.installed_repository()
info_dirs = self._get_info_directories()
for _pkg_id, item, _ftype in remove_content:
if not item:
continue # empty element??
sys_root_item = sys_root + item
sys_root_item_encoded = sys_root_item
if not const_is_python3():
# this is coming from the db, and it's pure utf-8
sys_root_item_encoded = const_convert_to_rawstring(
sys_root_item,
from_enctype = etpConst['conf_raw_encoding'])
# collision check
if col_protect > 0:
if inst_repo.isFileAvailable(item) \
and os.path.isfile(sys_root_item_encoded):
# in this way we filter out directories
colliding_path_messages.add(sys_root_item)
not_removed_due_to_collisions.add(item)
continue
protected = False
in_mask = False
if not self.pkgmeta['removeconfig']:
protected_item_test = sys_root_item
(in_mask, protected, _x,
do_continue) = self._handle_config_protect(
protect, mask, protectskip, None, protected_item_test,
do_allocation_check = False, do_quiet = True
)
if do_continue:
protected = True
# when files have not been modified by the user
# and they are inside a config protect directory
# we could even remove them directly
if in_mask:
oldprot_md5 = automerge_metadata.get(item)
if oldprot_md5:
try:
in_system_md5 = entropy.tools.md5sum(
protected_item_test)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
in_system_md5 = "?"
if oldprot_md5 == in_system_md5:
prot_msg = _("Removing config file, never modified")
mytxt = "%s: %s" % (
darkgreen(prot_msg),
blue(item),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
protected = False
do_continue = False
# Is file or directory a protected item?
if protected:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['verbose_loglevel_id'],
"[remove] Protecting config file: %s" % (sys_root_item,)
)
mytxt = "[%s] %s: %s" % (
red(_("remove")),
brown(_("Protecting config file")),
sys_root_item,
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = red(" ## ")
)
continue
try:
os.lstat(sys_root_item_encoded)
except OSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
continue # skip file, does not exist
raise
except UnicodeEncodeError:
msg = _("This package contains a badly encoded file !!!")
mytxt = brown(msg)
self._entropy.output(
red("QA: ")+mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
continue # file has a really bad encoding
if os.path.isdir(sys_root_item_encoded) and \
os.path.islink(sys_root_item_encoded):
# S_ISDIR returns False for directory symlinks,
# so using os.path.isdir valid directory symlink
if sys_root_item not in directories_cache:
# collect for Trigger
self.pkgmeta['affected_directories'].add(item)
directories.add((sys_root_item, "link"))
directories_cache.add(sys_root_item)
continue
if os.path.isdir(sys_root_item_encoded):
# plain directory
if sys_root_item not in directories_cache:
# collect for Trigger
self.pkgmeta['affected_directories'].add(item)
directories.add((sys_root_item, "dir"))
directories_cache.add(sys_root_item)
continue
# files, symlinks or not
# just a file or symlink or broken
# directory symlink (remove now)
try:
os.remove(sys_root_item_encoded)
except OSError as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"[remove] Unable to remove %s, error: %s" % (
sys_root_item, err,)
)
continue
# collect for Trigger
dir_name = os.path.dirname(item)
self.pkgmeta['affected_directories'].add(dir_name)
# account for info files, if any
if dir_name in info_dirs:
for _ext in self._INFO_EXTS:
if item.endswith(_ext):
self.pkgmeta['affected_infofiles'].add(item)
break
# add its parent directory
dirobj = const_convert_to_unicode(
os.path.dirname(sys_root_item_encoded))
if dirobj not in directories_cache:
if os.path.isdir(dirobj) and os.path.islink(dirobj):
directories.add((dirobj, "link"))
elif os.path.isdir(dirobj):
directories.add((dirobj, "dir"))
directories_cache.add(dirobj)
def _remove_content_from_system(self, automerge_metadata = None):
"""
Remove installed package content (files/directories) from live system.
@keyword automerge_metadata: Entropy "automerge metadata"
@type automerge_metadata: dict
"""
if automerge_metadata is None:
automerge_metadata = {}
sys_root = etpConst['systemroot']
# load CONFIG_PROTECT and CONFIG_PROTECT_MASK
sys_settings = self._settings
sys_set_plg_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
col_protect = sys_settings[sys_set_plg_id]['misc']['collisionprotect']
# remove files from system
directories = set()
directories_cache = set()
not_removed_due_to_collisions = set()
colliding_path_messages = set()
protect_mask = self.pkgmeta['config_protect+mask']
if protect_mask is not None:
protect, mask = protect_mask
else:
protect, mask = set(), set()
protectskip = self._get_config_protect_skip()
remove_content = None
try:
# simulate a removecontent list/set object
remove_content = []
if self.pkgmeta['removecontent_file'] is not None:
remove_content = Package.FileContentReader(
self.pkgmeta['removecontent_file'])
self._remove_content_from_system_loop(
remove_content, directories, directories_cache,
not_removed_due_to_collisions, colliding_path_messages,
automerge_metadata, col_protect, protect, mask, protectskip,
sys_root)
finally:
if hasattr(remove_content, "close"):
remove_content.close()
if colliding_path_messages:
self._entropy.output(
"%s:" % (_("Collision found during removal of"),),
importance = 1,
level = "warning",
header = red(" ## ")
)
for path in sorted(colliding_path_messages):
self._entropy.output(
purple(path),
importance = 0,
level = "warning",
header = red(" ## ")
)
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"Collision found during removal of %s - cannot overwrite" % (
path,)
)
# removing files not removed from removecontent.
# it happened that boot services not removed due to
# collisions got removed from their belonging runlevels
# by postremove step.
# since this is a set, it is a mapped type, so every
# other instance around will feature this update
if not_removed_due_to_collisions:
def _filter(_path):
return _path not in not_removed_due_to_collisions
Package._filter_content_file(
self.pkgmeta['removecontent_file'],
_filter)
# now handle directories
directories = sorted(directories, reverse = True)
while True:
taint = False
for directory, dirtype in directories:
mydir = "%s%s" % (sys_root, directory,)
if dirtype == "link":
try:
mylist = os.listdir(mydir)
if not mylist:
try:
os.remove(mydir)
taint = True
except OSError:
pass
except OSError:
pass
elif dirtype == "dir":
try:
mylist = os.listdir(mydir)
if not mylist:
try:
os.rmdir(mydir)
taint = True
except OSError:
pass
except OSError:
pass
if not taint:
break
del directories_cache
del directories
def _cleanup_package(self, unpack_dir):
# shutil.rmtree wants raw strings... otherwise it will explode
if const_isunicode(unpack_dir):
unpack_dir = const_convert_to_rawstring(unpack_dir)
# remove unpack dir
try:
shutil.rmtree(unpack_dir, True)
except Exception as err:
# fault tolerance here dude
self._entropy.logger.log(
"[Package]", etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to cleanup directory %s," \
" error: %s" % (unpack_dir, err,))
try:
os.rmdir(unpack_dir)
except OSError:
pass
return 0
def __install_package(self):
# clear on-disk cache
self._entropy.clear_cache()
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Installing package: %s" % (self.pkgmeta['atom'],)
)
inst_repo = self._entropy.installed_repository()
if self.pkgmeta['removepackage_id'] != -1:
self.pkgmeta['already_protected_config_files'] = \
inst_repo.retrieveAutomergefiles(
self.pkgmeta['removepackage_id'], get_dict = True
)
# items_*installed will be filled by _move_image_to_system
# then passed to _add_installed_package()
items_installed = set()
items_not_installed = set()
rc = self._move_image_to_system(
items_installed, items_not_installed)
if rc != 0:
return rc
# inject into database
mytxt = "%s: %s" % (
blue(_("Updating database")),
red(self.pkgmeta['atom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
self.pkgmeta['installed_package_id'] = self._add_installed_package(
items_installed, items_not_installed)
return 0
def _spm_install_package(self, installed_package_id):
"""
Call Source Package Manager interface and tell it to register our
newly installed package.
@param installed_package_id: Entropy repository package identifier
@type installed_package_id: int
@return: execution status
@rtype: int
"""
try:
Spm = self._entropy.Spm()
except Exception as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager not available: %s | %s" % (
type(Exception), err,
)
)
return -1
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Installing new SPM entry: %s" % (self.pkgmeta['atom'],)
)
spm_uid = Spm.add_installed_package(self.pkgmeta)
if spm_uid != -1:
self._entropy.installed_repository().insertSpmUid(
installed_package_id, spm_uid)
return 0
def _spm_update_package_uid(self, installed_package_id, entropy_atom):
"""
Update Source Package Manager <-> Entropy package identifiers coupling.
Entropy can handle multiple packages in the same scope from a SPM POV
(see the "package tag" feature to provide linux kernel module packages
for different kernel versions). This method just reassigns a new SPM
unique package identifier to Entropy.
@param installed_package_id: Entropy package identifier bound to
given entropy_atom
@type installed_package_id: int
@param entropy_atom: Entropy package atom, must be converted to a valid
SPM package atom.
@type entropy_atom: string
@return: execution status
@rtype: int
"""
try:
Spm = self._entropy.Spm()
except Exception as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager not available: %s | %s" % (
type(Exception), err,
)
)
return -1
spm_package = Spm.convert_from_entropy_package_name(entropy_atom)
try:
spm_uid = Spm.assign_uid_to_installed_package(spm_package)
except (SPMError, KeyError,):
# installed package not available, we must ignore it
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Spm uid not available for Spm package: %s (pkg not avail?)" % (
spm_package,
)
)
return 0
if spm_uid != -1:
self._entropy.installed_repository().insertSpmUid(
installed_package_id, spm_uid)
return 0
def _spm_remove_package(self):
"""
Call Source Package Manager interface and tell it to remove our
just removed package.
@return: execution status
@rtype: int
"""
try:
Spm = self._entropy.Spm()
except Exception as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Source Package Manager not available: %s | %s" % (
type(Exception), err,
)
)
return -1
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Removing from SPM: %s" % (self.pkgmeta['removeatom'],)
)
return Spm.remove_installed_package(self.pkgmeta)
def _add_installed_package(self, items_installed, items_not_installed):
"""
For internal use only.
Copy package from repository to installed packages one.
"""
def _merge_removecontent(inst_repo, repo, _package_id):
# NOTE: this could be a source of memory consumption
# but generally, the difference between two contents
# is really small
content_diff = list(inst_repo.contentDiff(
self.pkgmeta['removepackage_id'],
repo,
_package_id,
extended=True))
if content_diff:
# reverse-order compare
def _cmp_func(_path, _spath):
if _path > _spath:
return -1
elif _path == _spath:
return 0
return 1
# must be sorted, and in reverse order
# or the merge step won't work
content_diff.sort(reverse=True)
Package._merge_content_file(
self.pkgmeta['removecontent_file'],
content_diff, _cmp_func)
inst_repo = self._entropy.installed_repository()
smart_pkg = self.pkgmeta['smartpackage']
dbconn = self._entropy.open_repository(self.pkgmeta['repository'])
splitdebug, splitdebug_dirs = self.pkgmeta['splitdebug'], \
self.pkgmeta['splitdebug_dirs']
if smart_pkg or self.pkgmeta['merge_from']:
data = dbconn.getPackageData(self.pkgmeta['package_id'],
content_insert_formatted = True,
get_changelog = False, get_content = False,
get_content_safety = False)
content = dbconn.retrieveContentIter(
self.pkgmeta['package_id'])
content_file = self.__generate_content_file(
content, package_id = self.pkgmeta['package_id'],
filter_splitdebug = True,
splitdebug = splitdebug,
splitdebug_dirs = splitdebug_dirs)
content_safety = dbconn.retrieveContentSafetyIter(
self.pkgmeta['package_id'])
content_safety_file = self.__generate_content_safety_file(
content_safety)
if self.pkgmeta['removepackage_id'] != -1 and \
self.pkgmeta['removecontent_file'] is not None:
_merge_removecontent(
inst_repo, dbconn, self.pkgmeta['package_id'])
else:
# normal repositories
data = dbconn.getPackageData(self.pkgmeta['package_id'],
get_content = False, get_changelog = False)
# indexing_override = False : no need to index tables
# xcache = False : no need to use on-disk cache
# skipChecks = False : creating missing tables is unwanted,
# and also no foreign keys update
# readOnly = True: no need to open in write mode
pkg_dbconn = self._entropy.open_generic_repository(
self.pkgmeta['pkgdbpath'], skip_checks = True,
indexing_override = False, read_only = True,
xcache = False)
# it is safe to consider that package dbs coming from repos
# contain only one entry
pkg_package_id = sorted(pkg_dbconn.listAllPackageIds(),
reverse = True)[0]
content = pkg_dbconn.retrieveContentIter(
pkg_package_id)
content_file = self.__generate_content_file(
content, package_id = self.pkgmeta['package_id'],
filter_splitdebug = True,
splitdebug = splitdebug,
splitdebug_dirs = splitdebug_dirs)
# setup content safety metadata, get from package
content_safety = pkg_dbconn.retrieveContentSafetyIter(
pkg_package_id)
content_safety_file = self.__generate_content_safety_file(
content_safety)
if self.pkgmeta['removepackage_id'] != -1 and \
self.pkgmeta['removecontent_file'] is not None:
_merge_removecontent(inst_repo, pkg_dbconn, pkg_package_id)
pkg_dbconn.close()
# items_installed is useful to avoid the removal of installed
# files by __remove_package just because
# there's a difference in the directory path, perhaps,
# which is not handled correctly by
# EntropyRepository.contentDiff for obvious reasons
# (think about stuff in /usr/lib and /usr/lib64,
# where the latter is just a symlink to the former)
# --
# fix removecontent, need to check if we just installed files
# that resolves at the same directory path (different symlink)
if self.pkgmeta['removecontent_file'] is not None:
self.__filter_out_files_installed_on_diff_path(
self.pkgmeta['removecontent_file'],
items_installed)
# setup config_protect and config_protect_mask metadata before it's
# too late.
installed_package_id = self.pkgmeta['removepackage_id']
if installed_package_id != -1:
self._setup_config_protect_and_mask(
inst_repo, installed_package_id)
# filter out files not installed from content metadata
# these include splitdebug files, when splitdebug is
# disabled.
if items_not_installed:
def _filter(_path):
return _path not in items_not_installed
Package._filter_content_file(
content_file, _filter)
# this is needed to make postinstall trigger work properly
self.pkgmeta['triggers']['install']['affected_directories'] = \
self.pkgmeta['affected_directories']
self.pkgmeta['triggers']['install']['affected_infofiles'] = \
self.pkgmeta['affected_infofiles']
# always set data['injected'] to False
# installed packages database SHOULD never have more
# than one package for scope (key+slot)
data['injected'] = False
# spm counter will be set in self._install_package_into_spm_database()
data['counter'] = -1
# branch must be always set properly, it could happen it's not
# when installing packages through their .tbz2s
data['branch'] = self._settings['repositories']['branch']
# there is no need to store needed paths into db
if "needed_paths" in data:
del data['needed_paths']
# there is no need to store changelog data into db
if "changelog" in data:
del data['changelog']
# we don't want it to be added now, we want to add install source
# info too.
if "original_repository" in data:
del data['original_repository']
# rewrite extra_download metadata with the currently provided,
# and accepted extra_download items (in case of splitdebug being
# disable, we're not going to add those entries, for example)
data['extra_download'] = self.pkgmeta['extra_download']
data['content'] = None
data['content_safety'] = None
try:
# now we are ready to craft a 'content' iter object
data['content'] = Package.FileContentReader(
content_file)
data['content_safety'] = Package.FileContentSafetyReader(
content_safety_file)
package_id = inst_repo.handlePackage(
data, revision = data['revision'],
formattedContent = True)
finally:
if data['content'] is not None:
try:
data['content'].close()
data['content'] = None
except (OSError, IOError):
data['content'] = None
if data['content_safety'] is not None:
try:
data['content_safety'].close()
data['content_safety'] = None
except (OSError, IOError):
data['content_safety'] = None
# update datecreation
ctime = time.time()
inst_repo.setCreationDate(package_id, str(ctime))
# add idpk to the installedtable
inst_repo.dropInstalledPackageFromStore(package_id)
inst_repo.storeInstalledPackage(package_id,
self.pkgmeta['repository'], self.pkgmeta['install_source'])
automerge_data = self.pkgmeta.get('configprotect_data')
if automerge_data:
inst_repo.insertAutomergefiles(package_id, automerge_data)
inst_repo.commit()
# replace current empty "content" metadata info
# content metadata is required by
# _spm_install_package() -> Spm.add_installed_package()
# in case of injected packages (SPM metadata might be
# incomplete).
self.pkgmeta['triggers']['install']['content'] = \
Package.FileContentReader(content_file)
return package_id
def __fill_image_dir(self, merge_from, image_dir):
dbconn = self._entropy.open_repository(self.pkgmeta['repository'])
# this is triggered by merge_from pkgmeta metadata
# even if repositories are allowed to not have content
# metadata, in this particular case, it is mandatory
contents = dbconn.retrieveContentIter(
self.pkgmeta['package_id'], order_by = "file")
# collect files
for path, ftype in contents:
# convert back to filesystem str
encoded_path = path
path = os.path.join(merge_from, encoded_path[1:])
topath = os.path.join(image_dir, encoded_path[1:])
path = const_convert_to_rawstring(path)
topath = const_convert_to_rawstring(topath)
try:
exist = os.lstat(path)
except OSError:
continue # skip file
if 'dir' == ftype and \
not stat.S_ISDIR(exist.st_mode) and \
os.path.isdir(path):
# workaround for directory symlink issues
path = os.path.realpath(path)
copystat = False
# if our directory is a symlink instead, then copy the symlink
if os.path.islink(path):
tolink = os.readlink(path)
if os.path.islink(topath):
os.remove(topath)
os.symlink(tolink, topath)
elif os.path.isdir(path):
if not os.path.isdir(topath):
os.makedirs(topath)
copystat = True
elif os.path.isfile(path):
if os.path.isfile(topath):
os.remove(topath) # should never happen
shutil.copy2(path, topath)
copystat = True
if copystat:
user = os.stat(path)[stat.ST_UID]
group = os.stat(path)[stat.ST_GID]
os.chown(topath, user, group)
shutil.copystat(path, topath)
del contents
def _get_config_protect(self, entropy_repository, package_id, mask = False):
"""
Return configuration protection (or mask) metadata for the given
package.
This method should not be used as source for storing metadata into
repositories since the returned objects may not be decoded in utf-8.
Data returned by this method is expected to be used only by internal
functions.
"""
cl_id = etpConst['system_settings_plugins_ids']['client_plugin']
misc_data = self._settings[cl_id]['misc']
if mask:
paths = entropy_repository.retrieveProtectMask(package_id).split()
misc_key = "configprotectmask"
else:
paths = entropy_repository.retrieveProtect(package_id).split()
misc_key = "configprotect"
root = etpConst['systemroot']
config = set(("%s%s" % (root, path) for path in paths))
config.update(misc_data[misc_key])
# os.* methods in Python 2.x do not expect unicode strings
# This set of data is only used by _handle_config_protect atm.
if not const_is_python3():
config = set((const_convert_to_rawstring(x) for x in config))
return config
def _get_config_protect_skip(self):
"""
Return the configuration protection path set.
"""
sys_set_plg_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
client_settings = self._settings[sys_set_plg_id]
misc_settings = client_settings['misc']
protectskip = misc_settings['configprotectskip']
if not const_is_python3():
protectskip = set((
const_convert_to_rawstring(
x, from_enctype = etpConst['conf_encoding']) for x in
misc_settings['configprotectskip']))
return protectskip
def __get_sys_root(self):
return self.pkgmeta.get('unittest_root', '') + \
etpConst['systemroot']
def _move_image_to_system(self, items_installed, items_not_installed):
# load CONFIG_PROTECT and its mask
repo = self._entropy.open_repository(self.pkgmeta['repository'])
protect = self._get_config_protect(repo, self.pkgmeta['package_id'])
mask = self._get_config_protect(repo, self.pkgmeta['package_id'],
mask = True)
protectskip = self._get_config_protect_skip()
# support for unit testing settings
sys_root = self.__get_sys_root()
sys_set_plg_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
misc_data = self._settings[sys_set_plg_id]['misc']
col_protect = misc_data['collisionprotect']
splitdebug, splitdebug_dirs = self.pkgmeta['splitdebug'], \
self.pkgmeta['splitdebug_dirs']
info_dirs = self._get_info_directories()
# setup image_dir properly
image_dir = self.pkgmeta['imagedir'][:]
if not const_is_python3():
# image_dir comes from unpackdir, which comes from download
# metadatum, which is utf-8 (conf_encoding)
image_dir = const_convert_to_rawstring(image_dir,
from_enctype = etpConst['conf_encoding'])
movefile = entropy.tools.movefile
def workout_subdir(currentdir, subdir):
imagepath_dir = os.path.join(currentdir, subdir)
rel_imagepath_dir = imagepath_dir[len(image_dir):]
rootdir = sys_root + rel_imagepath_dir
# splitdebug (.debug files) support
# If splitdebug is not enabled, do not create splitdebug directories
# and move on instead (return)
if not splitdebug:
for split_dir in splitdebug_dirs:
if rootdir.startswith(split_dir):
# also drop item from content metadata. In this way
# SPM has in sync information on what the package
# content really is.
# ---
# we should really use unicode
# strings for items_not_installed
unicode_rootdir = const_convert_to_unicode(rootdir)
items_not_installed.add(unicode_rootdir)
return
# handle broken symlinks
if os.path.islink(rootdir) and not os.path.exists(rootdir):
# broken symlink
os.remove(rootdir)
# if our directory is a file on the live system
elif os.path.isfile(rootdir): # really weird...!
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! %s is a file when it should be " \
"a directory !! Removing in 20 seconds..." % (rootdir,)
)
mytxt = darkred(_("%s is a file when should be a " \
"directory !! Removing in 20 seconds...") % (rootdir,))
self._entropy.output(
red("QA: ")+mytxt,
importance = 1,
level = "warning",
header = red(" !!! ")
)
os.remove(rootdir)
# if our directory is a symlink instead, then copy the symlink
if os.path.islink(imagepath_dir):
# if our live system features a directory instead of
# a symlink, we should consider removing the directory
if not os.path.islink(rootdir) and os.path.isdir(rootdir):
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! %s is a directory when it should be " \
"a symlink !! Removing in 20 seconds..." % (
rootdir,)
)
mytxt = "%s: %s" % (
_("directory expected, symlink found"),
rootdir,
)
mytxt2 = _("Removing in 20 seconds !!")
for txt in (mytxt, mytxt2,):
self._entropy.output(
darkred("QA: ") + darkred(txt),
importance = 1,
level = "warning",
header = red(" !!! ")
)
# fucking kill it in any case!
# rootdir must die! die die die die!
# /me brings chainsaw
try:
shutil.rmtree(rootdir, True)
except (shutil.Error, OSError,) as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to rm %s " \
"directory ! [workout_subdir/1]: %s" % (
rootdir, err,
)
)
tolink = os.readlink(imagepath_dir)
live_tolink = None
if os.path.islink(rootdir):
live_tolink = os.readlink(rootdir)
if tolink != live_tolink:
_symfail = False
if os.path.lexists(rootdir):
# at this point, it must be a file
try:
os.remove(rootdir)
except OSError as err:
_symfail = True
# must be atomic, too bad if it fails
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to remove %s " \
"file ! [workout_file/0]: %s" % (
rootdir, err,
)
)
msg = _("Cannot remove symlink")
mytxt = "%s: %s => %s" % (
purple(msg),
blue(rootdir),
repr(err),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = brown(" ## ")
)
if not _symfail:
os.symlink(tolink, rootdir)
elif not os.path.isdir(rootdir):
# directory not found, we need to create it
try:
# really force a simple mkdir first of all
os.mkdir(rootdir)
except OSError:
os.makedirs(rootdir)
if not os.path.islink(rootdir):
# symlink doesn't need permissions, also
# until os.walk ends they might be broken
user = os.stat(imagepath_dir)[stat.ST_UID]
group = os.stat(imagepath_dir)[stat.ST_GID]
try:
os.chown(rootdir, user, group)
shutil.copystat(imagepath_dir, rootdir)
except (OSError, IOError) as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Error during workdir setup " \
"%s, %s, errno: %s" % (
rootdir,
err,
err.errno,
)
)
# skip some errors because we may have
# unwritable directories
if err.errno not in (
errno.EPERM, errno.ENOENT,
errno.ENOTDIR):
mytxt = "%s: %s, %s, %s" % (
brown("Error during workdir setup"),
purple(rootdir), err,
err.errno
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = darkred(" !!! ")
)
return 4
item_dir, item_base = os.path.split(rootdir)
item_dir = os.path.realpath(item_dir)
item_inst = os.path.join(item_dir, item_base)
item_inst = const_convert_to_unicode(item_inst)
items_installed.add(item_inst)
def workout_file(currentdir, item):
fromfile = os.path.join(currentdir, item)
rel_fromfile = fromfile[len(image_dir):]
rel_fromfile_dir = os.path.dirname(rel_fromfile)
tofile = sys_root + rel_fromfile
rel_fromfile_dir_utf = const_convert_to_unicode(
rel_fromfile_dir)
self.pkgmeta['affected_directories'].add(
rel_fromfile_dir_utf)
# account for info files, if any
if rel_fromfile_dir_utf in info_dirs:
rel_fromfile_utf = const_convert_to_unicode(
rel_fromfile)
for _ext in self._INFO_EXTS:
if rel_fromfile_utf.endswith(_ext):
self.pkgmeta['affected_infofiles'].add(
rel_fromfile_utf)
break
# splitdebug (.debug files) support
# If splitdebug is not enabled, do not create
# splitdebug directories and move on instead (return)
if not splitdebug:
for split_dir in splitdebug_dirs:
if tofile.startswith(split_dir):
# also drop item from content metadata. In this way
# SPM has in sync information on what the package
# content really is.
# ---
# we should really use unicode
# strings for items_not_installed
unicode_tofile = const_convert_to_unicode(tofile)
items_not_installed.add(unicode_tofile)
return 0
if col_protect > 1:
todbfile = fromfile[len(image_dir):]
myrc = self._handle_install_collision_protect(tofile,
todbfile)
if not myrc:
return 0
prot_old_tofile = tofile[len(sys_root):]
# configprotect_data is passed to insertAutomergefiles()
# which always expects unicode data.
# revert back to unicode (we previously called encode on
# image_dir (which is passed to os.walk, which generates
# raw strings)
prot_old_tofile = const_convert_to_unicode(prot_old_tofile)
pre_tofile = tofile[:]
(in_mask, protected,
tofile, do_return) = self._handle_config_protect(
protect, mask, protectskip, fromfile, tofile)
# collect new config automerge data
if in_mask and os.path.exists(fromfile):
try:
prot_md5 = const_convert_to_unicode(
entropy.tools.md5sum(fromfile))
self.pkgmeta['configprotect_data'].append(
(prot_old_tofile, prot_md5,))
except (IOError,) as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to get md5 of %s " \
"file ! [workout_file/1]: %s" % (
fromfile, err,
)
)
# check if it's really necessary to protect file
if protected:
# second task
# prot_old_tofile is always unicode, it must be, see above
oldprot_md5 = self.pkgmeta['already_protected_config_files'].get(
prot_old_tofile)
if oldprot_md5:
try:
in_system_md5 = entropy.tools.md5sum(pre_tofile)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
in_system_md5 = "?"
if oldprot_md5 == in_system_md5:
# we can merge it, files, even if
# contains changes have not been modified
# by the user
msg = _("Automerging config file, never modified")
mytxt = "%s: %s" % (
darkgreen(msg),
blue(pre_tofile),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
protected = False
do_return = False
tofile = pre_tofile
if do_return:
return 0
try:
from_r_path = os.path.realpath(fromfile)
except RuntimeError:
# circular symlink, fuck!
# really weird...!
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! %s is a circular symlink !!!" % (fromfile,)
)
mytxt = "%s: %s" % (
_("Circular symlink issue"),
const_convert_to_unicode(fromfile),
)
self._entropy.output(
darkred("QA: ") + darkred(mytxt),
importance = 1,
level = "warning",
header = red(" !!! ")
)
from_r_path = fromfile
try:
to_r_path = os.path.realpath(tofile)
except RuntimeError:
# circular symlink, fuck!
# really weird...!
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! %s is a circular symlink !!!" % (tofile,)
)
mytxt = "%s: %s" % (
_("Circular symlink issue"),
const_convert_to_unicode(tofile),
)
self._entropy.output(
darkred("QA: ") + darkred(mytxt),
importance = 1,
level = "warning",
header = red(" !!! ")
)
to_r_path = tofile
if from_r_path == to_r_path and os.path.islink(tofile):
# there is a serious issue here, better removing tofile,
# happened to someone.
try:
# try to cope...
os.remove(tofile)
except (OSError, IOError,) as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to cope to oddity of %s " \
"file ! [workout_file/2]: %s" % (
tofile, err,
)
)
# if our file is a dir on the live system
if os.path.isdir(tofile) and not os.path.islink(tofile):
# really weird...!
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! %s is a directory when it should " \
"be a file !! Removing in 20 seconds..." % (tofile,)
)
mytxt = "%s: %s" % (
_("file expected, directory found"),
const_convert_to_unicode(tofile),
)
mytxt2 = _("Removing in 20 seconds !!")
for txt in (mytxt, mytxt2,):
self._entropy.output(
darkred("QA: ") + darkred(txt),
importance = 1,
level = "warning",
header = red(" !!! ")
)
try:
shutil.rmtree(tofile, True)
except (shutil.Error, IOError,) as err:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to cope to oddity of %s " \
"file ! [workout_file/3]: %s" % (
tofile, err,
)
)
# moving file using the raw format
try:
done = movefile(fromfile, tofile, src_basedir = image_dir)
except (IOError,) as err:
# try to move forward, sometimes packages might be
# fucked up and contain broken things
if err.errno not in (errno.ENOENT, errno.EACCES,):
raise
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Error during file move" \
" to system: %s => %s | IGNORED: %s" % (
const_convert_to_unicode(fromfile),
const_convert_to_unicode(tofile),
err,
)
)
done = True
if not done:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Error during file move" \
" to system: %s => %s" % (fromfile, tofile,)
)
mytxt = "%s: %s => %s, %s" % (
_("File move error"),
const_convert_to_unicode(fromfile),
const_convert_to_unicode(tofile),
_("please report"),
)
self._entropy.output(
red("QA: ")+darkred(mytxt),
importance = 1,
level = "warning",
header = red(" !!! ")
)
return 4
item_dir = os.path.realpath(os.path.dirname(tofile))
item_inst = os.path.join(item_dir, os.path.basename(tofile))
item_inst = const_convert_to_unicode(item_inst)
items_installed.add(item_inst)
if protected and \
os.getenv("ENTROPY_CLIENT_ENABLE_OLD_FILEUPDATES"):
# add to disk cache
file_updates = self._entropy.PackageFileUpdates()
file_updates.add(tofile, quiet = True)
return 0
# merge data into system
for currentdir, subdirs, files in os.walk(image_dir):
# create subdirs
for subdir in subdirs:
workout_subdir(currentdir, subdir)
for item in files:
move_st = workout_file(currentdir, item)
if move_st != 0:
return move_st
return 0
def __filter_out_files_installed_on_diff_path(self, content_file,
installed_content):
"""
Use case: if a package provided files in /lib then, a new version
of that package moved the same files under /lib64, we need to check
if both directory paths solve to the same inode and if so,
add to our set that we're going to return.
"""
sys_root = self.__get_sys_root()
second_pass_removal = set()
if not installed_content:
# nothing to filter, no-op
return
def _main_filter(_path):
item_dir = os.path.dirname("%s%s" % (
sys_root, _path,))
item = os.path.join(
os.path.realpath(item_dir),
os.path.basename(_path))
if item in installed_content:
second_pass_removal.add(item)
return False
return True
# first pass, remove direct matches, schedule a second pass
# list of files
Package._filter_content_file(content_file, _main_filter)
if not second_pass_removal:
# done then
return
# second pass, drop remaining files
# unfortunately, this is the only way to work it out
# with iterators
def _filter(_path):
return _path not in second_pass_removal
Package._filter_content_file(content_file, _filter)
def _handle_config_protect(self, protect, mask, protectskip,
fromfile, tofile,
do_allocation_check = True,
do_quiet = False):
"""
Handle configuration file protection. This method contains the logic
for determining if a file should be protected from overwrite.
"""
protected = False
do_continue = False
in_mask = False
if tofile in protect:
protected = True
in_mask = True
elif os.path.dirname(tofile) in protect:
protected = True
in_mask = True
else:
tofile_testdir = os.path.dirname(tofile)
old_tofile_testdir = None
while tofile_testdir != old_tofile_testdir:
if tofile_testdir in protect:
protected = True
in_mask = True
break
old_tofile_testdir = tofile_testdir
tofile_testdir = os.path.dirname(tofile_testdir)
if protected: # check if perhaps, file is masked, so unprotected
if tofile in mask:
protected = False
in_mask = False
elif os.path.dirname(tofile) in mask:
protected = False
in_mask = False
else:
tofile_testdir = os.path.dirname(tofile)
old_tofile_testdir = None
while tofile_testdir != old_tofile_testdir:
if tofile_testdir in mask:
protected = False
in_mask = False
break
old_tofile_testdir = tofile_testdir
tofile_testdir = os.path.dirname(tofile_testdir)
if not os.path.lexists(tofile):
protected = False # file doesn't exist
# check if it's a text file
if protected:
protected = entropy.tools.istextfile(tofile)
in_mask = protected
if fromfile is not None:
if protected and os.path.lexists(fromfile) and \
(not os.path.exists(fromfile)) and os.path.islink(fromfile):
# broken symlink, don't protect
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Failed to handle file protection for: " \
"%s, broken symlink in package" % (
tofile,
)
)
msg = _("Cannot protect broken symlink")
mytxt = "%s:" % (
purple(msg),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = brown(" ## ")
)
self._entropy.output(
tofile,
level = "warning",
header = brown(" ## ")
)
protected = False
if not protected:
return in_mask, protected, tofile, do_continue
## ##
# file is protected #
##__________________##
# check if protection is disabled for this element
if tofile in protectskip:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Skipping config file installation/removal, " \
"as stated in client.conf: %s" % (tofile,)
)
if not do_quiet:
mytxt = "%s: %s" % (
_("Skipping file installation/removal"),
tofile,
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
do_continue = True
return in_mask, protected, tofile, do_continue
## ##
# file is protected (2) #
##______________________##
prot_status = True
if do_allocation_check:
spm_class = self._entropy.Spm_class()
tofile, prot_status = spm_class.allocate_protected_file(fromfile,
tofile)
if not prot_status:
# a protected file with the same content
# is already in place, so not going to protect
# the same file twice
protected = False
return in_mask, protected, tofile, do_continue
## ##
# file is protected (3) #
##______________________##
oldtofile = tofile
if oldtofile.find("._cfg") != -1:
oldtofile = os.path.join(os.path.dirname(oldtofile),
os.path.basename(oldtofile)[10:])
if not do_quiet:
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Protecting config file: %s" % (oldtofile,)
)
mytxt = red("%s: %s") % (_("Protecting config file"), oldtofile,)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
return in_mask, protected, tofile, do_continue
def _handle_install_collision_protect(self, tofile, todbfile):
avail = self._entropy.installed_repository().isFileAvailable(
const_convert_to_unicode(todbfile), get_id = True)
if (self.pkgmeta['removepackage_id'] not in avail) and avail:
mytxt = darkred(_("Collision found during install for"))
mytxt += " %s - %s" % (
blue(tofile),
darkred(_("cannot overwrite")),
)
self._entropy.output(
red("QA: ")+mytxt,
importance = 1,
level = "warning",
header = darkred(" ## ")
)
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"WARNING!!! Collision found during install " \
"for %s - cannot overwrite" % (tofile,)
)
return False
return True
def _sources_fetch_step(self):
down_data = self.pkgmeta['download']
down_keys = list(down_data.keys())
d_cache = set()
rc = 0
key_cache = [os.path.basename(x) for x in down_keys]
for key in sorted(down_keys):
key_name = os.path.basename(key)
if key_name in d_cache:
continue
# first fine wins
keyboard_interrupt = False
for url in down_data[key]:
file_name = os.path.basename(url)
if self.pkgmeta.get('fetch_path'):
dest_file = os.path.join(self.pkgmeta['fetch_path'],
file_name)
else:
dest_file = os.path.join(self.pkgmeta['unpackdir'],
file_name)
try:
rc = self._fetch_source(url, dest_file)
except KeyboardInterrupt:
keyboard_interrupt = True
break
if rc == -100:
keyboard_interrupt = True
break
if rc == 0:
d_cache.add(key_name)
break
if keyboard_interrupt:
rc = 1
break
key_cache.remove(key_name)
if rc != 0 and key_name not in key_cache:
break
rc = 0
return rc
def _fetch_source(self, url, dest_file):
rc = 1
mytxt = "%s: %s" % (blue(_("Downloading")), brown(url),)
# now fetch the new one
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
rc, data_transfer, _resumed = self.__fetch_file(
url,
dest_file,
digest = None,
resume = False
)
if rc == 0:
mytxt = blue("%s: ") % (_("Successfully downloaded from"),)
mytxt += red(self._get_url_name(url))
human_bytes = entropy.tools.bytes_into_human(data_transfer)
mytxt += " %s %s/%s" % (_("at"), human_bytes, _("second"),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
self._entropy.output(
"%s: %s" % (blue(_("Local path")), brown(dest_file),),
importance = 1,
level = "info",
header = red(" # ")
)
else:
error_message = blue("%s: %s") % (
_("Error downloading from"),
red(self._get_url_name(url)),
)
# something bad happened
if rc == -1:
error_message += " - %s." % (
_("file not available on this mirror"),
)
elif rc == -3:
error_message += " - not found."
elif rc == -100:
error_message += " - %s." % (_("discarded download"),)
else:
error_message += " - %s: %s" % (_("unknown reason"), rc,)
self._entropy.output(
error_message,
importance = 1,
level = "warning",
header = red(" ## ")
)
return rc
def _fetch_step(self):
if self.pkgmeta['verified']:
return 0
def _fetch(download, checksum):
mytxt = "%s: %s" % (blue(_("Downloading")),
red(os.path.basename(download)),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
pkg_disk_path = self.__get_fetch_disk_path(download)
return self._download_package(
self.pkgmeta['package_id'],
self.pkgmeta['repository'],
download,
pkg_disk_path,
checksum
)
rc = _fetch(self.pkgmeta['download'], self.pkgmeta['checksum'])
if rc == 0:
# go ahead with extra_download
for extra_download in self.pkgmeta['extra_download']:
rc = _fetch(extra_download['download'],
extra_download['md5'])
if rc != 0:
break
if rc == 0:
return 0
mytxt = "%s. %s: %s" % (
red(_("Package cannot be fetched. Try to update repositories")),
blue(_("Error")),
rc,
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = darkred(" ## ")
)
return rc
def _multi_fetch_step(self):
m_fetch_len = len(self.pkgmeta['multi_fetch_list'])
mytxt = "%s: %s %s" % (
blue(_("Downloading")),
darkred(str(m_fetch_len)),
ngettext("archive", "archives", m_fetch_len),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
rc, err_list = self._download_packages(
self.pkgmeta['multi_fetch_list'],
self.pkgmeta['checksum']
)
if rc == 0:
return 0
mytxt = _("Some packages cannot be fetched")
mytxt2 = _("Try to update your repositories and retry")
mytxt3 = "%s: %s" % (brown(_("Error")), bold(str(rc)),)
for txt in (mytxt, mytxt2,):
self._entropy.output(
"%s." % (darkred(txt),),
importance = 0,
level = "info",
header = red(" ## ")
)
self._entropy.output(
mytxt3,
importance = 0,
level = "info",
header = red(" ## ")
)
for _pkg_id, repo, fname, cksum, _signatures in err_list:
self._entropy.output(
"[%s|%s] %s" % (blue(repo), darkgreen(cksum), darkred(fname),),
importance = 1,
level = "error",
header = darkred(" # ")
)
return rc
def _fetch_not_available_step(self):
self._entropy.output(
blue(_("Package cannot be downloaded, unknown error.")),
importance = 1,
level = "info",
header = red(" ## ")
)
return 0
def _vanished_step(self):
self._entropy.output(
blue(_("Installed package in queue vanished, skipping.")),
importance = 1,
level = "info",
header = red(" ## ")
)
return 0
def _checksum_step(self):
base_pkg_rc = self._match_checksum(self.pkgmeta['package_id'],
self.pkgmeta['repository'], self.pkgmeta['checksum'],
self.pkgmeta['download'], self.pkgmeta['signatures'])
if base_pkg_rc != 0:
return base_pkg_rc
# now go with extra_download
for extra_download in self.pkgmeta['extra_download']:
download = extra_download['download']
checksum = extra_download['md5']
signatures = {
'sha1': extra_download['sha1'],
'sha256': extra_download['sha256'],
'sha512': extra_download['sha512'],
'gpg': extra_download['gpg'],
}
extra_rc = self._match_checksum(self.pkgmeta['package_id'],
self.pkgmeta['repository'], checksum, download, signatures)
if extra_rc != 0:
return extra_rc
self.pkgmeta['verified'] = True
return 0
def _multi_checksum_step(self):
return self.multi_match_checksum()
def _merge_from_unpack_step(self):
mytxt = "%s: %s" % (
blue(_("Merging package")),
red(os.path.basename(self.pkgmeta['atom'])),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Merging package: %s" % (self.pkgmeta['atom'],)
)
self.__fill_image_dir(self.pkgmeta['merge_from'],
self.pkgmeta['imagedir'])
spm_class = self._entropy.Spm_class()
# call Spm unpack hook
return spm_class.entropy_install_unpack_hook(self._entropy,
self.pkgmeta)
def _unpack_step(self):
unpack_dir = self.pkgmeta['unpackdir']
if not const_is_python3():
# unpackdir comes from download metadatum, which is utf-8
# (conf_encoding)
unpack_dir = const_convert_to_rawstring(unpack_dir,
from_enctype = etpConst['conf_encoding'])
if os.path.isdir(unpack_dir):
# this, if Python 2.x, must be fed with rawstrings
shutil.rmtree(unpack_dir)
elif os.path.isfile(unpack_dir):
os.remove(unpack_dir)
os.makedirs(unpack_dir)
rc = self.__unpack_package(self.pkgmeta['download'],
self.pkgmeta['pkgpath'], self.pkgmeta['imagedir'],
self.pkgmeta['pkgdbpath'])
if rc == 0:
for extra_download in self.pkgmeta['extra_download']:
download = extra_download['download']
pkgpath = self.__get_fetch_disk_path(download)
rc = self.__unpack_package(download, pkgpath,
self.pkgmeta['imagedir'], None)
if rc != 0:
break
if rc != 0:
if rc == 512:
errormsg = "%s. %s. %s: 512" % (
red(_("You are running out of disk space")),
red(_("I bet, you're probably Michele")),
blue(_("Error")),
)
else:
msg = _("An error occured while trying to unpack the package")
errormsg = "%s. %s. %s: %s" % (
red(msg),
red(_("Check if your system is healthy")),
blue(_("Error")),
rc,
)
self._entropy.output(
errormsg,
importance = 1,
level = "error",
header = red(" ## ")
)
return rc
spm_class = self._entropy.Spm_class()
# call Spm unpack hook
return spm_class.entropy_install_unpack_hook(self._entropy,
self.pkgmeta)
def _install_step(self):
mytxt = "%s: %s" % (
blue(_("Installing package")),
red(self.pkgmeta['atom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
if self.pkgmeta.get('description'):
mytxt = "[%s]" % (purple(self.pkgmeta.get('description')),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
if self.pkgmeta['splitdebug']:
if self.pkgmeta.get('splitdebug_pkgfile'):
mytxt = "[%s]" % (
teal(_("unsupported splitdebug usage (package files)")),)
level = "warning"
else:
mytxt = "[%s]" % (
teal(_("<3 debug files installation enabled <3")),)
level = "info"
self._entropy.output(
mytxt,
importance = 1,
level = level,
header = red(" ## ")
)
rc = self.__install_package()
if rc != 0:
mytxt = "%s. %s. %s: %s" % (
red(_("An error occured while trying to install the package")),
red(_("Check if your system is healthy")),
blue(_("Error")),
rc,
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = red(" ## ")
)
return rc
def _remove_step(self):
mytxt = "%s: %s" % (
blue(_("Removing data")),
red(self.pkgmeta['removeatom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
rc = self.__remove_package()
if rc != 0:
mytxt = "%s. %s. %s: %s" % (
red(_("An error occured while trying to remove the package")),
red(_("Check if you have enough disk space on your hard disk")),
blue(_("Error")),
rc,
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = red(" ## ")
)
return rc
def _cleanup_step(self):
mytxt = "%s: %s" % (
blue(_("Cleaning")),
red(self.pkgmeta['atom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
self._cleanup_package(self.pkgmeta['unpackdir'])
# we don't care if cleanupPackage fails since it's not critical
return 0
def _post_install_step(self):
pkgdata = self.pkgmeta['triggers'].get('install')
action_data = self.pkgmeta['triggers'].get('install')
code = 0
if pkgdata:
trigger = self._entropy.Triggers(
self._action, "postinstall",
pkgdata, action_data)
do = trigger.prepare()
if do:
code = trigger.run()
trigger.kill()
del pkgdata
return code
def _pre_install_step(self):
pkgdata = self.pkgmeta['triggers'].get('install')
action_data = self.pkgmeta['triggers'].get('install')
code = 0
if pkgdata:
trigger = self._entropy.Triggers(
self._action, "preinstall",
pkgdata, action_data)
do = trigger.prepare()
if do:
code = trigger.run()
trigger.kill()
del pkgdata
return code
def _setup_step(self):
pkgdata = self.pkgmeta['triggers'].get('install')
action_data = self.pkgmeta['triggers'].get('install')
code = 0
if pkgdata:
trigger = self._entropy.Triggers(
self._action, "setup",
pkgdata, action_data)
do = trigger.prepare()
if do:
code = trigger.run()
trigger.kill()
del pkgdata
if code != 0:
return code
# NOTE: fixup permissions in the image directory
# the setup phase could have created additional users and groups
package_path = self.pkgmeta['pkgpath']
prefix_dir = self.pkgmeta['imagedir']
try:
entropy.tools.apply_tarball_ownership(package_path, prefix_dir)
except IOError as err:
msg = "%s: %s" % (
brown(_("Error during package files permissions setup")),
err,)
self._entropy.output(
msg,
importance = 1,
level = "error",
header = darkred(" !!! ")
)
return 1
return 0
def _pre_remove_step(self):
remdata = self.pkgmeta['triggers'].get('remove')
action_data = self.pkgmeta['triggers'].get('install')
code = 0
if remdata:
trigger = self._entropy.Triggers(
self._action, 'preremove', remdata,
action_data)
do = trigger.prepare()
if do:
code = trigger.run()
trigger.kill()
del remdata
return code
def _package_install_clean(self):
"""
Cleanup package files not used anymore by newly installed version.
This is part of the atomic install, which overwrites the live fs with
new files and removes old afterwards.
"""
self._entropy.output(
blue(_("Cleaning previously installed application data.")),
importance = 1,
level = "info",
header = red(" ## ")
)
self._remove_content_from_system(
automerge_metadata = self.pkgmeta['already_protected_config_files'])
return 0
def _post_remove_step_install(self):
"""
Post-remove phase of package install, this step removes older SPM
package entries from SPM db.
"""
self._entropy.logger.log(
"[Package]",
etpConst['logging']['normal_loglevel_id'],
"Remove old package (spm data): %s" % (self.pkgmeta['removeatom'],)
)
return self._spm_remove_package()
def _post_remove_step_remove(self):
"""
Post-remove phase of package remove action, this step removes SPM
package entries if there are no other Entropy-tagged packages installed.
"""
# remove pkg
# -- now it's possible to remove SPM package entry.
# if another package with the same atom is installed in
# Entropy db, do not call SPM at all because it would cause
# to get that package removed from there resulting in matching
# inconsistencies.
# -- of course, we need to drop versiontag before being able to look
# for other pkgs with same atom but different tag (which is an
# entropy-only metadatum)
test_atom = entropy.dep.remove_tag(self.pkgmeta['removeatom'])
others_installed = self._entropy.installed_repository().getPackageIds(
test_atom)
# It's obvious that clientdb cannot have more than one package_id
# featuring the same "atom" value, but still, let's be fault-tolerant.
spm_rc = 0
if not others_installed:
spm_rc = self._spm_remove_package()
for other_installed in others_installed:
# we have one installed, we need to update SPM uid
spm_rc = self._spm_update_package_uid(other_installed,
self.pkgmeta['removeatom'])
if spm_rc != 0:
break # ohi ohi ohi
return spm_rc
def _post_remove_step(self):
remdata = self.pkgmeta['triggers'].get('remove')
action_data = self.pkgmeta['triggers'].get('install')
code = 0
if remdata:
trigger = self._entropy.Triggers(
self._action, "postremove",
remdata, action_data)
do = trigger.prepare()
if do:
code = trigger.run()
trigger.kill()
del remdata
return code
def _removeconflict_step(self):
inst_repo = self._entropy.installed_repository()
confl_package_ids = [x for x in self.pkgmeta['conflicts'] if \
inst_repo.isPackageIdAvailable(x)]
if not confl_package_ids:
return 0
# calculate removal dependencies
# system_packages must be False because we should not exclude
# them from the dependency tree in any case. Also, we cannot trigger
# DependenciesNotRemovable() exception, too.
proposed_pkg_ids = self._entropy.get_removal_queue(confl_package_ids,
system_packages = False)
# we don't want to remove the whole inverse dependencies of course,
# but just the conflicting ones, in a proper order
package_ids = [x for x in proposed_pkg_ids if x in confl_package_ids]
# make sure that every package is listed in package_ids before
# proceeding, cannot keep packages behind anyway, and must be fault
# tolerant. Besides, having missing packages here should never happen.
package_ids += [x for x in confl_package_ids if x not in \
package_ids]
for package_id in package_ids:
pkg = self._entropy.Package()
pkg.prepare((package_id,), "remove_conflict",
self.pkgmeta['remove_metaopts'])
rc = pkg.run(xterm_header = self._xterm_title)
pkg.kill()
if rc != 0:
return rc
return 0
def _config_step(self):
mytxt = "%s: %s" % (
blue(_("Configuring package")),
red(self.pkgmeta['atom']),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = red(" ## ")
)
conf_rc = self.__configure_package()
if conf_rc == 1:
mytxt = _("An error occured while trying to configure the package")
mytxt2 = "%s. %s: %s" % (
red(_("Make sure that your system is healthy")),
blue(_("Error")),
conf_rc,
)
self._entropy.output(
darkred(mytxt),
importance = 1,
level = "error",
header = red(" ## ")
)
self._entropy.output(
mytxt2,
importance = 1,
level = "error",
header = red(" ## ")
)
elif conf_rc == 2:
mytxt = _("An error occured while trying to configure the package")
mytxt2 = "%s. %s: %s" % (
red(_("It seems that Source Package Manager entry is missing")),
blue(_("Error")),
conf_rc,
)
self._entropy.output(
darkred(mytxt),
importance = 1,
level = "error",
header = red(" ## ")
)
self._entropy.output(
mytxt2,
importance = 1,
level = "error",
header = red(" ## ")
)
return conf_rc
def _stepper(self, xterm_header):
if xterm_header is None:
xterm_header = ""
if 'remove_installed_vanished' in self.pkgmeta:
self._xterm_title += ' %s' % (_("Installed package vanished"),)
self._entropy.set_title(self._xterm_title)
return self._vanished_step()
if 'fetch_not_available' in self.pkgmeta:
self._xterm_title += ' %s' % (_("Fetch not available"),)
self._entropy.set_title(self._xterm_title)
return self._fetch_not_available_step()
def do_fetch():
self._xterm_title += ' %s: %s' % (
_("Fetching"),
os.path.basename(self.pkgmeta['download']),
)
self._entropy.set_title(self._xterm_title)
return self._fetch_step()
def do_multi_fetch():
m_fetch_len = len(self.pkgmeta['multi_fetch_list']) / 2
self._xterm_title += ' %s: %s %s' % (_("Multi Fetching"),
m_fetch_len, ngettext("package", "packages", m_fetch_len),)
self._entropy.set_title(self._xterm_title)
return self._multi_fetch_step()
def do_sources_fetch():
self._xterm_title += ' %s: %s' % (
_("Fetching sources"),
os.path.basename(self.pkgmeta['atom']),)
self._entropy.set_title(self._xterm_title)
return self._sources_fetch_step()
def do_checksum():
self._xterm_title += ' %s: %s' % (_("Verifying"),
os.path.basename(self.pkgmeta['download']),)
self._entropy.set_title(self._xterm_title)
return self._checksum_step()
def do_multi_checksum():
m_checksum_len = len(self.pkgmeta['multi_checksum_list'])
self._xterm_title += ' %s: %s %s' % (
_("Multi Verification"),
m_checksum_len,
ngettext("package", "packages", m_checksum_len),)
self._entropy.set_title(self._xterm_title)
return self._multi_checksum_step()
def do_unpack():
if self.pkgmeta['merge_from']:
mytxt = _("Merging")
self._xterm_title += ' %s: %s' % (
mytxt,
os.path.basename(self.pkgmeta['atom']),
)
self._entropy.set_title(self._xterm_title)
return self._merge_from_unpack_step()
mytxt = _("Unpacking")
self._xterm_title += ' %s: %s' % (
mytxt,
os.path.basename(self.pkgmeta['download']),
)
self._entropy.set_title(self._xterm_title)
return self._unpack_step()
def do_install():
self._xterm_title += ' %s: %s' % (
_("Installing"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._install_step()
def do_install_spm():
return self._spm_install_package(
self.pkgmeta['installed_package_id'])
def do_remove():
self._xterm_title += ' %s: %s' % (
_("Removing"),
self.pkgmeta['removeatom'],
)
self._entropy.set_title(self._xterm_title)
return self._remove_step()
def do_cleanup():
self._xterm_title += ' %s: %s' % (
_("Cleaning"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._cleanup_step()
def do_postinstall():
self._xterm_title += ' %s: %s' % (
_("Postinstall"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._post_install_step()
def do_setup():
self._xterm_title += ' %s: %s' % (
_("Setup"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._setup_step()
def do_preinstall():
self._xterm_title += ' %s: %s' % (
_("Preinstall"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._pre_install_step()
def do_preremove():
self._xterm_title += ' %s: %s' % (
_("Preremove"),
self.pkgmeta['removeatom'],
)
self._entropy.set_title(self._xterm_title)
return self._pre_remove_step()
def do_postremove():
self._xterm_title += ' %s: %s' % (
_("Postremove"),
self.pkgmeta['removeatom'],
)
self._entropy.set_title(self._xterm_title)
return self._post_remove_step()
def do_config():
self._xterm_title += ' %s: %s' % (
_("Configuring"),
self.pkgmeta['atom'],
)
self._entropy.set_title(self._xterm_title)
return self._config_step()
steps_data = {
"fetch": do_fetch,
"multi_fetch": do_multi_fetch,
"multi_checksum": do_multi_checksum,
"sources_fetch": do_sources_fetch,
"checksum": do_checksum,
"unpack": do_unpack,
"remove_conflicts": self._removeconflict_step,
"install": do_install,
"install_spm": do_install_spm,
"install_clean": self._package_install_clean,
"remove": do_remove,
"cleanup": do_cleanup,
"postinstall": do_postinstall,
"setup": do_setup,
"preinstall": do_preinstall,
"postremove": do_postremove,
"postremove_install": self._post_remove_step_install,
"postremove_remove": self._post_remove_step_remove,
"preremove": do_preremove,
"config": do_config,
}
rc = 0
for step in self.pkgmeta['steps']:
self._xterm_title = xterm_header
rc = steps_data.get(step)()
if rc != 0:
break
return rc
def run(self, xterm_header = None):
self._error_on_not_prepared()
gave_up = self._entropy.wait_resources()
if gave_up:
return 20
# resources acquired
try:
rc = self._stepper(xterm_header)
finally:
self._entropy.unlock_resources()
if rc != 0:
self._entropy.output(
blue(_("An error occured. Action aborted.")),
importance = 2,
level = "error",
header = darkred(" ## ")
)
return rc
def prepare(self, package_match, action, metaopts = None):
self._error_on_prepared()
self._check_action_validity(action)
self._action = action
self._package_match = package_match
if metaopts is None:
metaopts = {}
self.metaopts = metaopts
# generate metadata dictionary
self._generate_metadata()
@classmethod
def splitdebug_enabled(cls, entropy_client, pkg_match):
"""
Return whether splitdebug is enabled for package.
"""
settings = entropy_client.Settings()
# this is a SystemSettings.CachingList object
splitdebug = settings['splitdebug']
splitdebug_mask = settings['splitdebug_mask']
_pkg_id, pkg_repo = pkg_match
def _generate_cache(lst_obj):
# compute the package matching then
pkg_matches = set()
for dep in lst_obj:
dep, repo_ids = entropy.dep.dep_get_match_in_repos(dep)
if repo_ids is not None:
if pkg_repo not in repo_ids:
# skip entry, not me
continue
dep_matches, _rc = entropy_client.atom_match(
dep, multi_match=True, multi_repo=True)
pkg_matches |= dep_matches
# set cache back
lst_obj.set(pkg_matches)
return pkg_matches
enabled = False
if not splitdebug:
# no entries, consider splitdebug always enabled
enabled = True
else:
# whitelist support
pkg_matches = splitdebug.get()
if pkg_matches is None:
pkg_matches = _generate_cache(splitdebug)
# determine if it's enabled then
enabled = pkg_match in pkg_matches
# if it's enabled, check whether it's blacklisted
if enabled:
# blacklist support
pkg_matches = splitdebug_mask.get()
if pkg_matches is None:
# compute the package matching
pkg_matches = _generate_cache(splitdebug_mask)
enabled = pkg_match not in pkg_matches
return enabled
def _package_splitdebug_enabled(self, pkg_match):
"""
Determine if splitdebug is enabled for the package being installed
or just fetched. This method should be called only if system-wide
splitdebug setting in client.conf is enabled already.
"""
return Package.splitdebug_enabled(self._entropy, pkg_match)
def __get_base_metadata(self, _action):
def get_splitdebug_data():
sys_set_plg_id = \
etpConst['system_settings_plugins_ids']['client_plugin']
misc_data = self._settings[sys_set_plg_id]['misc']
splitdebug = misc_data['splitdebug']
splitdebug_dirs = misc_data['splitdebug_dirs']
return splitdebug, splitdebug_dirs
splitdebug, splitdebug_dirs = get_splitdebug_data()
metadata = {
'splitdebug': splitdebug,
'splitdebug_dirs': splitdebug_dirs,
'__content_files__': [],
}
return metadata
def _generate_metadata(self):
self._error_on_prepared()
self._check_action_validity(self._action)
self.pkgmeta.clear()
self.pkgmeta.update(self.__get_base_metadata(self._action))
if self._action == "fetch":
self.__generate_fetch_metadata()
elif self._action == "multi_fetch":
self.__generate_multi_fetch_metadata()
elif self._action in ("remove", "remove_conflict"):
self.__generate_remove_metadata()
elif self._action == "install":
self.__generate_install_metadata()
elif self._action == "source":
self.__generate_fetch_metadata(sources = True)
elif self._action == "config":
self.__generate_config_metadata()
self.__prepared = True
@staticmethod
def _generate_content_safety_file(content_safety):
"""
Generate a file containing the "content_safety" metadata,
reading by content_safety list or iterator. Each item
of "content_safety" must contain (path, sha256, mtime).
Each item shall be written to file, one per line,
in the following form: "<mtime>|<sha256>|<path>".
The order of the element in "content_safety" will be kept.
"""
tmp_dir = os.path.join(
etpConst['entropyunpackdir'],
"__generate_content_safety_file_f")
try:
os.makedirs(tmp_dir, 0o755)
except OSError as err:
if err.errno != errno.EEXIST:
raise
tmp_fd, tmp_path = None, None
generated = False
try:
tmp_fd, tmp_path = const_mkstemp(
prefix="PackageContentSafety",
dir=tmp_dir)
with Package.FileContentSafetyWriter(tmp_fd) as tmp_f:
for path, sha256, mtime in content_safety:
tmp_f.write(path, sha256, mtime)
generated = True
return tmp_path
finally:
if tmp_fd is not None:
try:
os.close(tmp_fd)
except OSError:
pass
if tmp_path is not None and not generated:
try:
os.remove(tmp_path)
except (OSError, IOError):
pass
@staticmethod
def _generate_content_file(content, package_id = None,
filter_splitdebug = False,
splitdebug = None,
splitdebug_dirs = None):
"""
Generate a file containing the "content" metadata,
reading by content list or iterator. Each item
of "content" must contain (path, ftype).
Each item shall be written to file, one per line,
in the following form: "[<package_id>|]<ftype>|<path>".
The order of the element in "content" will be kept.
"""
tmp_dir = os.path.join(
etpConst['entropyunpackdir'],
"__generate_content_file_f")
try:
os.makedirs(tmp_dir, 0o755)
except OSError as err:
if err.errno != errno.EEXIST:
raise
tmp_fd, tmp_path = None, None
generated = False
try:
tmp_fd, tmp_path = const_mkstemp(
prefix="PackageContent",
dir=tmp_dir)
with Package.FileContentWriter(tmp_fd) as tmp_f:
for path, ftype in content:
if filter_splitdebug and not splitdebug:
# if filter_splitdebug is enabled, this
# code filters out all the paths starting
# with splitdebug_dirs, if splitdebug is
# disabled for package.
_skip = False
for split_dir in splitdebug_dirs:
if path.startswith(split_dir):
_skip = True
break
if _skip:
continue
tmp_f.write(package_id, path, ftype)
generated = True
return tmp_path
finally:
if tmp_fd is not None:
try:
os.close(tmp_fd)
except OSError:
pass
if tmp_path is not None and not generated:
try:
os.remove(tmp_path)
except (OSError, IOError):
pass
@staticmethod
def _merge_content_file(content_file, sorted_content,
cmp_func):
"""
Given a sorted content_file content and a sorted list of
content (sorted_content), apply the "merge" step of a merge
sort algorithm. In other words, add the sorted_content to
content_file keeping content_file content ordered.
It is of couse O(n+m) where n = lines in content_file and
m = sorted_content length.
"""
tmp_content_file = content_file + \
Package.FileContentWriter.TMP_SUFFIX
sorted_ptr = 0
_sorted_path = None
_sorted_ftype = None
_package_id = 0 # will be filled
try:
with Package.FileContentWriter(tmp_content_file) as tmp_w:
with Package.FileContentReader(content_file) as tmp_r:
for _package_id, _path, _ftype in tmp_r:
while True:
try:
_sorted_path, _sorted_ftype = \
sorted_content[sorted_ptr]
except IndexError:
_sorted_path = None
_sorted_ftype = None
if _sorted_path is None:
tmp_w.write(_package_id, _path, _ftype)
break
cmp_outcome = cmp_func(_path, _sorted_path)
if cmp_outcome < 0:
tmp_w.write(_package_id, _path, _ftype)
break
# always privilege _ftype over _sorted_ftype
# _sorted_ftype might be invalid
tmp_w.write(
_package_id, _sorted_path, _ftype)
sorted_ptr += 1
if cmp_outcome == 0:
# write only one
break
# add the remainder
if sorted_ptr < len(sorted_content):
_sorted_rem = sorted_content[sorted_ptr:]
for _sorted_path, _sorted_ftype in _sorted_rem:
tmp_w.write(
_package_id, _sorted_path, _sorted_ftype)
os.rename(tmp_content_file, content_file)
finally:
try:
os.remove(tmp_content_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise
@staticmethod
def _filter_content_file(content_file, filter_func):
"""
This method rewrites the content of content_file by applying
a filter to the path elements.
"""
tmp_content_file = content_file + \
Package.FileContentWriter.TMP_SUFFIX
try:
with Package.FileContentWriter(tmp_content_file) as tmp_w:
with Package.FileContentReader(content_file) as tmp_r:
for _package_id, _path, _ftype in tmp_r:
if filter_func(_path):
tmp_w.write(_package_id, _path, _ftype)
os.rename(tmp_content_file, content_file)
finally:
try:
os.remove(tmp_content_file)
except OSError as err:
if err.errno != errno.ENOENT:
raise
def __generate_content_file(self, content, package_id = None,
filter_splitdebug = False,
splitdebug = None,
splitdebug_dirs = None):
content_path = None
try:
content_path = Package._generate_content_file(
content, package_id = package_id,
filter_splitdebug = filter_splitdebug,
splitdebug = splitdebug,
splitdebug_dirs = splitdebug_dirs)
return content_path
finally:
if content_path is not None:
self.pkgmeta['__content_files__'].append(
content_path)
def __generate_content_safety_file(self, content_safety):
content_path = None
try:
content_path = Package._generate_content_safety_file(
content_safety)
return content_path
finally:
if content_path is not None:
self.pkgmeta['__content_files__'].append(
content_path)
def __generate_remove_metadata(self):
package_id = self._package_match[0]
inst_repo = self._entropy.installed_repository()
if not inst_repo.isPackageIdAvailable(package_id):
self.pkgmeta['remove_installed_vanished'] = True
return 0
self.pkgmeta['package_id'] = package_id
self.pkgmeta['removepackage_id'] = package_id
self.pkgmeta['configprotect_data'] = []
self.pkgmeta['triggers'] = {}
self.pkgmeta['removeatom'] = \
inst_repo.retrieveAtom(package_id)
self.pkgmeta['slot'] = \
inst_repo.retrieveSlot(package_id)
self.pkgmeta['versiontag'] = \
inst_repo.retrieveTag(package_id)
remove_config = False
if 'removeconfig' in self.metaopts:
remove_config = self.metaopts.get('removeconfig')
self.pkgmeta['removeconfig'] = remove_config
content = inst_repo.retrieveContentIter(
package_id, order_by="file", reverse=True)
self.pkgmeta['removecontent_file'] = \
self.__generate_content_file(content)
# collects directories whose content has been modified
# this information is then handed to the Trigger
self.pkgmeta['affected_directories'] = set()
self.pkgmeta['affected_infofiles'] = set()
# this will be set before calling removePackage() if we have
# a package to remove
self.pkgmeta['config_protect+mask'] = None
self.pkgmeta['triggers']['remove'] = \
inst_repo.getTriggerData(package_id)
if self.pkgmeta['triggers']['remove'] is None:
self.pkgmeta['remove_installed_vanished'] = True
return 0
self.pkgmeta['triggers']['remove']['affected_directories'] = \
self.pkgmeta['affected_directories']
self.pkgmeta['triggers']['remove']['affected_infofiles'] = \
self.pkgmeta['affected_infofiles']
self.pkgmeta['triggers']['remove']['spm_repository'] = \
inst_repo.retrieveSpmRepository(
package_id)
self.pkgmeta['triggers']['remove'].update(
self.__get_base_metadata(self._action))
pkg_license = inst_repo.retrieveLicense(
package_id)
if pkg_license is None:
pkg_license = set()
else:
pkg_license = set(pkg_license.split())
self.pkgmeta['triggers']['remove']['accept_license'] = pkg_license
self.pkgmeta['steps'] = ["preremove", "remove", "postremove",
"postremove_remove"]
return 0
def __generate_config_metadata(self):
package_id = self._package_match[0]
inst_repo = self._entropy.installed_repository()
self.pkgmeta['atom'] = inst_repo.retrieveAtom(package_id)
key, slot = inst_repo.retrieveKeySlot(package_id)
self.pkgmeta['key'], self.pkgmeta['slot'] = key, slot
self.pkgmeta['version'] = inst_repo.retrieveVersion(package_id)
self.pkgmeta['category'] = inst_repo.retrieveCategory(package_id)
self.pkgmeta['name'] = inst_repo.retrieveName(package_id)
self.pkgmeta['spm_repository'] = inst_repo.retrieveSpmRepository(
package_id)
pkg_license = inst_repo.retrieveLicense(package_id)
if pkg_license is None:
pkg_license = set()
else:
pkg_license = set(pkg_license.split())
self.pkgmeta['accept_license'] = pkg_license
self.pkgmeta['steps'] = []
self.pkgmeta['steps'].append("config")
return 0
def __get_match_conflicts(self, match):
m_id, m_repo = match
dbconn = self._entropy.open_repository(m_repo)
conflicts = dbconn.retrieveConflicts(m_id)
found_conflicts = set()
inst_repo = self._entropy.installed_repository()
for conflict in conflicts:
my_m_id, _my_m_rc = inst_repo.atomMatch(conflict)
if my_m_id != -1:
# check if the package shares the same slot
match_data = dbconn.retrieveKeySlot(m_id)
installed_match_data = inst_repo.retrieveKeySlot(my_m_id)
if match_data != installed_match_data:
found_conflicts.add(my_m_id)
# auto conflicts support
found_conflicts |= self._entropy._generate_dependency_inverse_conflicts(
match, just_id=True)
return found_conflicts
def __setup_package_to_remove(self, package_key, slot):
inst_repo = self._entropy.installed_repository()
inst_package_id, _inst_rc = inst_repo.atomMatch(package_key,
matchSlot = slot)
if inst_package_id != -1:
avail = inst_repo.isPackageIdAvailable(inst_package_id)
if avail:
inst_atom = inst_repo.retrieveAtom(inst_package_id)
self.pkgmeta['removeatom'] = inst_atom
else:
inst_package_id = -1
return inst_package_id
def __generate_install_metadata(self):
package_id, repository = self._package_match
inst_repo = self._entropy.installed_repository()
self.pkgmeta['package_id'] = package_id
self.pkgmeta['repository'] = repository
cl_id = etpConst['system_settings_plugins_ids']['client_plugin']
edelta_support = self._settings[cl_id]['misc']['edelta_support']
self.pkgmeta['edelta_support'] = edelta_support
is_package_repo = repository.endswith(etpConst['packagesext'])
# if splitdebug is enabled, check if it's also enabled
# via package.splitdebug
if self.pkgmeta['splitdebug']:
# yeah, this has to affect exported splitdebug setting
# because it is read during package files installation
# Older splitdebug data was in the same package file of
# the actual content. Later on, splitdebug data was moved
# to its own package file that gets downloaded and unpacked
# only if required (if splitdebug is enabled)
self.pkgmeta['splitdebug'] = self._package_splitdebug_enabled(
self._package_match)
# fetch abort function
self.pkgmeta['fetch_abort_function'] = \
self.metaopts.get('fetch_abort_function')
install_source = etpConst['install_sources']['unknown']
meta_inst_source = self.metaopts.get('install_source', install_source)
if meta_inst_source in list(etpConst['install_sources'].values()):
install_source = meta_inst_source
self.pkgmeta['install_source'] = install_source
self.pkgmeta['already_protected_config_files'] = {}
self.pkgmeta['configprotect_data'] = []
dbconn = self._entropy.open_repository(repository)
self.pkgmeta['triggers'] = {}
self.pkgmeta['atom'] = dbconn.retrieveAtom(package_id)
self.pkgmeta['slot'] = dbconn.retrieveSlot(package_id)
ver, tag, rev = dbconn.getVersioningData(package_id)
self.pkgmeta['version'] = ver
self.pkgmeta['versiontag'] = tag
self.pkgmeta['revision'] = rev
self.pkgmeta['extra_download'] = []
self.pkgmeta['splitdebug_pkgfile'] = True
if not is_package_repo:
self.pkgmeta['splitdebug_pkgfile'] = False
extra_download = dbconn.retrieveExtraDownload(package_id)
if not self.pkgmeta['splitdebug']:
extra_download = [x for x in extra_download if \
x['type'] != "debug"]
self.pkgmeta['extra_download'] += extra_download
self.pkgmeta['category'] = dbconn.retrieveCategory(package_id)
self.pkgmeta['download'] = dbconn.retrieveDownloadURL(package_id)
self.pkgmeta['name'] = dbconn.retrieveName(package_id)
self.pkgmeta['checksum'] = dbconn.retrieveDigest(package_id)
sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id)
signatures = {
'sha1': sha1,
'sha256': sha256,
'sha512': sha512,
'gpg': gpg,
}
self.pkgmeta['signatures'] = signatures
self.pkgmeta['conflicts'] = self.__get_match_conflicts(
self._package_match)
description = dbconn.retrieveDescription(package_id)
if description:
if len(description) > 74:
description = description[:74].strip()
description += "..."
self.pkgmeta['description'] = description
# this is set by __install_package() and required by spm_install
# phase
self.pkgmeta['installed_package_id'] = None
# fill action queue
self.pkgmeta['removepackage_id'] = -1
remove_config = False
if 'removeconfig' in self.metaopts:
remove_config = self.metaopts.get('removeconfig')
self.pkgmeta['remove_metaopts'] = {
'removeconfig': True,
}
if 'remove_metaopts' in self.metaopts:
self.pkgmeta['remove_metaopts'] = \
self.metaopts.get('remove_metaopts')
self.pkgmeta['merge_from'] = None
mf = self.metaopts.get('merge_from')
if mf != None:
self.pkgmeta['merge_from'] = const_convert_to_unicode(mf)
self.pkgmeta['removeconfig'] = remove_config
self.pkgmeta['removepackage_id'] = self.__setup_package_to_remove(
entropy.dep.dep_getkey(self.pkgmeta['atom']),
self.pkgmeta['slot'])
# collects directories whose content has been modified
# this information is then handed to the Trigger
self.pkgmeta['affected_directories'] = set()
self.pkgmeta['affected_infofiles'] = set()
# smartpackage ?
self.pkgmeta['smartpackage'] = False
# set unpack dir and image dir
if is_package_repo:
try:
compiled_arch = dbconn.getSetting("arch")
arch_fine = compiled_arch == etpConst['currentarch']
except KeyError:
arch_fine = True # sorry, old db, cannot check
if not arch_fine:
self.__prepared = False
return -1
repo_data = self._settings['repositories']
repo_meta = repo_data['available'][self.pkgmeta['repository']]
self.pkgmeta['smartpackage'] = repo_meta['smartpackage']
self.pkgmeta['pkgpath'] = repo_meta['pkgpath']
else:
self.pkgmeta['pkgpath'] = self.__get_fetch_disk_path(
self.pkgmeta['download'])
self.pkgmeta['unpackdir'] = etpConst['entropyunpackdir'] + \
os.path.sep + self.__escape_path(self.pkgmeta['download'])
self.pkgmeta['imagedir'] = self.pkgmeta['unpackdir'] + os.path.sep + \
etpConst['entropyimagerelativepath']
self.pkgmeta['pkgdbpath'] = os.path.join(self.pkgmeta['unpackdir'],
"edb/pkg.db")
# this will be set before calling handlePackage()
# if we have a package to remove
self.pkgmeta['config_protect+mask'] = None
if self.pkgmeta['removepackage_id'] == -1:
# nothing to remove, fresh install
self.pkgmeta['removecontent_file'] = None
else:
# generate content file
content = inst_repo.retrieveContentIter(
self.pkgmeta['removepackage_id'],
order_by="file", reverse=True)
self.pkgmeta['removecontent_file'] = \
self.__generate_content_file(content)
# There is a pkg to remove, but...
# compare both versions and if they match, disable removepackage_id
trigger_data = inst_repo.getTriggerData(
self.pkgmeta['removepackage_id'])
if trigger_data is None:
# installed repository entry is corrupted
self.pkgmeta['removepackage_id'] = -1
else:
self.pkgmeta['removeatom'] = inst_repo.retrieveAtom(
self.pkgmeta['removepackage_id'])
self.pkgmeta['triggers']['remove'] = trigger_data
# pass reference, not copy! nevva!
self.pkgmeta['triggers']['remove']['affected_directories'] = \
self.pkgmeta['affected_directories']
self.pkgmeta['triggers']['remove']['affected_infofiles'] = \
self.pkgmeta['affected_infofiles']
self.pkgmeta['triggers']['remove']['spm_repository'] = \
inst_repo.retrieveSpmRepository(package_id)
self.pkgmeta['triggers']['remove'].update(
self.__get_base_metadata(self._action))
pkg_rm_license = inst_repo.retrieveLicense(
self.pkgmeta['removepackage_id'])
if pkg_rm_license is None:
pkg_rm_license = set()
else:
pkg_rm_license = set(pkg_rm_license.split())
self.pkgmeta['triggers']['remove']['accept_license'] = \
pkg_rm_license
# set steps
self.pkgmeta['steps'] = []
if self.pkgmeta['conflicts']:
self.pkgmeta['steps'].append("remove_conflicts")
# install
self.pkgmeta['steps'].append("unpack")
# preinstall placed before preremove in order
# to respect Spm order
self.pkgmeta['steps'].append("setup")
self.pkgmeta['steps'].append("preinstall")
self.pkgmeta['steps'].append("install")
if self.pkgmeta['removepackage_id'] != -1:
self.pkgmeta['steps'].append("preremove")
self.pkgmeta['steps'].append("install_clean")
if self.pkgmeta['removepackage_id'] != -1:
self.pkgmeta['steps'].append("postremove")
self.pkgmeta['steps'].append("postremove_install")
self.pkgmeta['steps'].append("install_spm")
self.pkgmeta['steps'].append("postinstall")
self.pkgmeta['steps'].append("cleanup")
self.pkgmeta['triggers']['install'] = dbconn.getTriggerData(package_id)
if self.pkgmeta['triggers']['install'] is None:
# wtf!?
return 1
pkg_license = dbconn.retrieveLicense(package_id)
if pkg_license is None:
pkg_license = set()
else:
pkg_license = set(pkg_license.split())
self.pkgmeta['accept_license'] = pkg_license
self.pkgmeta['triggers']['install']['accept_license'] = pkg_license
self.pkgmeta['triggers']['install']['unpackdir'] = \
self.pkgmeta['unpackdir']
self.pkgmeta['triggers']['install']['imagedir'] = \
self.pkgmeta['imagedir']
self.pkgmeta['triggers']['install']['spm_repository'] = \
dbconn.retrieveSpmRepository(package_id)
self.pkgmeta['triggers']['install'].update(
self.__get_base_metadata(self._action))
spm_class = self._entropy.Spm_class()
# call Spm setup hook
return spm_class.entropy_install_setup_hook(self._entropy, self.pkgmeta)
def __generate_fetch_metadata(self, sources = False):
package_id, repository = self._package_match
dochecksum = True
# fetch abort function
self.pkgmeta['fetch_abort_function'] = \
self.metaopts.get('fetch_abort_function')
if 'dochecksum' in self.metaopts:
dochecksum = self.metaopts.get('dochecksum')
# NOTE: if you want to implement download-to-dir feature in your
# client, you've found what you were looking for.
# fetch_path is the path where data should be downloaded
# it overrides default path
if 'fetch_path' in self.metaopts:
fetch_path = self.metaopts.get('fetch_path')
if entropy.tools.is_valid_path(fetch_path):
self.pkgmeta['fetch_path'] = fetch_path
# if splitdebug is enabled, check if it's also enabled
# via package.splitdebug
splitdebug = self.pkgmeta['splitdebug']
if splitdebug:
splitdebug = self._package_splitdebug_enabled(
self._package_match)
self.pkgmeta['repository'] = repository
self.pkgmeta['package_id'] = package_id
dbconn = self._entropy.open_repository(repository)
self.pkgmeta['atom'] = dbconn.retrieveAtom(package_id)
self.pkgmeta['slot'] = dbconn.retrieveSlot(package_id)
self.pkgmeta['removepackage_id'] = self.__setup_package_to_remove(
entropy.dep.dep_getkey(self.pkgmeta['atom']), self.pkgmeta['slot'])
if sources:
self.pkgmeta['edelta_support'] = False
self.pkgmeta['extra_download'] = tuple()
self.pkgmeta['download'] = dbconn.retrieveSources(package_id,
extended = True)
# fake path, don't use
self.pkgmeta['pkgpath'] = etpConst['entropypackagesworkdir']
else:
cl_id = etpConst['system_settings_plugins_ids']['client_plugin']
edelta_support = self._settings[cl_id]['misc']['edelta_support']
self.pkgmeta['edelta_support'] = edelta_support
self.pkgmeta['checksum'] = dbconn.retrieveDigest(package_id)
sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id)
signatures = {
'sha1': sha1,
'sha256': sha256,
'sha512': sha512,
'gpg': gpg,
}
self.pkgmeta['signatures'] = signatures
extra_download = dbconn.retrieveExtraDownload(package_id)
if not splitdebug:
extra_download = [x for x in extra_download if \
x['type'] != "debug"]
self.pkgmeta['extra_download'] = extra_download
self.pkgmeta['download'] = dbconn.retrieveDownloadURL(package_id)
# export main package download path to metadata
# this is actually used by PackageKit backend in order
# to signal downloaded package files
self.pkgmeta['pkgpath'] = self.__get_fetch_disk_path(
self.pkgmeta['download'])
if not self.pkgmeta['download']:
self.pkgmeta['fetch_not_available'] = True
return 0
self.pkgmeta['verified'] = False
self.pkgmeta['steps'] = []
if not repository.endswith(etpConst['packagesext']) and not sources:
dl_check = self.__check_pkg_path_download(self.pkgmeta['download'],
None)
dl_fetch = dl_check < 0
if not dl_fetch:
for extra_download in self.pkgmeta['extra_download']:
dl_check = self.__check_pkg_path_download(
extra_download['download'], None)
if dl_check < 0:
# dl_check checked again right below
break
if dl_check < 0:
self.pkgmeta['steps'].append("fetch")
if dochecksum:
self.pkgmeta['steps'].append("checksum")
elif sources:
self.pkgmeta['steps'].append("sources_fetch")
if sources:
# create sources destination directory
unpack_dir = os.path.join(etpConst['entropyunpackdir'],
"sources", self.pkgmeta['atom'])
self.pkgmeta['unpackdir'] = unpack_dir
if not self.pkgmeta.get('fetch_path'):
if os.path.lexists(unpack_dir):
if os.path.isfile(unpack_dir):
os.remove(unpack_dir)
elif os.path.isdir(unpack_dir):
shutil.rmtree(unpack_dir, True)
if not os.path.lexists(unpack_dir):
os.makedirs(unpack_dir, 0o755)
const_setup_perms(unpack_dir, etpConst['entropygid'],
recursion = False, uid = etpConst['uid'])
return 0
def _check_matching_size(download, size):
d_path = self.__get_fetch_disk_path(download)
try:
st = os.stat(d_path)
return size == st.st_size
except OSError as err:
if err.errno != errno.ENOENT:
raise
return False
matching_size = _check_matching_size(self.pkgmeta['download'],
dbconn.retrieveSize(package_id))
if matching_size:
for extra_download in self.pkgmeta['extra_download']:
matching_size = _check_matching_size(
extra_download['download'],
extra_download['size'])
if not matching_size:
break
# downloading binary package
# if file exists, first checksum then fetch
if matching_size:
self.pkgmeta['steps'].reverse()
return 0
def __generate_multi_fetch_metadata(self):
if not isinstance(self._package_match, list):
raise AttributeError(
"package_match must be a list of tuples, not %s" % (
type(self._package_match,)
)
)
dochecksum = True
# meta options
self.pkgmeta['fetch_abort_function'] = \
self.metaopts.get('fetch_abort_function')
if 'dochecksum' in self.metaopts:
dochecksum = self.metaopts.get('dochecksum')
self.pkgmeta['checksum'] = dochecksum
matches = self._package_match
cl_id = etpConst['system_settings_plugins_ids']['client_plugin']
edelta_support = self._settings[cl_id]['misc']['edelta_support']
self.pkgmeta['edelta_support'] = edelta_support
self.pkgmeta['matches'] = matches
self.pkgmeta['atoms'] = []
self.pkgmeta['repository_atoms'] = {}
temp_fetch_list = []
temp_checksum_list = []
temp_already_downloaded_count = 0
def _setup_download(download, size, package_id, repository, digest,
signatures):
if dochecksum:
obj = (package_id, repository, download, digest,
signatures)
temp_checksum_list.append(obj)
if self.__check_pkg_path_download(download, None) < 0:
obj = (package_id, repository, download, digest, signatures)
temp_fetch_list.append(obj)
else:
down_path = self.__get_fetch_disk_path(download)
if os.path.isfile(down_path):
with open(down_path, "rb") as f:
f.seek(0, os.SEEK_END)
disk_size = f.tell()
if size == disk_size:
return 1
return 0
for package_id, repository in matches:
if repository.endswith(etpConst['packagesext']):
continue
dbconn = self._entropy.open_repository(repository)
myatom = dbconn.retrieveAtom(package_id)
# general purpose metadata
self.pkgmeta['atoms'].append(myatom)
if repository not in self.pkgmeta['repository_atoms']:
self.pkgmeta['repository_atoms'][repository] = set()
self.pkgmeta['repository_atoms'][repository].add(myatom)
download = dbconn.retrieveDownloadURL(package_id)
digest = dbconn.retrieveDigest(package_id)
sha1, sha256, sha512, gpg = dbconn.retrieveSignatures(package_id)
size = dbconn.retrieveSize(package_id)
signatures = {
'sha1': sha1,
'sha256': sha256,
'sha512': sha512,
'gpg': gpg,
}
temp_already_downloaded_count += _setup_download(download, size,
package_id, repository, digest, signatures)
extra_downloads = dbconn.retrieveExtraDownload(package_id)
splitdebug = self.pkgmeta['splitdebug']
# if splitdebug is enabled, check if it's also enabled
# via package.splitdebug
if splitdebug:
splitdebug = self._package_splitdebug_enabled(
(package_id, repository))
if not splitdebug:
extra_downloads = [x for x in extra_downloads if \
x['type'] != "debug"]
for extra_download in extra_downloads:
download = extra_download['download']
size = extra_download['size']
digest = extra_download['md5']
signatures = {
'sha1': extra_download['sha1'],
'sha256': extra_download['sha256'],
'sha512': extra_download['sha512'],
'gpg': extra_download['gpg'],
}
temp_already_downloaded_count += _setup_download(download,
size, package_id, repository, digest, signatures)
self.pkgmeta['steps'] = []
self.pkgmeta['multi_fetch_list'] = temp_fetch_list
self.pkgmeta['multi_checksum_list'] = temp_checksum_list
if self.pkgmeta['multi_fetch_list']:
self.pkgmeta['steps'].append("multi_fetch")
if self.pkgmeta['multi_checksum_list']:
self.pkgmeta['steps'].append("multi_checksum")
if temp_already_downloaded_count == len(temp_checksum_list):
self.pkgmeta['steps'].reverse()
return 0
Reference in New Issue View Git Blame Copy Permalink