749e3ff29d
As reported in bug #3495, do not remove the old repository database file before validating the download outcome (UrlFetcher download result)
2581 lines
90 KiB
Python
2581 lines
90 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
|
|
@author: Fabio Erculiani <lxnay@sabayon.org>
|
|
@contact: lxnay@sabayon.org
|
|
@copyright: Fabio Erculiani
|
|
@license: GPL-2
|
|
|
|
B{Entropy Package Manager Client EntropyRepository plugin code}.
|
|
|
|
"""
|
|
import os
|
|
import sys
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import shutil
|
|
import time
|
|
import codecs
|
|
import errno
|
|
|
|
from entropy.const import const_debug_write, const_setup_perms, etpConst, \
|
|
etpUi, const_set_nice_level, const_setup_file
|
|
from entropy.output import blue, darkred, red, darkgreen, purple, teal, brown, \
|
|
bold, TextInterface
|
|
from entropy.dump import dumpobj, loadobj
|
|
from entropy.cache import EntropyCacher
|
|
from entropy.db import EntropyRepository
|
|
from entropy.exceptions import RepositoryError, SystemDatabaseError, \
|
|
PermissionDenied
|
|
from entropy.security import Repository as RepositorySecurity
|
|
from entropy.misc import TimeScheduled, ParallelTask
|
|
from entropy.i18n import _
|
|
from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase
|
|
from entropy.db.exceptions import IntegrityError, OperationalError, Error, \
|
|
DatabaseError
|
|
from entropy.core.settings.base import SystemSettings
|
|
from entropy.services.client import WebService
|
|
from entropy.client.services.interfaces import RepositoryWebService, \
|
|
RepositoryWebServiceFactory
|
|
|
|
import entropy.dep
|
|
import entropy.tools
|
|
|
|
__all__ = ["CachedRepository", "ClientEntropyRepositoryPlugin",
|
|
"InstalledPackagesRepository", "AvailablePackagesRepository",
|
|
"GenericRepository"]
|
|
|
|
class ClientEntropyRepositoryPlugin(EntropyRepositoryPlugin):
|
|
|
|
def __init__(self, client_interface, metadata = None):
|
|
"""
|
|
Entropy client-side repository EntropyRepository Plugin class.
|
|
This class will be instantiated and automatically added to
|
|
EntropyRepository instances generated by Entropy Client.
|
|
|
|
@param client_interface: Entropy Client interface instance
|
|
@type client_interface: entropy.client.interfaces.Client class
|
|
@param metadata: any dict form metadata map (key => value)
|
|
@type metadata: dict
|
|
"""
|
|
EntropyRepositoryPlugin.__init__(self)
|
|
self._client = client_interface
|
|
if metadata is None:
|
|
self._metadata = {}
|
|
else:
|
|
self._metadata = metadata
|
|
|
|
def get_id(self):
|
|
return "__client__"
|
|
|
|
def get_metadata(self):
|
|
return self._metadata
|
|
|
|
def add_plugin_hook(self, entropy_repository_instance):
|
|
const_debug_write(__name__,
|
|
"ClientEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
|
|
self,)
|
|
)
|
|
|
|
out_intf = self._metadata.get('output_interface')
|
|
if out_intf is not None:
|
|
entropy_repository_instance.output = out_intf.output
|
|
entropy_repository_instance.ask_question = out_intf.ask_question
|
|
|
|
return 0
|
|
|
|
class CachedRepository(EntropyRepository):
|
|
"""
|
|
This kind of repository cannot have close() called directly, without
|
|
a valid token passed. This is because the class object is cached somewhere
|
|
and calling close() would turn into a software bug.
|
|
"""
|
|
def setCloseToken(self, token):
|
|
"""
|
|
Set a token that can be used to validate close() calls. Calling
|
|
close() on these repos is prohibited and considered a software bug.
|
|
Only Entropy Client should be able to close them.
|
|
"""
|
|
self._close_token = token
|
|
|
|
def close(self, _token = None):
|
|
"""
|
|
Reimplemented from EntropyRepository
|
|
"""
|
|
close_token = getattr(self, "_close_token", None)
|
|
if close_token is not None:
|
|
if (_token is None) or (_token != close_token):
|
|
raise PermissionDenied(
|
|
"cannot close this repository directly. Software bug!")
|
|
return EntropyRepository.close(self)
|
|
|
|
|
|
class InstalledPackagesRepository(CachedRepository):
|
|
"""
|
|
This class represents the installed packages repository and is a direct
|
|
subclass of EntropyRepository.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
EntropyRepository.__init__(self, *args, **kwargs)
|
|
# ensure proper repository file permissions
|
|
if entropy.tools.is_root() and os.path.isfile(self._db_path):
|
|
const_setup_file(self._db_path, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
|
|
def handlePackage(self, pkg_data, forcedRevision = -1,
|
|
formattedContent = False):
|
|
"""
|
|
Reimplemented from EntropyRepository.
|
|
"""
|
|
removelist = self.getPackagesToRemove(
|
|
pkg_data['name'], pkg_data['category'],
|
|
pkg_data['slot'], pkg_data['injected']
|
|
)
|
|
for r_package_id in removelist:
|
|
self.removePackage(r_package_id, do_cleanup = False,
|
|
do_commit = False)
|
|
return self.addPackage(pkg_data, revision = forcedRevision,
|
|
formatted_content = formattedContent)
|
|
|
|
|
|
class AvailablePackagesRepositoryUpdater(object):
|
|
|
|
"""
|
|
AvailablePackagesRepository update logic class.
|
|
The required logic for updating a repository is stored here.
|
|
"""
|
|
WEBSERV_CACHE_ID = 'webserv_repo/segment_'
|
|
|
|
def __init__(self, entropy_client, repository_id, force, gpg):
|
|
self.__force = force
|
|
self.__big_sock_timeout = 20
|
|
self._repository_id = repository_id
|
|
self._cacher = EntropyCacher()
|
|
self._last_rev = -1
|
|
self._entropy = entropy_client
|
|
self._settings = SystemSettings()
|
|
self._gpg_feature = gpg
|
|
self._supported_apis = etpConst['supportedapis']
|
|
self._supported_download_items = (
|
|
"db", "dbck", "dblight", "ck", "cklight", "compck",
|
|
"lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck",
|
|
"meta_file", "meta_file_gpg", "notice_board"
|
|
)
|
|
self._developer_repo = \
|
|
self._settings['repositories']['developer_repo']
|
|
self._differential_update = \
|
|
self._settings['repositories']['differential_update']
|
|
if self._developer_repo:
|
|
const_debug_write(__name__,
|
|
"__init__: developer repo mode enabled")
|
|
self.__webservices = None
|
|
self.__webservice = None
|
|
self._repo_eapi = self.__get_repo_eapi()
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
if self._repository_id not in avail_data:
|
|
raise KeyError("Repository not available")
|
|
|
|
@property
|
|
def _webservices(self):
|
|
if self.__webservices is None:
|
|
if hasattr(self._entropy, "RepositoryWebServices"):
|
|
self.__webservices = self._entropy.RepositoryWebServices()
|
|
else:
|
|
# in case self._entropy is a simple TextInterface()
|
|
# like how it's called in remote_revision().
|
|
self.__webservices = RepositoryWebServiceFactory(self._entropy)
|
|
# cross fingers!
|
|
|
|
return self.__webservices
|
|
|
|
@property
|
|
def _webservice(self):
|
|
if self.__webservice is None:
|
|
self.__webservice = self._webservices.new(self._repository_id)
|
|
self.__webservice._set_timeout(self.__big_sock_timeout)
|
|
return self.__webservice
|
|
|
|
def __get_webserv_repository_metadata(self):
|
|
try:
|
|
data = self._webservice.get_repository_metadata()
|
|
except WebService.WebServiceException as err:
|
|
const_debug_write(__name__,
|
|
"__get_webserv_repository_metadata: error: %s" % (err,))
|
|
data = {}
|
|
return data
|
|
|
|
def __get_webserv_repository_revision(self):
|
|
try:
|
|
revision = self._webservice.get_revision()
|
|
except WebService.WebServiceException as err:
|
|
const_debug_write(__name__,
|
|
"__get_webserv_repository_revision: error: %s" % (err,))
|
|
revision = None
|
|
return revision
|
|
|
|
def __check_webserv_availability(self):
|
|
try:
|
|
webserv = self._webservices.new(self._repository_id)
|
|
except WebService.UnsupportedService:
|
|
return False
|
|
|
|
try:
|
|
available = webserv.service_available(cache = False)
|
|
except WebService.WebServiceException:
|
|
available = False
|
|
return available
|
|
|
|
def __get_webserv_local_database(self):
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
dbfile = os.path.join(repo_data['dbpath'],
|
|
etpConst['etpdatabasefile'])
|
|
dbconn = None
|
|
try:
|
|
dbconn = self._entropy.open_generic_repository(dbfile,
|
|
xcache = False, indexing_override = False)
|
|
dbconn.validate()
|
|
except (OperationalError, IntegrityError, SystemDatabaseError,
|
|
IOError, OSError,):
|
|
dbconn = None
|
|
return dbconn
|
|
|
|
def __get_webserv_database_differences(self, webserv, package_ids):
|
|
|
|
try:
|
|
remote_package_ids = webserv.get_package_ids()
|
|
except WebService.WebServiceException as err:
|
|
const_debug_write(__name__,
|
|
"__get_webserv_database_differences: error: %s" % (err,))
|
|
return None, None
|
|
|
|
added = [x for x in remote_package_ids if x not in package_ids]
|
|
removed = [x for x in package_ids if x not in remote_package_ids]
|
|
return added, removed
|
|
|
|
def __eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):
|
|
|
|
dbconn = self._entropy.open_generic_repository(dbfile,
|
|
xcache = False, indexing_override = False)
|
|
old_dbconn = self._entropy.open_generic_repository(dbfile_old,
|
|
xcache = False, indexing_override = False)
|
|
upd_rc = 0
|
|
try:
|
|
upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t")
|
|
except (OperationalError, IntegrityError, DatabaseError,):
|
|
pass
|
|
old_dbconn.close()
|
|
dbconn.close()
|
|
if upd_rc > 0:
|
|
# -1 means no changes, == force used
|
|
# 0 means too much hassle
|
|
os.rename(dbfile_old, dbfile)
|
|
return upd_rc
|
|
|
|
def __eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):
|
|
|
|
# load the dump into database
|
|
mytxt = "%s %s, %s %s" % (
|
|
red(_("Injecting downloaded dump")),
|
|
darkgreen(etpConst['etpdatabasedumplight']),
|
|
red(_("please wait")),
|
|
red("..."),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
dbconn = self._entropy.open_generic_repository(dbfile,
|
|
xcache = False, indexing_override = False)
|
|
rc = dbconn.importRepository(dumpfile, dbfile)
|
|
dbconn.close()
|
|
return rc
|
|
|
|
def __get_repo_eapi(self):
|
|
|
|
eapi_env = os.getenv("FORCE_EAPI")
|
|
sqlite3_access = os.access("/usr/bin/sqlite3", os.X_OK)
|
|
sqlite3_rc = subprocess.call("/usr/bin/sqlite3 -version > /dev/null",
|
|
shell = True)
|
|
try:
|
|
eapi_env_clear = int(eapi_env)
|
|
if eapi_env_clear not in self._supported_apis:
|
|
raise ValueError()
|
|
except (ValueError, TypeError,):
|
|
eapi_env_clear = None
|
|
|
|
repo_eapi = 2
|
|
eapi_avail = self.__check_webserv_availability()
|
|
if eapi_avail:
|
|
repo_eapi = 3
|
|
else:
|
|
if not sqlite3_access or entropy.tools.islive():
|
|
repo_eapi = 1
|
|
elif sqlite3_rc != 0:
|
|
repo_eapi = 1
|
|
|
|
# if differential update is disabled and FORCE_EAPI is not overriding
|
|
# we cannot use EAPI=3
|
|
if (eapi_env_clear is None) and (not self._differential_update) and \
|
|
(repo_eapi == 3):
|
|
const_debug_write(__name__,
|
|
"__get_repo_eapi: differential update is disabled !")
|
|
repo_eapi -= 1
|
|
|
|
# check EAPI
|
|
if eapi_env_clear is not None:
|
|
repo_eapi = eapi_env_clear
|
|
|
|
# FORCE_EAPI is triggered, disable
|
|
# developer_repo mode
|
|
self._developer_repo = False
|
|
const_debug_write(__name__,
|
|
"__get_repo_eapi: developer repo mode disabled FORCE_EAPI")
|
|
|
|
elif repo_eapi > 1 and self._developer_repo:
|
|
# enforce EAPI=1
|
|
repo_eapi = 1
|
|
|
|
const_debug_write(__name__,
|
|
"__get_repo_eapi: final eapi set to %s" % (repo_eapi,))
|
|
|
|
return repo_eapi
|
|
|
|
def __is_repository_updatable(self):
|
|
|
|
online = self.remote_revision()
|
|
if online != -1:
|
|
local = AvailablePackagesRepository.revision(
|
|
self._repository_id)
|
|
if (local == online) and (not self.__force):
|
|
return False
|
|
return True
|
|
|
|
def __show_repository_information(self):
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
self._entropy.output(
|
|
bold("%s") % ( repo_data['description'] ),
|
|
importance = 2,
|
|
level = "info",
|
|
header = blue(" # ")
|
|
)
|
|
mytxt = "%s: %s" % (red(_("Repository URL")),
|
|
darkgreen(repo_data['database']),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = blue(" # ")
|
|
)
|
|
mytxt = "%s: %s" % (red(_("Repository local path")),
|
|
darkgreen(repo_data['dbpath']),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # ")
|
|
)
|
|
mytxt = "%s: %s" % (red(_("Repository API")),
|
|
darkgreen(str(self._repo_eapi)),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # ")
|
|
)
|
|
|
|
def __handle_repository_lock(self):
|
|
# get database lock
|
|
unlocked = self._is_repository_unlocked()
|
|
if not unlocked:
|
|
mytxt = "%s: %s. %s." % (
|
|
bold(_("Attention")),
|
|
red(_("Repository is being updated")),
|
|
red(_("Try again in a few minutes")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
return True
|
|
return False
|
|
|
|
def __append_gpg_signature_to_path(self, path):
|
|
return path + etpConst['etpgpgextension']
|
|
|
|
def __ensure_repository_path(self):
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
# create dir if it doesn't exist
|
|
if not os.path.isdir(repo_data['dbpath']):
|
|
os.makedirs(repo_data['dbpath'], 0o755)
|
|
|
|
try:
|
|
items = os.listdir(etpConst['etpdatabaseclientdir'])
|
|
except OSError:
|
|
items = []
|
|
|
|
# we cannot operate dir wide (etpdatabaseclientdir) because
|
|
# there are lock files in there and we should not touch them
|
|
for item in items:
|
|
repo_dir_path = os.path.join(
|
|
etpConst['etpdatabaseclientdir'],
|
|
item)
|
|
if not os.path.isdir(repo_dir_path):
|
|
continue
|
|
const_setup_perms(
|
|
repo_dir_path,
|
|
etpConst['entropygid'],
|
|
f_perms = 0o644)
|
|
|
|
def __validate_compression_method(self):
|
|
|
|
repo = self._repository_id
|
|
repo_settings = self._settings['repositories']
|
|
dbc_format = repo_settings['available'][repo]['dbcformat']
|
|
cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format)
|
|
if cmethod is None:
|
|
raise AttributeError("Wrong repository compression method")
|
|
|
|
return cmethod
|
|
|
|
def __remove_repository_files(self):
|
|
sys_set = self._settings
|
|
avail_data = sys_set['repositories']['available']
|
|
repo_dbpath = avail_data[self._repository_id]['dbpath']
|
|
shutil.rmtree(repo_dbpath, True)
|
|
|
|
def __handle_database_download(self, cmethod):
|
|
|
|
# starting to download
|
|
mytxt = "%s ..." % (red(_("Downloading repository")),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
downloaded_item = None
|
|
down_status = False
|
|
sig_status = False
|
|
if self._repo_eapi == 2:
|
|
|
|
down_item = "dbdumplight"
|
|
|
|
down_status = self._download_item(down_item, cmethod,
|
|
disallow_redirect = True)
|
|
if down_status:
|
|
# get GPG file if available
|
|
sig_status = self._download_item(down_item, cmethod,
|
|
disallow_redirect = True, get_signature = True)
|
|
|
|
downloaded_item = down_item
|
|
|
|
if not down_status: # fallback to old db
|
|
|
|
self._repo_eapi = 1
|
|
down_item = "dblight"
|
|
if self._developer_repo:
|
|
# if developer repo mode is enabled, fetch full-blown db
|
|
down_item = "db"
|
|
const_debug_write(__name__,
|
|
"__handle_database_download: developer repo mode enabled")
|
|
|
|
down_status = self._download_item(down_item, cmethod,
|
|
disallow_redirect = True)
|
|
if down_status:
|
|
sig_status = self._download_item(down_item, cmethod,
|
|
disallow_redirect = True, get_signature = True)
|
|
|
|
downloaded_item = down_item
|
|
|
|
if not down_status:
|
|
mytxt = "%s: %s." % (bold(_("Attention")),
|
|
red(_("unable to download the repository")),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
|
|
return down_status, sig_status, downloaded_item
|
|
|
|
def __handle_database_checksum_download(self, cmethod):
|
|
|
|
downitem = 'cklight'
|
|
if self._developer_repo:
|
|
downitem = 'dbck'
|
|
if self._repo_eapi == 2: # EAPI = 2
|
|
downitem = 'dbdumplightck'
|
|
|
|
garbage_url, hashfile = self._construct_paths(downitem, cmethod)
|
|
mytxt = "%s %s %s" % (
|
|
red(_("Downloading checksum")),
|
|
darkgreen(os.path.basename(hashfile)),
|
|
red("..."),
|
|
)
|
|
# download checksum
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
db_down_status = self._download_item(downitem, cmethod,
|
|
disallow_redirect = True)
|
|
|
|
if not db_down_status and (downitem not in ('cklight', 'dbck',)):
|
|
# fallback to old method
|
|
retryitem = 'cklight'
|
|
if self._developer_repo:
|
|
retryitem = 'dbck'
|
|
db_down_status = self._download_item(retryitem, cmethod,
|
|
disallow_redirect = True)
|
|
|
|
if not db_down_status:
|
|
mytxt = "%s %s !" % (
|
|
red(_("Cannot fetch checksum")),
|
|
red(_("Cannot verify repository integrity")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
return db_down_status
|
|
|
|
def __verify_file_checksum(self, file_path, md5_checksum_path):
|
|
enc = etpConst['conf_encoding']
|
|
with codecs.open(md5_checksum_path, "r", encoding=enc) as ck_f:
|
|
md5hash = ck_f.readline().strip()
|
|
if not md5hash: # invalid !! => [] would cause IndexError
|
|
return False
|
|
md5hash = md5hash.split()[0]
|
|
return entropy.tools.compare_md5(file_path, md5hash)
|
|
|
|
def __verify_database_checksum(self, cmethod = None):
|
|
|
|
sys_settings_repos = self._settings['repositories']
|
|
avail_config = sys_settings_repos['available'][self._repository_id]
|
|
|
|
sep = os.path.sep
|
|
if self._repo_eapi == 1:
|
|
if self._developer_repo:
|
|
remote_gb, dbfile = self._construct_paths('db', cmethod)
|
|
remote_gb, md5file = self._construct_paths('dbck', cmethod)
|
|
else:
|
|
remote_gb, dbfile = self._construct_paths('dblight',
|
|
cmethod)
|
|
remote_gb, md5file = self._construct_paths('cklight',
|
|
cmethod)
|
|
|
|
elif self._repo_eapi == 2:
|
|
remote_gb, dbfile = self._construct_paths('dbdumplight',
|
|
cmethod)
|
|
remote_gb, md5file = self._construct_paths('dbdumplightck',
|
|
cmethod)
|
|
|
|
else:
|
|
raise AttributeError("EAPI must be = 1 or 2")
|
|
|
|
if not (os.access(md5file, os.R_OK) and os.path.isfile(md5file)):
|
|
return -1
|
|
|
|
return self.__verify_file_checksum(dbfile, md5file)
|
|
|
|
def __unpack_downloaded_database(self, down_item, cmethod):
|
|
|
|
rc = 0
|
|
path = None
|
|
sys_set_repos = self._settings['repositories']['available']
|
|
repo_data = sys_set_repos[self._repository_id]
|
|
|
|
garbage, myfile = self._construct_paths(down_item, cmethod)
|
|
|
|
if self._repo_eapi in (1, 2,):
|
|
try:
|
|
|
|
myfunc = getattr(entropy.tools, cmethod[1])
|
|
path = myfunc(myfile)
|
|
# rename path correctly
|
|
if self._repo_eapi == 1:
|
|
new_path = os.path.join(os.path.dirname(path),
|
|
etpConst['etpdatabasefile'])
|
|
os.rename(path, new_path)
|
|
path = new_path
|
|
|
|
except (OSError, EOFError):
|
|
rc = 1
|
|
|
|
else:
|
|
mytxt = "invalid EAPI must be = 1 or 2"
|
|
raise AttributeError(mytxt)
|
|
|
|
if rc == 0:
|
|
const_setup_file(path, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
|
|
return rc
|
|
|
|
def __handle_downloaded_database_unpack(self, cmethod):
|
|
|
|
file_to_unpack = etpConst['etpdatabasedump']
|
|
if self._repo_eapi == 1:
|
|
file_to_unpack = etpConst['etpdatabasefile']
|
|
elif self._repo_eapi == 2:
|
|
file_to_unpack = etpConst['etpdatabasedumplight']
|
|
|
|
mytxt = "%s %s %s" % (red(_("Unpacking database to")),
|
|
darkgreen(file_to_unpack), red("..."),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
myitem = 'dblight'
|
|
if self._repo_eapi == 2:
|
|
myitem = 'dbdumplight'
|
|
elif self._developer_repo:
|
|
myitem = 'db'
|
|
|
|
myrc = self.__unpack_downloaded_database(myitem, cmethod)
|
|
if myrc != 0:
|
|
mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),
|
|
red(_("Skipping repository")),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
return False, myitem
|
|
return True, myitem
|
|
|
|
def __update_repository_revision(self):
|
|
cur_rev = AvailablePackagesRepository.revision(self._repository_id)
|
|
repo_data = self._settings['repositories']
|
|
db_data = repo_data['available'][self._repository_id]
|
|
db_data['dbrevision'] = "0"
|
|
if cur_rev != -1:
|
|
db_data['dbrevision'] = str(cur_rev)
|
|
|
|
# update repository revision file
|
|
# self.remote_revision() output must be
|
|
# written into packages.db.revision for consistency
|
|
# otherwise WebService sync when WebService is on a separate
|
|
# server (and uses rsync) doesn't work at its best
|
|
# self._last_revs
|
|
rev_file = os.path.join(db_data['dbpath'],
|
|
etpConst['etpdatabaserevisionfile'])
|
|
enc = etpConst['conf_encoding']
|
|
with codecs.open(rev_file, "w", encoding=enc) as rev_f:
|
|
# safe anyway
|
|
rev_f.write(str(self._last_rev) + "\n")
|
|
rev_f.flush()
|
|
|
|
def __validate_database(self):
|
|
def tell_error(err):
|
|
mytxt = "%s: %s" % (darkred(_("Repository is invalid")),
|
|
repr(err),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "error",
|
|
header = "\t"
|
|
)
|
|
try:
|
|
dbconn = self._entropy.open_repository(self._repository_id)
|
|
except RepositoryError as err:
|
|
tell_error(err)
|
|
return False
|
|
try:
|
|
dbconn.validate()
|
|
except SystemDatabaseError as err:
|
|
tell_error(err)
|
|
return False
|
|
return True
|
|
|
|
def __database_indexing(self):
|
|
|
|
# renice a bit, to avoid eating resources
|
|
old_prio = const_set_nice_level(15)
|
|
mytxt = red("%s ...") % (_("Indexing Repository metadata"),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
dbconn = self._entropy.open_repository(self._repository_id)
|
|
dbconn.createAllIndexes()
|
|
dbconn.commit(force = True)
|
|
if self._entropy.installed_repository() is not None:
|
|
try: # client db can be absent
|
|
self._entropy.installed_repository().createAllIndexes()
|
|
except (DatabaseError, OperationalError, IntegrityError,):
|
|
pass
|
|
const_set_nice_level(old_prio)
|
|
|
|
def _construct_paths(self, item, cmethod, get_signature = False):
|
|
|
|
if item not in self._supported_download_items:
|
|
raise AttributeError("Invalid item: %s" % (item,))
|
|
|
|
items_needing_cmethod = (
|
|
"db", "dbck", "dblight", "cklight", "dbdump", "dbdumpck",
|
|
"dbdumplight", "dbdumplightck", "compck",
|
|
)
|
|
if (item in items_needing_cmethod) and (cmethod is None):
|
|
mytxt = "For %s, cmethod can't be None" % (item,)
|
|
raise AttributeError(mytxt)
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
repo_db = repo_data['database']
|
|
repo_dbpath = repo_data['dbpath']
|
|
ec_hash = etpConst['etpdatabasehashfile']
|
|
repo_lock_file = etpConst['etpdatabasedownloadlockfile']
|
|
notice_board_filename = os.path.basename(repo_data['notice_board'])
|
|
meta_file = etpConst['etpdatabasemetafilesfile']
|
|
meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
|
|
etpConst['etpgpgextension']
|
|
md5_ext = etpConst['packagesmd5fileext']
|
|
ec_cm2 = None
|
|
ec_cm3 = None
|
|
ec_cm4 = None
|
|
ec_cm5 = None
|
|
ec_cm6 = None
|
|
ec_cm7 = None
|
|
ec_cm8 = None
|
|
ec_cm9 = None
|
|
if cmethod is not None:
|
|
ec_cm2 = etpConst[cmethod[2]]
|
|
ec_cm3 = etpConst[cmethod[3]]
|
|
ec_cm4 = etpConst[cmethod[4]]
|
|
ec_cm5 = etpConst[cmethod[5]]
|
|
ec_cm6 = etpConst[cmethod[6]]
|
|
ec_cm7 = etpConst[cmethod[7]]
|
|
ec_cm8 = etpConst[cmethod[8]]
|
|
ec_cm9 = etpConst[cmethod[9]]
|
|
|
|
mymap = {
|
|
'db': (
|
|
"%s/%s" % (repo_db, ec_cm2,),
|
|
"%s/%s" % (repo_dbpath, ec_cm2,),
|
|
),
|
|
'dbck': (
|
|
"%s/%s" % (repo_db, ec_cm9,),
|
|
"%s/%s" % (repo_dbpath, ec_cm9,),
|
|
),
|
|
'dblight': (
|
|
"%s/%s" % (repo_db, ec_cm7,),
|
|
"%s/%s" % (repo_dbpath, ec_cm7,),
|
|
),
|
|
'dbdump': (
|
|
"%s/%s" % (repo_db, ec_cm3,),
|
|
"%s/%s" % (repo_dbpath, ec_cm3,),
|
|
),
|
|
'dbdumplight': (
|
|
"%s/%s" % (repo_db, ec_cm5,),
|
|
"%s/%s" % (repo_dbpath, ec_cm5,),
|
|
),
|
|
'ck': (
|
|
"%s/%s" % (repo_db, ec_hash,),
|
|
"%s/%s" % (repo_dbpath, ec_hash,),
|
|
),
|
|
'cklight': (
|
|
"%s/%s" % (repo_db, ec_cm8,),
|
|
"%s/%s" % (repo_dbpath, ec_cm8,),
|
|
),
|
|
'compck': (
|
|
"%s/%s%s" % (repo_db, ec_cm2, md5_ext,),
|
|
"%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),
|
|
),
|
|
'dbdumpck': (
|
|
"%s/%s" % (repo_db, ec_cm4,),
|
|
"%s/%s" % (repo_dbpath, ec_cm4,),
|
|
),
|
|
'dbdumplightck': (
|
|
"%s/%s" % (repo_db, ec_cm6,),
|
|
"%s/%s" % (repo_dbpath, ec_cm6,),
|
|
),
|
|
'lock': (
|
|
"%s/%s" % (repo_db, repo_lock_file,),
|
|
"%s/%s" % (repo_dbpath, repo_lock_file,),
|
|
),
|
|
'notice_board': (
|
|
repo_data['notice_board'],
|
|
"%s/%s" % (repo_dbpath, notice_board_filename,),
|
|
),
|
|
'meta_file': (
|
|
"%s/%s" % (repo_db, meta_file,),
|
|
"%s/%s" % (repo_dbpath, meta_file,),
|
|
),
|
|
'meta_file_gpg': (
|
|
"%s/%s" % (repo_db, meta_file_gpg,),
|
|
"%s/%s" % (repo_dbpath, meta_file_gpg,),
|
|
),
|
|
}
|
|
|
|
url, path = mymap.get(item)
|
|
if get_signature:
|
|
url = self.__append_gpg_signature_to_path(url)
|
|
path = self.__append_gpg_signature_to_path(path)
|
|
|
|
return url, path
|
|
|
|
def _download_item(self, item, cmethod = None,
|
|
disallow_redirect = True, get_signature = False):
|
|
|
|
url, filepath = self._construct_paths(item, cmethod,
|
|
get_signature = get_signature)
|
|
|
|
# See bug #3495, download the file to
|
|
# a temporary location and then move it
|
|
# if we are successful
|
|
temp_filepath = filepath + ".edownload"
|
|
|
|
# to avoid having permissions issues
|
|
# it's better to remove the file before,
|
|
# otherwise new permissions won't be written
|
|
if os.path.isfile(temp_filepath):
|
|
os.remove(temp_filepath)
|
|
filepath_dir = os.path.dirname(temp_filepath)
|
|
|
|
if not os.path.isdir(filepath_dir) and not \
|
|
os.path.lexists(filepath_dir):
|
|
|
|
os.makedirs(filepath_dir, 0o755)
|
|
const_setup_perms(filepath_dir, etpConst['entropygid'],
|
|
f_perms = 0o644)
|
|
|
|
try:
|
|
|
|
fetcher = self._entropy._url_fetcher(
|
|
url,
|
|
temp_filepath,
|
|
resume = False,
|
|
disallow_redirect = disallow_redirect
|
|
)
|
|
|
|
rc = fetcher.download()
|
|
if rc in ("-1", "-2", "-3", "-4"):
|
|
return False
|
|
try:
|
|
os.rename(temp_filepath, filepath)
|
|
except (OSError, IOError) as err:
|
|
if err.errno != errno.ENOENT:
|
|
raise
|
|
return False # not downloaded?
|
|
|
|
const_setup_file(filepath, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
return True
|
|
|
|
finally:
|
|
# cleanup temp file
|
|
try:
|
|
os.remove(temp_filepath)
|
|
except (OSError, IOError) as err:
|
|
if err.errno != errno.ENOENT:
|
|
raise
|
|
|
|
def _is_repository_unlocked(self):
|
|
"""
|
|
Returns whether the repository is remotely locked or not.
|
|
|
|
@return: repository being remotely locked
|
|
@rtype:bool
|
|
"""
|
|
rc = self._download_item("lock", disallow_redirect = True)
|
|
if rc: # cannot download database
|
|
return False
|
|
return True
|
|
|
|
def _standard_items_download(self):
|
|
|
|
repos_data = self._settings['repositories']
|
|
repo_data = repos_data['available'][self._repository_id]
|
|
notice_board = os.path.basename(repo_data['local_notice_board'])
|
|
db_meta_file = etpConst['etpdatabasemetafilesfile']
|
|
db_meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
|
|
etpConst['etpgpgextension']
|
|
|
|
objects_to_unpack = ("meta_file",)
|
|
|
|
download_items = [
|
|
(
|
|
"meta_file",
|
|
db_meta_file,
|
|
False,
|
|
"%s %s %s" % (
|
|
red(_("Downloading repository metafile")),
|
|
darkgreen(db_meta_file),
|
|
red("..."),
|
|
)
|
|
),
|
|
(
|
|
"meta_file_gpg",
|
|
db_meta_file_gpg,
|
|
True,
|
|
"%s %s %s" % (
|
|
red(_("Downloading GPG signature of repository metafile")),
|
|
darkgreen(db_meta_file_gpg),
|
|
red("..."),
|
|
)
|
|
),
|
|
(
|
|
"notice_board",
|
|
notice_board,
|
|
True,
|
|
"%s %s %s" % (
|
|
red(_("Downloading Notice Board")),
|
|
darkgreen(notice_board),
|
|
red("..."),
|
|
)
|
|
),
|
|
]
|
|
|
|
def my_show_info(txt):
|
|
self._entropy.output(
|
|
txt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = "\t",
|
|
back = True
|
|
)
|
|
|
|
def my_show_down_status(message, mytype):
|
|
self._entropy.output(
|
|
message,
|
|
importance = 0,
|
|
level = mytype,
|
|
header = "\t"
|
|
)
|
|
|
|
def my_show_file_unpack(fp):
|
|
self._entropy.output(
|
|
"%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),),
|
|
header = blue("\t << ")
|
|
)
|
|
|
|
def my_show_file_rm(fp):
|
|
self._entropy.output(
|
|
"%s: %s" % (darkgreen(_("removed meta file")), purple(fp),),
|
|
header = blue("\t << ")
|
|
)
|
|
|
|
downloaded_files = []
|
|
|
|
for item, myfile, ignorable, mytxt in download_items:
|
|
|
|
my_show_info(mytxt)
|
|
mystatus = self._download_item(item, disallow_redirect = True)
|
|
mytype = 'info'
|
|
myurl, mypath = self._construct_paths(item, None)
|
|
|
|
# download failed, is it critical?
|
|
if not mystatus:
|
|
if ignorable:
|
|
message = "%s: %s." % (blue(myfile),
|
|
red(_("not available, it's ok")))
|
|
else:
|
|
mytype = 'warning'
|
|
message = "%s: %s." % (blue(myfile),
|
|
darkred(_("not available, not very ok!")))
|
|
my_show_down_status(message, mytype)
|
|
|
|
# remove garbage
|
|
if os.path.isfile(mypath):
|
|
try:
|
|
os.remove(mypath)
|
|
except OSError:
|
|
continue
|
|
|
|
continue
|
|
|
|
message = "%s: %s." % (blue(myfile),
|
|
darkgreen(_("available, w00t!")))
|
|
my_show_down_status(message, mytype)
|
|
downloaded_files.append(mypath)
|
|
|
|
if item not in objects_to_unpack:
|
|
continue
|
|
if not (os.path.isfile(mypath) and os.access(mypath, os.R_OK)):
|
|
continue
|
|
|
|
tmpdir = tempfile.mkdtemp()
|
|
repo_dir = repo_data['dbpath']
|
|
enc = etpConst['conf_encoding']
|
|
try:
|
|
done = entropy.tools.universal_uncompress(mypath, tmpdir,
|
|
catch_empty = True)
|
|
if not done:
|
|
mytype = 'warning'
|
|
message = "%s: %s." % (blue(myfile),
|
|
darkred(_("cannot be unpacked, not very ok!")))
|
|
my_show_down_status(message, mytype)
|
|
continue
|
|
myfiles_to_move = set(os.listdir(tmpdir))
|
|
|
|
# exclude files not available by default
|
|
files_not_found_file = etpConst['etpdatabasemetafilesnotfound']
|
|
if files_not_found_file in myfiles_to_move:
|
|
myfiles_to_move.remove(files_not_found_file)
|
|
fnf_path = os.path.join(tmpdir, files_not_found_file)
|
|
|
|
if os.path.isfile(fnf_path) and \
|
|
os.access(fnf_path, os.R_OK):
|
|
with codecs.open(fnf_path, "r", encoding=enc) as f:
|
|
f_nf = [x.strip() for x in f.readlines()]
|
|
|
|
for myfile in f_nf:
|
|
myfile = os.path.basename(myfile) # avoid lamerz
|
|
myfpath = os.path.join(repo_dir, myfile)
|
|
if os.path.isfile(myfpath) and \
|
|
os.access(myfpath, os.W_OK):
|
|
try:
|
|
os.remove(myfpath)
|
|
my_show_file_rm(myfile)
|
|
except OSError:
|
|
continue
|
|
|
|
for myfile in sorted(myfiles_to_move):
|
|
from_mypath = os.path.join(tmpdir, myfile)
|
|
to_mypath = os.path.join(repo_dir, myfile)
|
|
try:
|
|
os.rename(from_mypath, to_mypath)
|
|
my_show_file_unpack(myfile)
|
|
except OSError:
|
|
# try non atomic way
|
|
try:
|
|
shutil.copy2(from_mypath, to_mypath)
|
|
my_show_file_unpack(myfile)
|
|
os.remove(from_mypath)
|
|
except (shutil.Error, IOError, OSError,):
|
|
continue
|
|
continue
|
|
|
|
const_setup_file(to_mypath, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
|
|
finally:
|
|
shutil.rmtree(tmpdir, True)
|
|
|
|
repo_r = AvailablePackagesRepository.revision(self._repository_id)
|
|
mytxt = "%s: %s" % (
|
|
red(_("Repository revision")),
|
|
bold(str(repo_r)),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
|
|
return downloaded_files
|
|
|
|
def _check_downloaded_database(self, cmethod):
|
|
|
|
dbitem = 'dblight'
|
|
if self._repo_eapi == 2:
|
|
dbitem = 'dbdumplight'
|
|
elif self._developer_repo:
|
|
dbitem = 'db'
|
|
garbage, dbfilename = self._construct_paths(dbitem, cmethod)
|
|
|
|
# verify checksum
|
|
mytxt = "%s %s %s" % (
|
|
red(_("Checking downloaded repository")),
|
|
darkgreen(os.path.basename(dbfilename)),
|
|
red("..."),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
back = True,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
db_status = self.__verify_database_checksum(cmethod)
|
|
if db_status == -1:
|
|
mytxt = "%s. %s !" % (
|
|
red(_("Cannot open digest")),
|
|
red(_("Cannot verify repository integrity")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
elif db_status:
|
|
mytxt = "%s: %s" % (
|
|
red(_("Downloaded repository status")),
|
|
bold(_("OK")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
else:
|
|
mytxt = "%s: %s" % (
|
|
red(_("Downloaded repository status")),
|
|
darkred(_("ERROR")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "error",
|
|
header = "\t"
|
|
)
|
|
mytxt = "%s. %s" % (
|
|
red(_("An error occured while checking repository integrity")),
|
|
red(_("Giving up")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "error",
|
|
header = "\t"
|
|
)
|
|
return 1
|
|
return 0
|
|
|
|
def _install_gpg_key_if_available(self):
|
|
|
|
my_repos = self._settings['repositories']
|
|
avail_data = my_repos['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
gpg_path = repo_data['gpg_pubkey']
|
|
|
|
if not (os.path.isfile(gpg_path) and os.access(gpg_path, os.R_OK)):
|
|
return False # gpg key not available
|
|
|
|
def do_warn_user(fingerprint):
|
|
mytxt = purple(_("Make sure to verify the imported key and set an appropriate trust level"))
|
|
self._entropy.output(
|
|
mytxt + ":",
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
mytxt = brown("gpg --homedir '%s' --edit-key '%s'" % (
|
|
etpConst['etpclientgpgdir'], fingerprint,)
|
|
)
|
|
self._entropy.output(
|
|
"$ " + mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
|
|
try:
|
|
repo_sec = self._entropy.RepositorySecurity()
|
|
except RepositorySecurity.GPGError:
|
|
mytxt = "%s," % (
|
|
purple(_("This repository suports GPG-signed packages")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
mytxt = purple(_("you may want to install GnuPG to take advantage of this feature"))
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
return False # GPG not available
|
|
|
|
pk_expired = False
|
|
try:
|
|
pk_avail = repo_sec.is_pubkey_available(self._repository_id)
|
|
except repo_sec.KeyExpired:
|
|
pk_avail = False
|
|
pk_expired = True
|
|
|
|
if pk_avail:
|
|
|
|
tmp_dir = tempfile.mkdtemp()
|
|
repo_tmp_sec = self._entropy.RepositorySecurity(
|
|
keystore_dir = tmp_dir)
|
|
# try to install and get fingerprint
|
|
try:
|
|
downloaded_key_fp = repo_tmp_sec.install_key(
|
|
self._repository_id, gpg_path)
|
|
except RepositorySecurity.GPGError:
|
|
downloaded_key_fp = None
|
|
|
|
fingerprint = repo_sec.get_key_metadata(
|
|
self._repository_id)['fingerprint']
|
|
shutil.rmtree(tmp_dir, True)
|
|
|
|
if downloaded_key_fp != fingerprint and \
|
|
(downloaded_key_fp is not None):
|
|
mytxt = "%s: %s !!!" % (
|
|
purple(_("GPG key changed for")),
|
|
bold(self._repository_id),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
mytxt = "[%s => %s]" % (
|
|
darkgreen(fingerprint),
|
|
purple(downloaded_key_fp),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
do_warn_user(downloaded_key_fp)
|
|
else:
|
|
mytxt = "%s: %s" % (
|
|
purple(_("GPG key already installed for")),
|
|
bold(self._repository_id),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
do_warn_user(fingerprint)
|
|
return True # already installed
|
|
|
|
elif pk_expired:
|
|
mytxt = "%s: %s" % (
|
|
purple(_("GPG key EXPIRED for repository")),
|
|
bold(self._repository_id),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
|
|
|
|
# actually install
|
|
mytxt = "%s: %s" % (
|
|
purple(_("Installing GPG key for repository")),
|
|
brown(self._repository_id),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = "\t",
|
|
back = True
|
|
)
|
|
|
|
try_ignore = False
|
|
while True:
|
|
try:
|
|
fingerprint = repo_sec.install_key(self._repository_id,
|
|
gpg_path)
|
|
except RepositorySecurity.NothingImported as err:
|
|
if try_ignore:
|
|
mytxt = "%s: %s" % (
|
|
darkred(_("Error during GPG key installation")),
|
|
err,
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "error",
|
|
header = "\t"
|
|
)
|
|
return False
|
|
self._entropy.output(
|
|
purple(_("GPG key seems already installed but not properly recorded, resetting")),
|
|
level = "warning",
|
|
header = "\t"
|
|
)
|
|
target_fingerprint = repo_sec.get_key_fingerprint(gpg_path)
|
|
if target_fingerprint is not None:
|
|
# kill it, this is usually caused by shadow repos
|
|
# like sabayon-weekly
|
|
dead_repository_ids = set()
|
|
for _repository_id, key_meta in repo_sec.get_keys().items():
|
|
if key_meta['fingerprint'] == target_fingerprint:
|
|
dead_repository_ids.add(_repository_id)
|
|
for _repository_id in dead_repository_ids:
|
|
try:
|
|
repo_sec.delete_pubkey(_repository_id)
|
|
except KeyError:
|
|
# wtf, fault tolerance
|
|
pass
|
|
try_ignore = True
|
|
continue
|
|
except RepositorySecurity.GPGError as err:
|
|
mytxt = "%s: %s" % (
|
|
darkred(_("Error during GPG key installation")),
|
|
err,
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "error",
|
|
header = "\t"
|
|
)
|
|
return False
|
|
break
|
|
|
|
mytxt = "%s: %s" % (
|
|
purple(_("Successfully installed GPG key for repository")),
|
|
brown(self._repository_id),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
mytxt = "%s: %s" % (
|
|
darkgreen(_("Fingerprint")),
|
|
bold(fingerprint),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
do_warn_user(fingerprint)
|
|
return True
|
|
|
|
def _gpg_verify_downloaded_files(self, downloaded_files):
|
|
|
|
try:
|
|
repo_sec = self._entropy.RepositorySecurity()
|
|
except RepositorySecurity.GPGServiceNotAvailable:
|
|
# wtf! it was available a while ago!
|
|
return 0 # GPG not available
|
|
|
|
gpg_sign_ext = etpConst['etpgpgextension']
|
|
sign_files = [x for x in downloaded_files if \
|
|
x.endswith(gpg_sign_ext)]
|
|
sign_files = [x for x in sign_files if os.path.isfile(x) and \
|
|
os.access(x, os.R_OK)]
|
|
|
|
to_be_verified = []
|
|
|
|
for sign_path in sign_files:
|
|
target_path = sign_path[:-len(gpg_sign_ext)]
|
|
if os.path.isfile(target_path) and \
|
|
os.access(target_path, os.R_OK):
|
|
to_be_verified.append((target_path, sign_path,))
|
|
|
|
gpg_rc = 0
|
|
|
|
for target_path, sign_path in to_be_verified:
|
|
|
|
file_name = os.path.basename(target_path)
|
|
|
|
mytxt = "%s: %s ..." % (
|
|
darkgreen(_("Verifying GPG signature of")),
|
|
brown(file_name),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = blue("\t@@ "),
|
|
back = True
|
|
)
|
|
|
|
is_valid, err_msg = repo_sec.verify_file(self._repository_id,
|
|
target_path, sign_path)
|
|
if is_valid:
|
|
mytxt = "%s: %s" % (
|
|
darkgreen(_("Verified GPG signature of")),
|
|
brown(file_name),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "info",
|
|
header = blue("\t@@ ")
|
|
)
|
|
else:
|
|
mytxt = "%s: %s" % (
|
|
darkred(_("Error during GPG verification of")),
|
|
file_name,
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "error",
|
|
header = "\t%s " % (bold("!!!"),)
|
|
)
|
|
mytxt = "%s: %s" % (
|
|
purple(_("It could mean a potential security risk")),
|
|
err_msg,
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
level = "error",
|
|
header = "\t%s " % (bold("!!!"),)
|
|
)
|
|
gpg_rc = 1
|
|
|
|
return gpg_rc
|
|
|
|
def _handle_webserv_database_sync(self):
|
|
|
|
repo_db = None
|
|
try:
|
|
repo_db = self.__get_webserv_local_database()
|
|
if repo_db is None:
|
|
raise AttributeError()
|
|
return self.__handle_webserv_database_sync(repo_db)
|
|
except (DatabaseError, IntegrityError, OperationalError,
|
|
AttributeError,):
|
|
return False
|
|
finally:
|
|
if repo_db is not None:
|
|
repo_db.close()
|
|
|
|
return False
|
|
|
|
def __handle_webserv_database_sync(self, mydbconn):
|
|
|
|
try:
|
|
webserv = self._webservice
|
|
except WebService.UnsupportedService as err:
|
|
const_debug_write(__name__,
|
|
"__handle_webserv_database_sync: error: %s" % (err,))
|
|
return False
|
|
|
|
try:
|
|
myidpackages = mydbconn.listAllPackageIds()
|
|
except (DatabaseError, IntegrityError, OperationalError,):
|
|
return False
|
|
|
|
added_ids, removed_ids = self.__get_webserv_database_differences(
|
|
webserv, myidpackages)
|
|
if (None in (added_ids, removed_ids)) or \
|
|
(not added_ids and not removed_ids and self.__force):
|
|
# nothing to sync, it seems, if force is True, fallback to EAPI2
|
|
return False
|
|
|
|
threshold = 500
|
|
# is it worth it?
|
|
if len(added_ids) > threshold:
|
|
mytxt = "%s: %s (%s: %s/%s)" % (
|
|
blue(_("Web Service")),
|
|
darkred(_("skipping differential sync")),
|
|
brown(_("threshold")),
|
|
blue(str(len(added_ids))),
|
|
darkred(str(threshold)),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # "),
|
|
)
|
|
return False
|
|
|
|
count = 0
|
|
chunk_size = RepositoryWebService.MAXIMUM_PACKAGE_REQUEST_SIZE
|
|
added_segments = []
|
|
mytmp = []
|
|
|
|
for package_id in added_ids:
|
|
count += 1
|
|
mytmp.append(package_id)
|
|
if count % chunk_size == 0:
|
|
added_segments.append(mytmp[:])
|
|
del mytmp[:]
|
|
if mytmp:
|
|
added_segments.append(mytmp[:])
|
|
|
|
def _do_fetch(fetch_sts_map, segment, count, maxcount):
|
|
try:
|
|
try:
|
|
pkg_meta = webserv.get_packages_metadata(segment)
|
|
except WebService.WebServiceException as err:
|
|
const_debug_write(__name__,
|
|
"__handle_webserv_database_sync: error: %s" % (err,))
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Web Service communication error")),
|
|
err,
|
|
)
|
|
self._entropy.output(
|
|
mytxt, importance = 1, level = "info",
|
|
header = "\t", count = (count, maxcount,)
|
|
)
|
|
fetch_sts_map['error'] = True
|
|
return
|
|
except KeyboardInterrupt:
|
|
const_debug_write(__name__,
|
|
"__handle_webserv_database_sync: keyboard interrupt")
|
|
fetch_sts_map['error'] = True
|
|
return
|
|
|
|
if not pkg_meta:
|
|
const_debug_write(__name__,
|
|
"__handle_webserv_database_sync: empty data: %s" % (
|
|
pkg_meta,))
|
|
self._entropy.output(
|
|
_("Web Service data error"), importance = 1,
|
|
level = "info", header = "\t",
|
|
count = (count, maxcount,)
|
|
)
|
|
fetch_sts_map['error'] = True
|
|
return
|
|
|
|
try:
|
|
for package_id, pkg_data in pkg_meta.items():
|
|
dumpobj(
|
|
"%s%s" % (self.WEBSERV_CACHE_ID, package_id,),
|
|
pkg_data,
|
|
ignore_exceptions = False
|
|
)
|
|
except (IOError, EOFError, OSError,) as e:
|
|
mytxt = "%s: %s: %s." % (
|
|
blue(_("Local status")),
|
|
darkred("Error storing data"),
|
|
e,
|
|
)
|
|
self._entropy.output(
|
|
mytxt, importance = 1, level = "info",
|
|
header = "\t", count = (count, maxcount,)
|
|
)
|
|
fetch_sts_map['error'] = True
|
|
return
|
|
|
|
return
|
|
finally:
|
|
fetch_sts_map['sem'].release()
|
|
|
|
# do not exagerate or you're going to need a way to block
|
|
# further requests as long as some threads are still running
|
|
# to avoid timeout errors
|
|
max_threads = 4
|
|
fetch_sts_map = {
|
|
'sem': threading.Semaphore(max_threads),
|
|
'error': False,
|
|
}
|
|
|
|
# fetch and store
|
|
count = 0
|
|
maxcount = len(added_segments)
|
|
product = self._settings['repositories']['product']
|
|
segment = None
|
|
threads = []
|
|
for segment in added_segments:
|
|
count += 1
|
|
mytxt = "%s %s" % (blue(_("Fetching segments")), "...",)
|
|
self._entropy.output(
|
|
mytxt, importance = 0, level = "info",
|
|
header = "\t", back = True, count = (count, maxcount,)
|
|
)
|
|
fetch_sts_map['sem'].acquire()
|
|
if len(threads) >= max_threads:
|
|
const_debug_write(__name__,
|
|
purple("joining all the parallel threads"))
|
|
# give them the chance to complete
|
|
# since long delays on socket could cause timeouts
|
|
for th in threads:
|
|
th.join()
|
|
const_debug_write(__name__, purple("parallel threads joined"))
|
|
del threads[:]
|
|
if fetch_sts_map['error']:
|
|
return None
|
|
th = ParallelTask(_do_fetch, fetch_sts_map, segment, count,
|
|
maxcount)
|
|
th.daemon = True
|
|
th.start()
|
|
threads.append(th)
|
|
|
|
for th in threads:
|
|
th.join()
|
|
|
|
if fetch_sts_map['error']:
|
|
return None
|
|
|
|
del added_segments
|
|
|
|
# get repository metadata
|
|
repo_metadata = self.__get_webserv_repository_metadata()
|
|
# this gives us the "checksum" data too
|
|
if not repo_metadata:
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Web Service status")),
|
|
darkred(_("cannot fetch repository metadata")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # "),
|
|
)
|
|
return None
|
|
|
|
# update treeupdates
|
|
try:
|
|
mydbconn.setRepositoryUpdatesDigest(self._repository_id,
|
|
repo_metadata['treeupdates_digest'])
|
|
mydbconn.bumpTreeUpdatesActions(
|
|
repo_metadata['treeupdates_actions'])
|
|
except (Error,):
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Web Service status")),
|
|
darkred(_("cannot update treeupdates data")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # "),
|
|
)
|
|
return None
|
|
|
|
# update package sets
|
|
try:
|
|
mydbconn.clearPackageSets()
|
|
mydbconn.insertPackageSets(repo_metadata['sets'])
|
|
except (Error,):
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Web Service status")),
|
|
darkred(_("cannot update package sets data")),
|
|
)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 0,
|
|
level = "info",
|
|
header = blue(" # "),
|
|
)
|
|
return None
|
|
|
|
# now that we have all stored, add
|
|
for idpackage in added_ids:
|
|
mydata = self._cacher.pop("%s%s" % (self.WEBSERV_CACHE_ID,
|
|
idpackage,))
|
|
if mydata is None:
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Fetch error on segment while adding")),
|
|
darkred(str(segment)),
|
|
)
|
|
self._entropy.output(
|
|
mytxt, importance = 1, level = "warning",
|
|
header = " "
|
|
)
|
|
return False
|
|
|
|
mytxt = "%s %s" % (
|
|
darkgreen("++"),
|
|
teal(mydata['atom']),
|
|
)
|
|
self._entropy.output(
|
|
mytxt, importance = 0, level = "info",
|
|
header = " ", back = (not etpUi['verbose'])
|
|
)
|
|
try:
|
|
mydbconn.addPackage(
|
|
mydata, revision = mydata['revision'],
|
|
package_id = idpackage, do_commit = False,
|
|
formatted_content = True
|
|
)
|
|
except (Error,) as err:
|
|
if etpUi['debug']:
|
|
entropy.tools.print_traceback()
|
|
self._entropy.output("%s: %s" % (
|
|
blue(_("repository error while adding packages")),
|
|
err,),
|
|
importance = 1, level = "warning",
|
|
header = " "
|
|
)
|
|
return False
|
|
|
|
# now remove
|
|
# preload atoms names to improve speed during removePackage
|
|
atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids)
|
|
for idpackage in removed_ids:
|
|
myatom = atoms_map.get(idpackage)
|
|
|
|
mytxt = "%s %s" % (
|
|
darkred("--"),
|
|
purple(str(myatom)),)
|
|
self._entropy.output(
|
|
mytxt, importance = 0, level = "info",
|
|
header = " ", back = (not etpUi['verbose'])
|
|
)
|
|
try:
|
|
mydbconn.removePackage(idpackage, do_cleanup = False,
|
|
do_commit = False)
|
|
except (Error,):
|
|
self._entropy.output(
|
|
blue(_("repository error while removing packages")),
|
|
importance = 1, level = "warning",
|
|
header = " "
|
|
)
|
|
return False
|
|
|
|
mydbconn.commit()
|
|
mydbconn.clearCache()
|
|
# now verify if both checksums match
|
|
result = False
|
|
mychecksum = mydbconn.checksum(do_order = True,
|
|
strict = False, strings = True, include_signatures = True)
|
|
if repo_metadata['checksum'] == mychecksum:
|
|
result = True
|
|
else:
|
|
self._entropy.output(
|
|
blue(_("Repository checksum doesn't match remote.")),
|
|
importance = 0, level = "info", header = "\t",
|
|
)
|
|
mytxt = "%s: %s" % (_('local'), mychecksum,)
|
|
self._entropy.output(
|
|
mytxt, importance = 0,
|
|
level = "info", header = "\t",
|
|
)
|
|
mytxt = "%s: %s" % (_('remote'), repo_metadata['checksum'],)
|
|
self._entropy.output(
|
|
mytxt, importance = 0,
|
|
level = "info", header = "\t",
|
|
)
|
|
|
|
return result
|
|
|
|
def remote_revision(self):
|
|
|
|
if self._repo_eapi == 3:
|
|
# ask WebService then
|
|
revision = self.__get_webserv_repository_revision()
|
|
if revision is not None:
|
|
try:
|
|
revision = int(revision)
|
|
except ValueError:
|
|
revision = None
|
|
if revision is not None:
|
|
self._last_rev = revision
|
|
return revision
|
|
# otherwise, fallback to previous EAPI
|
|
self._repo_eapi -= 1
|
|
|
|
avail_data = self._settings['repositories']['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
url = repo_data['database'] + "/" + \
|
|
etpConst['etpdatabaserevisionfile']
|
|
status = entropy.tools.get_remote_data(url,
|
|
timeout = self.__big_sock_timeout)
|
|
if status:
|
|
status = status[0].strip()
|
|
try:
|
|
status = int(status)
|
|
except ValueError:
|
|
status = -1
|
|
else:
|
|
status = -1
|
|
|
|
self._last_rev = status
|
|
return status
|
|
|
|
def update(self):
|
|
|
|
# disallow unprivileged update
|
|
if not entropy.tools.is_root():
|
|
raise PermissionDenied(
|
|
"cannot update repository as unprivileged user")
|
|
|
|
self.__show_repository_information()
|
|
|
|
# this calls writes self._last_rev which is used to write back
|
|
# updated repository revision, do not remove!
|
|
updatable = self.__is_repository_updatable()
|
|
if not self.__force:
|
|
if not updatable:
|
|
mytxt = "%s: %s." % (bold(_("Attention")),
|
|
red(_("repository is already up to date")),)
|
|
self._entropy.output(
|
|
mytxt,
|
|
importance = 1,
|
|
level = "info",
|
|
header = "\t"
|
|
)
|
|
return EntropyRepositoryBase.REPOSITORY_ALREADY_UPTODATE
|
|
|
|
locked = self.__handle_repository_lock()
|
|
if locked:
|
|
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
|
|
|
|
# clear database interface cache belonging to this repository
|
|
self.__ensure_repository_path()
|
|
|
|
# dealing with EAPI
|
|
# setting some vars
|
|
db_checksum_down_status = False
|
|
do_db_update_transfer = False
|
|
rc = 0
|
|
|
|
my_repos = self._settings['repositories']
|
|
avail_data = my_repos['available']
|
|
repo_data = avail_data[self._repository_id]
|
|
|
|
# some variables
|
|
dumpfile = os.path.join(repo_data['dbpath'],
|
|
etpConst['etpdatabasedumplight'])
|
|
dbfile = os.path.join(repo_data['dbpath'],
|
|
etpConst['etpdatabasefile'])
|
|
dbfile_old = dbfile+".sync"
|
|
cmethod = self.__validate_compression_method()
|
|
|
|
while True:
|
|
|
|
downloaded_db_item = None
|
|
sig_down_status = False
|
|
db_checksum_down_status = False
|
|
if self._repo_eapi < 3:
|
|
|
|
down_status, sig_down_status, downloaded_db_item = \
|
|
self.__handle_database_download(cmethod)
|
|
if not down_status:
|
|
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
|
|
db_checksum_down_status = \
|
|
self.__handle_database_checksum_download(cmethod)
|
|
break
|
|
|
|
elif self._repo_eapi == 3 and not \
|
|
(os.path.isfile(dbfile) and os.access(dbfile, os.W_OK)):
|
|
do_db_update_transfer = None
|
|
self._repo_eapi -= 1
|
|
continue
|
|
|
|
elif self._repo_eapi == 3:
|
|
|
|
status = False
|
|
try:
|
|
status = self._handle_webserv_database_sync()
|
|
except:
|
|
# avoid broken entries, deal with every exception
|
|
self.__remove_repository_files()
|
|
raise
|
|
|
|
if status is None: # remote db not available anymore ?
|
|
time.sleep(5)
|
|
locked = self.__handle_repository_lock()
|
|
if locked:
|
|
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
|
|
do_db_update_transfer = None
|
|
self._repo_eapi -= 1
|
|
continue
|
|
|
|
elif not status: # (status == False)
|
|
# set to none and completely skip database alignment
|
|
do_db_update_transfer = None
|
|
self._repo_eapi -= 1
|
|
continue
|
|
|
|
break
|
|
|
|
downloaded_files = self._standard_items_download()
|
|
# also add db file to downloaded item
|
|
# and md5 check repository
|
|
if downloaded_db_item is not None:
|
|
|
|
durl, dpath = self._construct_paths(downloaded_db_item,
|
|
cmethod)
|
|
downloaded_files.append(dpath)
|
|
if sig_down_status:
|
|
d_sig_path = self.__append_gpg_signature_to_path(dpath)
|
|
downloaded_files.append(d_sig_path)
|
|
|
|
# 1. we're always in EAPI1 or 2 here
|
|
# 2. new policy, always deny repository if
|
|
# its database checksum cannot be fetched
|
|
if not db_checksum_down_status:
|
|
# delete all
|
|
self.__remove_repository_files()
|
|
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
|
|
|
|
rc = self._check_downloaded_database(cmethod)
|
|
if rc != 0:
|
|
# delete all
|
|
self.__remove_repository_files()
|
|
return EntropyRepositoryBase.REPOSITORY_CHECKSUM_ERROR
|
|
|
|
# GPG pubkey install hook
|
|
if self._gpg_feature:
|
|
gpg_available = self._install_gpg_key_if_available()
|
|
if gpg_available:
|
|
gpg_rc = self._gpg_verify_downloaded_files(downloaded_files)
|
|
|
|
# Now we can unpack
|
|
files_to_remove = []
|
|
if self._repo_eapi in (1, 2,):
|
|
|
|
# if do_db_update_transfer == False and not None
|
|
if (do_db_update_transfer is not None) and not \
|
|
do_db_update_transfer:
|
|
|
|
if os.access(dbfile, os.R_OK | os.W_OK) and \
|
|
os.path.isfile(dbfile):
|
|
try:
|
|
os.rename(dbfile, dbfile_old)
|
|
do_db_update_transfer = True
|
|
except OSError:
|
|
do_db_update_transfer = False
|
|
|
|
unpack_status, unpacked_item = \
|
|
self.__handle_downloaded_database_unpack(cmethod)
|
|
|
|
if not unpack_status:
|
|
# delete all
|
|
self.__remove_repository_files()
|
|
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
|
|
|
|
unpack_url, unpack_path = self._construct_paths(unpacked_item,
|
|
cmethod)
|
|
files_to_remove.append(unpack_path)
|
|
|
|
# re-validate
|
|
if not os.path.isfile(dbfile):
|
|
do_db_update_transfer = False
|
|
|
|
elif os.path.isfile(dbfile) and not do_db_update_transfer and \
|
|
(self._repo_eapi != 1):
|
|
os.remove(dbfile)
|
|
|
|
if self._repo_eapi == 2:
|
|
rc = self.__eapi2_inject_downloaded_dump(dumpfile,
|
|
dbfile, cmethod)
|
|
|
|
if do_db_update_transfer:
|
|
self.__eapi1_eapi2_databases_alignment(dbfile, dbfile_old)
|
|
|
|
if self._repo_eapi == 2:
|
|
# remove the dump
|
|
files_to_remove.append(dumpfile)
|
|
|
|
if rc != 0:
|
|
# delete all
|
|
self.__remove_repository_files()
|
|
files_to_remove.append(dbfile_old)
|
|
for path in files_to_remove:
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
continue
|
|
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
|
|
|
|
# make sure that all the repository files are stored with proper
|
|
# permissions to avoid possible XSS and trust boundary problems.
|
|
downloaded_files.append(dbfile)
|
|
for downloaded_file in sorted(set(downloaded_files)):
|
|
if os.path.isfile(downloaded_file) and \
|
|
os.access(downloaded_file, os.W_OK | os.R_OK):
|
|
const_setup_file(downloaded_file, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
|
|
# remove garbage left around
|
|
for path in files_to_remove:
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
continue
|
|
|
|
valid = self.__validate_database()
|
|
if not valid:
|
|
# repository failed validation
|
|
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
|
|
|
|
self.__update_repository_revision()
|
|
if self._entropy._indexing:
|
|
self.__database_indexing()
|
|
|
|
try:
|
|
spm_class = self._entropy.Spm_class()
|
|
spm_class.entropy_client_post_repository_update_hook(
|
|
self._entropy, self._repository_id)
|
|
except Exception as err:
|
|
entropy.tools.print_traceback()
|
|
mytxt = "%s: %s" % (
|
|
blue(_("Configuration files update error, not critical, continuing")),
|
|
err,
|
|
)
|
|
self._entropy.output(mytxt, importance = 0,
|
|
level = "info", header = blue(" # "),)
|
|
|
|
# remove garbage
|
|
if os.access(dbfile_old, os.R_OK) and os.path.isfile(dbfile_old):
|
|
os.remove(dbfile_old)
|
|
|
|
return EntropyRepositoryBase.REPOSITORY_UPDATED_OK
|
|
|
|
|
|
_CL_PLUGIN_ID = etpConst['system_settings_plugins_ids']['client_plugin']
|
|
|
|
class MaskableRepository(EntropyRepositoryBase):
|
|
"""
|
|
Objects inheriting from this class support package masking.
|
|
A masked package is a package that is not visible to user and thus not
|
|
selectable in dependency calculation and also not directly installable.
|
|
The only repositories that need to support the feature are those containing
|
|
installable packages, like AvailablePackagesRepository.
|
|
"""
|
|
_MASK_FILTER_CACHE_ID = EntropyCacher.CACHE_IDS['mask_filter']
|
|
|
|
def _mask_filter_fetch_cache(self, package_id):
|
|
if self._caching:
|
|
return loadobj("%s/%s/%s" % (
|
|
MaskableRepository._MASK_FILTER_CACHE_ID, self.name,
|
|
package_id,))
|
|
|
|
def _mask_filter_store_cache(self, package_id, value):
|
|
if self._caching:
|
|
dumpobj("%s/%s/%s" % (MaskableRepository._MASK_FILTER_CACHE_ID,
|
|
self.name, package_id,), value)
|
|
|
|
def _maskFilter_live(self, package_id):
|
|
|
|
ref = self._settings['pkg_masking_reference']
|
|
if (package_id, self.name) in \
|
|
self._settings['live_packagemasking']['mask_matches']:
|
|
|
|
# do not cache this
|
|
return -1, ref['user_live_mask']
|
|
|
|
elif (package_id, self.name) in \
|
|
self._settings['live_packagemasking']['unmask_matches']:
|
|
|
|
return package_id, ref['user_live_unmask']
|
|
|
|
def _maskFilter_user_package_mask(self, package_id, live):
|
|
|
|
with self._settings['mask']:
|
|
# thread-safe in here
|
|
cache_obj = self._settings['mask'].get()
|
|
if cache_obj is None:
|
|
cache_obj = {}
|
|
self._settings['mask'].set(cache_obj)
|
|
user_package_mask_ids = cache_obj.get(self.name)
|
|
|
|
if user_package_mask_ids is None:
|
|
user_package_mask_ids = set()
|
|
|
|
for atom in self._settings['mask']:
|
|
atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
|
|
if repository_ids is not None:
|
|
if self.name not in repository_ids:
|
|
# then the mask doesn't involve us
|
|
continue
|
|
# check if @repository is specified
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
user_package_mask_ids |= set(matches)
|
|
|
|
cache_obj[self.name] = user_package_mask_ids
|
|
|
|
if package_id in user_package_mask_ids:
|
|
# sorry, masked
|
|
ref = self._settings['pkg_masking_reference']
|
|
myr = ref['user_package_mask']
|
|
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = -1, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return -1, myr
|
|
|
|
def _maskFilter_user_package_unmask(self, package_id, live):
|
|
|
|
with self._settings['unmask']:
|
|
# thread-safe in here
|
|
cache_obj = self._settings['unmask'].get()
|
|
if cache_obj is None:
|
|
cache_obj = {}
|
|
self._settings['unmask'].set(cache_obj)
|
|
user_package_unmask_ids = cache_obj.get(self.name)
|
|
|
|
if user_package_unmask_ids is None:
|
|
|
|
user_package_unmask_ids = set()
|
|
for atom in self._settings['unmask']:
|
|
atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
|
|
if repository_ids is not None:
|
|
if self.name not in repository_ids:
|
|
# then the mask doesn't involve us
|
|
continue
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
user_package_unmask_ids |= set(matches)
|
|
|
|
cache_obj[self.name] = user_package_unmask_ids
|
|
|
|
if package_id in user_package_unmask_ids:
|
|
|
|
ref = self._settings['pkg_masking_reference']
|
|
myr = ref['user_package_unmask']
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return package_id, myr
|
|
|
|
def _maskFilter_packages_db_mask(self, package_id, live):
|
|
|
|
# check if repository packages.db.mask needs it masked
|
|
repos_mask = {}
|
|
client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin']
|
|
client_settings = self._settings.get(client_plg_id, {})
|
|
if client_settings:
|
|
repos_mask = client_settings['repositories']['mask']
|
|
|
|
repomask = repos_mask.get(self.name)
|
|
if isinstance(repomask, (list, set, frozenset)):
|
|
|
|
# first, seek into generic masking, all branches
|
|
# (below) avoid issues with repository names
|
|
mask_repo_id = "%s_ids@@:of:%s" % (self.name, self.name,)
|
|
repomask_ids = repos_mask.get(mask_repo_id)
|
|
|
|
if not isinstance(repomask_ids, set):
|
|
repomask_ids = set()
|
|
for atom in repomask:
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
repomask_ids |= set(matches)
|
|
repos_mask[mask_repo_id] = repomask_ids
|
|
|
|
if package_id in repomask_ids:
|
|
|
|
ref = self._settings['pkg_masking_reference']
|
|
myr = ref['repository_packages_db_mask']
|
|
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
-1, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return -1, myr
|
|
|
|
def _maskFilter_package_license_mask(self, package_id, live):
|
|
|
|
if not self._settings['license_mask']:
|
|
return
|
|
|
|
mylicenses = self.retrieveLicense(package_id)
|
|
mylicenses = mylicenses.strip().split()
|
|
lic_mask = self._settings['license_mask']
|
|
for mylicense in mylicenses:
|
|
|
|
if mylicense not in lic_mask:
|
|
continue
|
|
|
|
ref = self._settings['pkg_masking_reference']
|
|
myr = ref['user_license_mask']
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = -1, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return -1, myr
|
|
|
|
def _maskFilter_keyword_mask(self, package_id, live):
|
|
|
|
# WORKAROUND for buggy entries
|
|
# ** is fine then
|
|
# TODO: remove this before 31-12-2011
|
|
mykeywords = self.retrieveKeywords(package_id)
|
|
if mykeywords == set([""]):
|
|
mykeywords = set(['**'])
|
|
|
|
mask_ref = self._settings['pkg_masking_reference']
|
|
|
|
# firstly, check if package keywords are in etpConst['keywords']
|
|
# (universal keywords have been merged from package.keywords)
|
|
same_keywords = etpConst['keywords'] & mykeywords
|
|
if same_keywords:
|
|
myr = mask_ref['system_keyword']
|
|
try:
|
|
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return package_id, myr
|
|
|
|
# if we get here, it means we didn't find mykeywords
|
|
# in etpConst['keywords']
|
|
# we need to seek self._settings['keywords']
|
|
# seek in repository first
|
|
keyword_repo = self._settings['keywords']['repositories']
|
|
|
|
for keyword in tuple(keyword_repo.get(self.name, {}).keys()):
|
|
|
|
if keyword not in mykeywords:
|
|
continue
|
|
|
|
keyword_data = keyword_repo[self.name].get(keyword)
|
|
if not keyword_data:
|
|
continue
|
|
|
|
if "*" in keyword_data:
|
|
# all packages in this repo with keyword "keyword" are ok
|
|
myr = mask_ref['user_repo_package_keywords_all']
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return package_id, myr
|
|
|
|
kwd_key = "%s_ids" % (keyword,)
|
|
keyword_data_ids = keyword_repo[self.name].get(kwd_key)
|
|
if not isinstance(keyword_data_ids, set):
|
|
|
|
keyword_data_ids = set()
|
|
for atom in keyword_data:
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
keyword_data_ids |= matches
|
|
|
|
keyword_repo[self.name][kwd_key] = keyword_data_ids
|
|
|
|
if package_id in keyword_data_ids:
|
|
|
|
myr = mask_ref['user_repo_package_keywords']
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
return package_id, myr
|
|
|
|
keyword_pkg = self._settings['keywords']['packages']
|
|
|
|
# if we get here, it means we didn't find a match in repositories
|
|
# so we scan packages, last chance
|
|
for keyword in keyword_pkg.keys():
|
|
# use .keys() because keyword_pkg gets modified during iteration
|
|
|
|
# first of all check if keyword is in mykeywords
|
|
if keyword not in mykeywords:
|
|
continue
|
|
|
|
keyword_data = keyword_pkg.get(keyword)
|
|
if not keyword_data:
|
|
continue
|
|
|
|
kwd_key = "%s_ids" % (keyword,)
|
|
keyword_data_ids = keyword_pkg.get(self.name+kwd_key)
|
|
|
|
if not isinstance(keyword_data_ids, (list, set)):
|
|
keyword_data_ids = set()
|
|
for atom in keyword_data:
|
|
# match atom
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
keyword_data_ids |= matches
|
|
|
|
keyword_pkg[self.name+kwd_key] = keyword_data_ids
|
|
|
|
if package_id in keyword_data_ids:
|
|
|
|
# valid!
|
|
myr = mask_ref['user_package_keywords']
|
|
try:
|
|
cl_data = self._settings[_CL_PLUGIN_ID]
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
except KeyError: # system settings client plugin not found
|
|
pass
|
|
|
|
return package_id, myr
|
|
|
|
|
|
## if we get here, it means that pkg it keyword masked
|
|
## and we should look at the very last resort, per-repository
|
|
## package keywords
|
|
# check if repository contains keyword unmasking data
|
|
|
|
cl_data = self._settings.get(_CL_PLUGIN_ID)
|
|
if cl_data is None:
|
|
# SystemSettings Entropy Client plugin not available
|
|
return
|
|
# let's see if something is available in repository config
|
|
repo_keywords = cl_data['repositories']['repos_keywords'].get(
|
|
self.name)
|
|
if repo_keywords is None:
|
|
# nopers, sorry!
|
|
return
|
|
|
|
# check universal keywords
|
|
same_keywords = repo_keywords.get('universal') & mykeywords
|
|
if same_keywords:
|
|
# universal keyword matches!
|
|
myr = mask_ref['repository_packages_db_keywords']
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
return package_id, myr
|
|
|
|
## if we get here, it means that even universal masking failed
|
|
## and we need to look at per-package settings
|
|
repo_settings = repo_keywords.get('packages')
|
|
if not repo_settings:
|
|
# it's empty, not worth checking
|
|
return
|
|
|
|
cached_key = "packages_ids"
|
|
keyword_data_ids = repo_keywords.get(cached_key)
|
|
if not isinstance(keyword_data_ids, dict):
|
|
# create cache
|
|
|
|
keyword_data_ids = {}
|
|
for atom, values in repo_settings.items():
|
|
matches, r = self.atomMatch(atom, multiMatch = True,
|
|
maskFilter = False)
|
|
if r != 0:
|
|
continue
|
|
for match in matches:
|
|
obj = keyword_data_ids.setdefault(match, set())
|
|
obj.update(values)
|
|
|
|
repo_keywords[cached_key] = keyword_data_ids
|
|
|
|
pkg_keywords = keyword_data_ids.get(package_id, set())
|
|
if "**" in pkg_keywords:
|
|
same_keywords = True
|
|
else:
|
|
same_keywords = pkg_keywords & etpConst['keywords']
|
|
if same_keywords:
|
|
# found! this pkg is not masked, yay!
|
|
myr = mask_ref['repository_packages_db_keywords']
|
|
validator_cache = cl_data['masking_validation']['cache']
|
|
validator_cache[(package_id, self.name, live)] = \
|
|
package_id, myr
|
|
return package_id, myr
|
|
|
|
def maskFilter(self, package_id, live = True):
|
|
"""
|
|
Reimplemented from EntropyRepositoryBase
|
|
"""
|
|
validator_cache = self._settings.get(_CL_PLUGIN_ID, {}).get(
|
|
'masking_validation', {}).get('cache', {})
|
|
|
|
cached = validator_cache.get((package_id, self.name, live))
|
|
if cached is not None:
|
|
return cached
|
|
|
|
# use on-disk cache?
|
|
cached = self._mask_filter_fetch_cache(package_id)
|
|
if cached is not None:
|
|
return cached
|
|
|
|
# avoid memleaks
|
|
if len(validator_cache) > 100000:
|
|
validator_cache.clear()
|
|
|
|
if live:
|
|
data = self._maskFilter_live(package_id)
|
|
if data:
|
|
return data
|
|
|
|
data = self._maskFilter_user_package_mask(package_id, live)
|
|
if data:
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return data
|
|
|
|
data = self._maskFilter_user_package_unmask(package_id, live)
|
|
if data:
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return data
|
|
|
|
data = self._maskFilter_packages_db_mask(package_id, live)
|
|
if data:
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return data
|
|
|
|
data = self._maskFilter_package_license_mask(package_id, live)
|
|
if data:
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return data
|
|
|
|
data = self._maskFilter_keyword_mask(package_id, live)
|
|
if data:
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return data
|
|
|
|
# holy crap, can't validate
|
|
myr = self._settings['pkg_masking_reference']['completely_masked']
|
|
validator_cache[(package_id, self.name, live)] = -1, myr
|
|
self._mask_filter_store_cache(package_id, data)
|
|
return -1, myr
|
|
|
|
|
|
class AvailablePackagesRepository(CachedRepository, MaskableRepository):
|
|
"""
|
|
This class represents the available packages repository and is a direct
|
|
subclass of EntropyRepository. It implements the update() method in order
|
|
to make possible to update the repository.
|
|
"""
|
|
def __init__(self, *args, **kwargs):
|
|
EntropyRepository.__init__(self, *args, **kwargs)
|
|
# ensure proper repository file permissions
|
|
if entropy.tools.is_root() and os.path.isfile(self._db_path):
|
|
const_setup_file(self._db_path, etpConst['entropygid'], 0o644,
|
|
uid = etpConst['uid'])
|
|
|
|
@staticmethod
|
|
def update(entropy_client, repository_id, force, gpg):
|
|
"""
|
|
Reimplemented from EntropyRepositoryBase
|
|
"""
|
|
try:
|
|
return AvailablePackagesRepositoryUpdater(entropy_client, repository_id,
|
|
force, gpg).update()
|
|
except KeyError:
|
|
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
|
|
|
|
@staticmethod
|
|
def revision(repository_id):
|
|
"""
|
|
Reimplemented from EntropyRepositoryBase
|
|
"""
|
|
db_data = SystemSettings()['repositories']['available'][repository_id]
|
|
fname = os.path.join(db_data['dbpath'],
|
|
etpConst['etpdatabaserevisionfile'])
|
|
revision = -1
|
|
|
|
enc = etpConst['conf_encoding']
|
|
if os.path.isfile(fname) and os.access(fname, os.R_OK):
|
|
with codecs.open(fname, "r", encoding=enc) as f:
|
|
try:
|
|
read_data = f.readline().strip()
|
|
revision = int(read_data)
|
|
except (OSError, IOError, ValueError,):
|
|
pass
|
|
|
|
return revision
|
|
|
|
@staticmethod
|
|
def remote_revision(repository_id):
|
|
"""
|
|
Reimplemented from EntropyRepositoryBase
|
|
"""
|
|
return AvailablePackagesRepositoryUpdater(TextInterface(),
|
|
repository_id, False, False).remote_revision()
|
|
|
|
def handlePackage(self, pkg_data, forcedRevision = -1,
|
|
formattedContent = False):
|
|
"""
|
|
Reimplemented from EntropyRepository
|
|
"""
|
|
raise PermissionDenied(
|
|
"cannot execute handlePackage on this repository")
|
|
|
|
def addPackage(self, pkg_data, revision = -1, package_id = None,
|
|
do_commit = True, formatted_content = False):
|
|
"""
|
|
Reimplemented from EntropyRepository
|
|
"""
|
|
raise PermissionDenied(
|
|
"cannot execute addPackage on this repository")
|
|
|
|
def removePackage(self, package_id, do_cleanup = True, do_commit = True,
|
|
from_add_package = False):
|
|
"""
|
|
Reimplemented from EntropyRepository
|
|
"""
|
|
raise PermissionDenied(
|
|
"cannot execute removePackage on this repository")
|
|
|
|
def clearCache(self):
|
|
# clear package masking filter
|
|
cl_data = self._settings.get(_CL_PLUGIN_ID, {})
|
|
cl_data.get('masking_validation', {}).get('cache', {}).clear()
|
|
EntropyRepository.clearCache(self)
|
|
|
|
|
|
class GenericRepository(CachedRepository, MaskableRepository):
|
|
"""
|
|
This class represents a generic packages repository and is a direct
|
|
subclass of EntropyRepository.
|
|
Even GenericRepository is a CachedRepository because its object could
|
|
get cached by 3rd party. Actually, we require this because our installed
|
|
packages repository could end up being a GenericRepository, when running
|
|
in fail-safe mode.
|
|
"""
|
|
|
|
def handlePackage(self, pkg_data, forcedRevision = -1,
|
|
formattedContent = False):
|
|
"""
|
|
Reimplemented from EntropyRepository.
|
|
It is supposed that a generic repository should not support
|
|
handlePackage. You can override this (at your own risk) by setting the
|
|
"override_handlePackage" property to True. In this case, a generic
|
|
addPackage() call is issued.
|
|
"""
|
|
override = getattr(self, 'override_handlePackage', False)
|
|
if not override:
|
|
raise PermissionDenied(
|
|
"cannot execute handlePackage on this repository")
|
|
|
|
return self.addPackage(pkg_data, revision = forcedRevision,
|
|
formatted_content = formattedContent)
|
|
|
|
def maskFilter(self, package_id, live = True):
|
|
"""
|
|
Reimplemented from EntropyRepository.
|
|
It is supposed that a generic repository doesn't support package
|
|
masking. You can override this by setting the "enable_mask_filter"
|
|
to True.
|
|
"""
|
|
enabled = getattr(self, 'enable_mask_filter', False)
|
|
if not enabled:
|
|
return package_id, 0
|
|
return MaskableRepository.maskFilter(self, package_id, live = live)
|