sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
Files
050e700dcf5a875fa2877bef27063cf9d9df1ba6
entropy/lib/entropy/client/interfaces/db.py
T
Fabio Erculiani 050e700dcf [entropy.client.db] fallback to package URLs if the repository database URL is broken. 
This commit introduces the ability for Entropy Client to fall back to
package URLs if the repository database URL is not responding as expected
(tested at the application level).
This improves fault-tolerance greatly.
2012-09-20 12:55:34 +02:00

2675 lines
93 KiB
Python
Raw Blame History

# -*- coding: utf-8 -*-
"""
@author: Fabio Erculiani <lxnay@sabayon.org>
@contact: lxnay@sabayon.org
@copyright: Fabio Erculiani
@license: GPL-2
B{Entropy Package Manager Client EntropyRepository plugin code}.
"""
import os
import sys
import subprocess
import tempfile
import threading
import shutil
import time
import codecs
import errno
from entropy.const import const_debug_write, const_setup_perms, etpConst, \
etpUi, const_set_nice_level, const_setup_file, const_convert_to_unicode
from entropy.output import blue, darkred, red, darkgreen, purple, teal, brown, \
bold, TextInterface
from entropy.dump import dumpobj, loadobj
from entropy.cache import EntropyCacher
from entropy.db import EntropyRepository
from entropy.exceptions import RepositoryError, SystemDatabaseError, \
PermissionDenied
from entropy.security import Repository as RepositorySecurity
from entropy.misc import TimeScheduled, ParallelTask
from entropy.fetchers import UrlFetcher
from entropy.i18n import _
from entropy.db.skel import EntropyRepositoryPlugin, EntropyRepositoryBase
from entropy.db.exceptions import IntegrityError, OperationalError, Error, \
DatabaseError
from entropy.core.settings.base import SystemSettings
from entropy.services.client import WebService
from entropy.client.services.interfaces import RepositoryWebService, \
RepositoryWebServiceFactory
import entropy.dep
import entropy.tools
__all__ = ["CachedRepository", "ClientEntropyRepositoryPlugin",
"InstalledPackagesRepository", "AvailablePackagesRepository",
"GenericRepository"]
class ClientEntropyRepositoryPlugin(EntropyRepositoryPlugin):
def __init__(self, client_interface, metadata = None):
"""
Entropy client-side repository EntropyRepository Plugin class.
This class will be instantiated and automatically added to
EntropyRepository instances generated by Entropy Client.
@param client_interface: Entropy Client interface instance
@type client_interface: entropy.client.interfaces.Client class
@param metadata: any dict form metadata map (key => value)
@type metadata: dict
"""
EntropyRepositoryPlugin.__init__(self)
self._client = client_interface
if metadata is None:
self._metadata = {}
else:
self._metadata = metadata
def get_id(self):
return "__client__"
def get_metadata(self):
return self._metadata
def add_plugin_hook(self, entropy_repository_instance):
const_debug_write(__name__,
"ClientEntropyRepositoryPlugin: calling add_plugin_hook => %s" % (
self,)
)
out_intf = self._metadata.get('output_interface')
if out_intf is not None:
entropy_repository_instance.output = out_intf.output
entropy_repository_instance.ask_question = out_intf.ask_question
return 0
class CachedRepository(EntropyRepository):
"""
This kind of repository cannot have close() called directly, without
a valid token passed. This is because the class object is cached somewhere
and calling close() would turn into a software bug.
"""
def setCloseToken(self, token):
"""
Set a token that can be used to validate close() calls. Calling
close() on these repos is prohibited and considered a software bug.
Only Entropy Client should be able to close them.
"""
self._close_token = token
def close(self, safe=False, _token = None):
"""
Reimplemented from EntropyRepository
"""
close_token = getattr(self, "_close_token", None)
if close_token is not None:
if (_token is None) or (_token != close_token):
raise PermissionDenied(
"cannot close this repository directly. Software bug!")
return EntropyRepository.close(self, safe=safe)
class InstalledPackagesRepository(CachedRepository):
"""
This class represents the installed packages repository and is a direct
subclass of EntropyRepository.
"""
def __init__(self, *args, **kwargs):
EntropyRepository.__init__(self, *args, **kwargs)
# ensure proper repository file permissions
if entropy.tools.is_root() and os.path.isfile(self._db):
const_setup_file(self._db, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
def handlePackage(self, pkg_data, forcedRevision = -1,
formattedContent = False):
"""
Reimplemented from EntropyRepository.
"""
removelist = self.getPackagesToRemove(
pkg_data['name'], pkg_data['category'],
pkg_data['slot'], pkg_data['injected']
)
for r_package_id in removelist:
self.removePackage(r_package_id)
return self.addPackage(pkg_data, revision = forcedRevision,
formatted_content = formattedContent)
class AvailablePackagesRepositoryUpdater(object):
"""
AvailablePackagesRepository update logic class.
The required logic for updating a repository is stored here.
"""
WEBSERV_CACHE_ID = 'webserv_repo/segment_'
def __init__(self, entropy_client, repository_id, force, gpg):
self.__force = force
self.__big_sock_timeout = 20
self._repository_id = repository_id
self._cacher = EntropyCacher()
self._last_rev = -1
self._entropy = entropy_client
self._settings = SystemSettings()
self._gpg_feature = gpg
self._supported_apis = etpConst['supportedapis']
self._supported_download_items = (
"db", "dbck", "dblight", "ck", "cklight", "compck",
"lock", "dbdump", "dbdumplight", "dbdumplightck", "dbdumpck",
"meta_file", "meta_file_gpg", "notice_board"
)
self._developer_repo = \
self._settings['repositories']['developer_repo']
self._differential_update = \
self._settings['repositories']['differential_update']
if self._developer_repo:
const_debug_write(__name__,
"__init__: developer repo mode enabled")
self.__webservices = None
self.__webservice = None
self._repo_eapi = self.__get_repo_eapi()
avail_data = self._settings['repositories']['available']
if self._repository_id not in avail_data:
raise KeyError("Repository not available")
# default to the officially listed Repository URL
# in case of connection issues, we will fallback
# to packages mirrors.
# This is particularily for improving fault-tolerance
# and for Chinese people.
repo_data = avail_data[self._repository_id]
self._repo_uri = repo_data['database']
@property
def _webservices(self):
if self.__webservices is None:
if hasattr(self._entropy, "RepositoryWebServices"):
self.__webservices = self._entropy.RepositoryWebServices()
else:
# in case self._entropy is a simple TextInterface()
# like how it's called in remote_revision().
self.__webservices = RepositoryWebServiceFactory(self._entropy)
# cross fingers!
return self.__webservices
@property
def _webservice(self):
if self.__webservice is None:
self.__webservice = self._webservices.new(self._repository_id)
self.__webservice._set_timeout(self.__big_sock_timeout)
return self.__webservice
def __get_webserv_repository_metadata(self):
try:
data = self._webservice.get_repository_metadata()
except WebService.WebServiceException as err:
const_debug_write(__name__,
"__get_webserv_repository_metadata: error: %s" % (err,))
data = {}
return data
def __get_webserv_repository_revision(self):
try:
revision = self._webservice.get_revision()
except WebService.WebServiceException as err:
const_debug_write(__name__,
"__get_webserv_repository_revision: error: %s" % (err,))
revision = None
return revision
def __check_webserv_availability(self):
try:
webserv = self._webservices.new(self._repository_id)
except WebService.UnsupportedService:
return False
try:
available = webserv.service_available(cache = False)
except WebService.WebServiceException:
available = False
return available
def __get_webserv_local_database(self):
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
dbfile = os.path.join(repo_data['dbpath'],
etpConst['etpdatabasefile'])
dbconn = None
try:
dbconn = self._entropy.open_generic_repository(dbfile,
xcache = False, indexing_override = False)
dbconn.validate()
except (OperationalError, IntegrityError, SystemDatabaseError,
IOError, OSError,):
dbconn = None
return dbconn
def __get_webserv_database_differences(self, webserv, package_ids):
try:
remote_package_ids = webserv.get_package_ids()
except WebService.WebServiceException as err:
const_debug_write(__name__,
"__get_webserv_database_differences: error: %s" % (err,))
return None, None
added = [x for x in remote_package_ids if x not in package_ids]
removed = [x for x in package_ids if x not in remote_package_ids]
# sort, to increase determinism
added.sort()
return added, removed
def __eapi1_eapi2_databases_alignment(self, dbfile, dbfile_old):
dbconn = self._entropy.open_generic_repository(dbfile,
xcache = False, indexing_override = False)
old_dbconn = self._entropy.open_generic_repository(dbfile_old,
xcache = False, indexing_override = False)
upd_rc = 0
try:
upd_rc = old_dbconn.alignDatabases(dbconn, output_header = "\t")
except (OperationalError, IntegrityError, DatabaseError,):
pass
old_dbconn.close()
dbconn.close()
if upd_rc > 0:
# -1 means no changes, == force used
# 0 means too much hassle
os.rename(dbfile_old, dbfile)
return upd_rc
def __eapi2_inject_downloaded_dump(self, dumpfile, dbfile, cmethod):
# load the dump into database
mytxt = "%s %s, %s %s" % (
red(_("Injecting downloaded dump")),
darkgreen(etpConst['etpdatabasedumplight']),
red(_("please wait")),
red("..."),
)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = "\t"
)
dbconn = self._entropy.open_generic_repository(dbfile,
xcache = False, indexing_override = False)
rc = dbconn.importRepository(dumpfile, dbfile)
dbconn.close()
return rc
def __get_repo_eapi(self):
eapi_env = os.getenv("FORCE_EAPI")
sqlite3_access = os.access("/usr/bin/sqlite3", os.X_OK)
sqlite3_rc = subprocess.call("/usr/bin/sqlite3 -version > /dev/null",
shell = True)
try:
eapi_env_clear = int(eapi_env)
if eapi_env_clear not in self._supported_apis:
raise ValueError()
except (ValueError, TypeError,):
eapi_env_clear = None
repo_eapi = 2
eapi_avail = self.__check_webserv_availability()
if eapi_avail:
repo_eapi = 3
else:
if not sqlite3_access or entropy.tools.islive():
repo_eapi = 1
elif sqlite3_rc != 0:
repo_eapi = 1
# if differential update is disabled and FORCE_EAPI is not overriding
# we cannot use EAPI=3
if (eapi_env_clear is None) and (not self._differential_update) and \
(repo_eapi == 3):
const_debug_write(__name__,
"__get_repo_eapi: differential update is disabled !")
repo_eapi -= 1
# check EAPI
if eapi_env_clear is not None:
repo_eapi = eapi_env_clear
# FORCE_EAPI is triggered, disable
# developer_repo mode
self._developer_repo = False
const_debug_write(__name__,
"__get_repo_eapi: developer repo mode disabled FORCE_EAPI")
elif repo_eapi > 1 and self._developer_repo:
# enforce EAPI=1
repo_eapi = 1
const_debug_write(__name__,
"__get_repo_eapi: final eapi set to %s" % (repo_eapi,))
return repo_eapi
def __is_repository_updatable(self):
online = self.remote_revision()
if online != -1:
local = AvailablePackagesRepository.revision(
self._repository_id)
if (local == online) and (not self.__force):
return False
return True
def __select_database_mirror(self):
"""
Verify that the database mirror URL is available and working.
If this is not the case, fallback to the first available
and working packages mirror URL.
"""
current_uri = self._repo_uri
revision = self.remote_revision()
if revision != -1:
return
self._entropy.output(
"%s: %s" % (
darkred(_("Attention")),
brown(_("repository is not available at the following URL:")),
),
importance = 1, level = "warning", header = "\t",
)
self._entropy.output(
"%s" % (current_uri,),
importance = 1, level = "warning", header = "\t",
)
self._entropy.output(
"%s" % (
purple(
_("Looking for an alternate route using package mirrors")
),),
importance = 1, level = "warning", header = "\t",
)
repos_data = self._settings['repositories']
avail_data = repos_data['available']
repo_data = avail_data[self._repository_id]
package_uris = repo_data['plain_packages']
for package_uri in package_uris:
self._entropy.output(
"%s:" % (brown(_("Checking repository URL:")),),
importance = 1, level = "warning", header = "\t",
)
self._entropy.output(
"%s" % (package_uri,),
importance = 1, level = "warning", header = "\t",
)
url = entropy.tools.expand_plain_database_mirror(
package_uri, repos_data['product'],
self._repository_id,
repos_data['branch'])
if url == current_uri:
# skip same URL
continue
revision = self.remote_revision(_repo_uri = url)
if revision != -1:
# found
self._repo_uri = url
self._entropy.output(
"%s:" % (brown(_("Found repository at URL:")),),
importance = 1, level = "warning", header = "\t",
)
self._entropy.output(
"%s" % (url,),
importance = 1, level = "warning", header = "\t",
)
return
self._entropy.output(
"%s" % (
purple(
_("Unable to find alternate repository mirrors. Sorry.")
),),
importance = 1, level = "warning", header = "\t",
)
def __show_repository_information(self):
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
self._entropy.output(
bold("%s") % ( repo_data['description'] ),
importance = 2,
level = "info",
header = blue(" # ")
)
mytxt = "%s: %s" % (red(_("Repository URL")),
darkgreen(self._repo_uri),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = blue(" # ")
)
mytxt = "%s: %s" % (red(_("Repository local path")),
darkgreen(repo_data['dbpath']),)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # ")
)
mytxt = "%s: %s" % (red(_("Repository API")),
darkgreen(str(self._repo_eapi)),)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # ")
)
def __handle_repository_lock(self):
# get database lock
unlocked = self._is_repository_unlocked()
if not unlocked:
mytxt = "%s: %s. %s." % (
bold(_("Attention")),
red(_("Repository is being updated")),
red(_("Try again in a few minutes")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = "\t"
)
return True
return False
def __append_gpg_signature_to_path(self, path):
return path + etpConst['etpgpgextension']
def __ensure_repository_path(self):
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
# create dir if it doesn't exist
if not os.path.isdir(repo_data['dbpath']):
os.makedirs(repo_data['dbpath'], 0o755)
try:
items = os.listdir(etpConst['etpdatabaseclientdir'])
except OSError:
items = []
# we cannot operate dir wide (etpdatabaseclientdir) because
# there are lock files in there and we should not touch them
for item in items:
repo_dir_path = os.path.join(
etpConst['etpdatabaseclientdir'],
item)
if not os.path.isdir(repo_dir_path):
continue
const_setup_perms(
repo_dir_path,
etpConst['entropygid'],
f_perms = 0o644)
def __validate_compression_method(self):
repo = self._repository_id
repo_settings = self._settings['repositories']
dbc_format = repo_settings['available'][repo]['dbcformat']
cmethod = etpConst['etpdatabasecompressclasses'].get(dbc_format)
if cmethod is None:
raise AttributeError("Wrong repository compression method")
return cmethod
def __remove_repository_files(self):
sys_set = self._settings
avail_data = sys_set['repositories']['available']
repo_dbpath = avail_data[self._repository_id]['dbpath']
shutil.rmtree(repo_dbpath, True)
def __handle_database_download(self, cmethod):
# starting to download
mytxt = "%s ..." % (red(_("Downloading repository")),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = "\t"
)
downloaded_item = None
down_status = False
sig_status = False
if self._repo_eapi == 2:
down_item = "dbdumplight"
down_status = self._download_item(down_item, cmethod,
disallow_redirect = True)
if down_status:
# get GPG file if available
sig_status = self._download_item(down_item, cmethod,
disallow_redirect = True, get_signature = True)
downloaded_item = down_item
if not down_status: # fallback to old db
self._repo_eapi = 1
down_item = "dblight"
if self._developer_repo:
# if developer repo mode is enabled, fetch full-blown db
down_item = "db"
const_debug_write(__name__,
"__handle_database_download: developer repo mode enabled")
down_status = self._download_item(down_item, cmethod,
disallow_redirect = True)
if down_status:
sig_status = self._download_item(down_item, cmethod,
disallow_redirect = True, get_signature = True)
downloaded_item = down_item
if not down_status:
mytxt = "%s: %s." % (bold(_("Attention")),
red(_("unable to download the repository")),)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = "\t"
)
return down_status, sig_status, downloaded_item
def __handle_database_checksum_download(self, cmethod):
downitem = 'cklight'
if self._developer_repo:
downitem = 'dbck'
if self._repo_eapi == 2: # EAPI = 2
downitem = 'dbdumplightck'
garbage_url, hashfile = self._construct_paths(downitem, cmethod)
mytxt = "%s %s %s" % (
red(_("Downloading checksum")),
darkgreen(os.path.basename(hashfile)),
red("..."),
)
# download checksum
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = "\t"
)
db_down_status = self._download_item(downitem, cmethod,
disallow_redirect = True)
if not db_down_status and (downitem not in ('cklight', 'dbck',)):
# fallback to old method
retryitem = 'cklight'
if self._developer_repo:
retryitem = 'dbck'
db_down_status = self._download_item(retryitem, cmethod,
disallow_redirect = True)
if not db_down_status:
mytxt = "%s %s !" % (
red(_("Cannot fetch checksum")),
red(_("Cannot verify repository integrity")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = "\t"
)
return db_down_status
def __verify_file_checksum(self, file_path, md5_checksum_path):
enc = etpConst['conf_encoding']
with codecs.open(md5_checksum_path, "r", encoding=enc) as ck_f:
md5hash = ck_f.readline().strip()
if not md5hash: # invalid !! => [] would cause IndexError
return False
md5hash = md5hash.split()[0]
return entropy.tools.compare_md5(file_path, md5hash)
def __verify_database_checksum(self, cmethod = None):
sys_settings_repos = self._settings['repositories']
avail_config = sys_settings_repos['available'][self._repository_id]
sep = os.path.sep
if self._repo_eapi == 1:
if self._developer_repo:
remote_gb, dbfile = self._construct_paths('db', cmethod)
remote_gb, md5file = self._construct_paths('dbck', cmethod)
else:
remote_gb, dbfile = self._construct_paths('dblight',
cmethod)
remote_gb, md5file = self._construct_paths('cklight',
cmethod)
elif self._repo_eapi == 2:
remote_gb, dbfile = self._construct_paths('dbdumplight',
cmethod)
remote_gb, md5file = self._construct_paths('dbdumplightck',
cmethod)
else:
raise AttributeError("EAPI must be = 1 or 2")
if not (os.access(md5file, os.R_OK) and os.path.isfile(md5file)):
return -1
return self.__verify_file_checksum(dbfile, md5file)
def __unpack_downloaded_database(self, down_item, cmethod):
rc = 0
path = None
sys_set_repos = self._settings['repositories']['available']
repo_data = sys_set_repos[self._repository_id]
garbage, myfile = self._construct_paths(down_item, cmethod)
if self._repo_eapi in (1, 2,):
try:
myfunc = getattr(entropy.tools, cmethod[1])
path = myfunc(myfile)
# rename path correctly
if self._repo_eapi == 1:
new_path = os.path.join(os.path.dirname(path),
etpConst['etpdatabasefile'])
os.rename(path, new_path)
path = new_path
except (OSError, EOFError):
rc = 1
else:
mytxt = "invalid EAPI must be = 1 or 2"
raise AttributeError(mytxt)
if rc == 0:
const_setup_file(path, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
return rc
def __handle_downloaded_database_unpack(self, cmethod):
file_to_unpack = etpConst['etpdatabasedump']
if self._repo_eapi == 1:
file_to_unpack = etpConst['etpdatabasefile']
elif self._repo_eapi == 2:
file_to_unpack = etpConst['etpdatabasedumplight']
mytxt = "%s %s %s" % (red(_("Unpacking database to")),
darkgreen(file_to_unpack), red("..."),)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = "\t"
)
myitem = 'dblight'
if self._repo_eapi == 2:
myitem = 'dbdumplight'
elif self._developer_repo:
myitem = 'db'
myrc = self.__unpack_downloaded_database(myitem, cmethod)
if myrc != 0:
mytxt = "%s %s !" % (red(_("Cannot unpack compressed package")),
red(_("Skipping repository")),)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = "\t"
)
return False, myitem
return True, myitem
def __update_repository_revision(self):
cur_rev = AvailablePackagesRepository.revision(self._repository_id)
repo_data = self._settings['repositories']
db_data = repo_data['available'][self._repository_id]
db_data['dbrevision'] = "0"
if cur_rev != -1:
db_data['dbrevision'] = str(cur_rev)
# update repository revision file
# self.remote_revision() output must be
# written into packages.db.revision for consistency
# otherwise WebService sync when WebService is on a separate
# server (and uses rsync) doesn't work at its best
# self._last_revs
rev_file = os.path.join(db_data['dbpath'],
etpConst['etpdatabaserevisionfile'])
enc = etpConst['conf_encoding']
with codecs.open(rev_file, "w", encoding=enc) as rev_f:
# safe anyway
rev_f.write(str(self._last_rev) + "\n")
rev_f.flush()
def __validate_database(self):
def tell_error(err):
mytxt = "%s: %s" % (darkred(_("Repository is invalid")),
repr(err),)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = "\t"
)
try:
dbconn = self._entropy.open_repository(self._repository_id)
except RepositoryError as err:
tell_error(err)
return False
try:
dbconn.validate()
except SystemDatabaseError as err:
tell_error(err)
return False
return True
def __database_indexing(self):
# renice a bit, to avoid eating resources
old_prio = const_set_nice_level(15)
mytxt = red("%s ...") % (_("Indexing Repository metadata"),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = "\t"
)
dbconn = self._entropy.open_repository(self._repository_id)
dbconn.createAllIndexes()
dbconn.commit(force = True)
if self._entropy.installed_repository() is not None:
try: # client db can be absent
self._entropy.installed_repository().createAllIndexes()
except (DatabaseError, OperationalError, IntegrityError,):
pass
const_set_nice_level(old_prio)
def _construct_paths(self, item, cmethod, get_signature = False):
if item not in self._supported_download_items:
raise AttributeError("Invalid item: %s" % (item,))
items_needing_cmethod = (
"db", "dbck", "dblight", "cklight", "dbdump", "dbdumpck",
"dbdumplight", "dbdumplightck", "compck",
)
if (item in items_needing_cmethod) and (cmethod is None):
mytxt = "For %s, cmethod can't be None" % (item,)
raise AttributeError(mytxt)
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
repo_db = self._repo_uri
repo_dbpath = repo_data['dbpath']
ec_hash = etpConst['etpdatabasehashfile']
repo_lock_file = etpConst['etpdatabasedownloadlockfile']
notice_board_filename = os.path.basename(repo_data['notice_board'])
meta_file = etpConst['etpdatabasemetafilesfile']
meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
etpConst['etpgpgextension']
md5_ext = etpConst['packagesmd5fileext']
ec_cm2 = None
ec_cm3 = None
ec_cm4 = None
ec_cm5 = None
ec_cm6 = None
ec_cm7 = None
ec_cm8 = None
ec_cm9 = None
if cmethod is not None:
ec_cm2 = etpConst[cmethod[2]]
ec_cm3 = etpConst[cmethod[3]]
ec_cm4 = etpConst[cmethod[4]]
ec_cm5 = etpConst[cmethod[5]]
ec_cm6 = etpConst[cmethod[6]]
ec_cm7 = etpConst[cmethod[7]]
ec_cm8 = etpConst[cmethod[8]]
ec_cm9 = etpConst[cmethod[9]]
mymap = {
'db': (
"%s/%s" % (repo_db, ec_cm2,),
"%s/%s" % (repo_dbpath, ec_cm2,),
),
'dbck': (
"%s/%s" % (repo_db, ec_cm9,),
"%s/%s" % (repo_dbpath, ec_cm9,),
),
'dblight': (
"%s/%s" % (repo_db, ec_cm7,),
"%s/%s" % (repo_dbpath, ec_cm7,),
),
'dbdump': (
"%s/%s" % (repo_db, ec_cm3,),
"%s/%s" % (repo_dbpath, ec_cm3,),
),
'dbdumplight': (
"%s/%s" % (repo_db, ec_cm5,),
"%s/%s" % (repo_dbpath, ec_cm5,),
),
'ck': (
"%s/%s" % (repo_db, ec_hash,),
"%s/%s" % (repo_dbpath, ec_hash,),
),
'cklight': (
"%s/%s" % (repo_db, ec_cm8,),
"%s/%s" % (repo_dbpath, ec_cm8,),
),
'compck': (
"%s/%s%s" % (repo_db, ec_cm2, md5_ext,),
"%s/%s%s" % (repo_dbpath, ec_cm2, md5_ext,),
),
'dbdumpck': (
"%s/%s" % (repo_db, ec_cm4,),
"%s/%s" % (repo_dbpath, ec_cm4,),
),
'dbdumplightck': (
"%s/%s" % (repo_db, ec_cm6,),
"%s/%s" % (repo_dbpath, ec_cm6,),
),
'lock': (
"%s/%s" % (repo_db, repo_lock_file,),
"%s/%s" % (repo_dbpath, repo_lock_file,),
),
'notice_board': (
repo_data['notice_board'],
"%s/%s" % (repo_dbpath, notice_board_filename,),
),
'meta_file': (
"%s/%s" % (repo_db, meta_file,),
"%s/%s" % (repo_dbpath, meta_file,),
),
'meta_file_gpg': (
"%s/%s" % (repo_db, meta_file_gpg,),
"%s/%s" % (repo_dbpath, meta_file_gpg,),
),
}
url, path = mymap.get(item)
if get_signature:
url = self.__append_gpg_signature_to_path(url)
path = self.__append_gpg_signature_to_path(path)
return url, path
def _download_item(self, item, cmethod = None,
disallow_redirect = True, get_signature = False):
url, filepath = self._construct_paths(item, cmethod,
get_signature = get_signature)
# See bug #3495, download the file to
# a temporary location and then move it
# if we are successful
temp_filepath = filepath + ".edownload"
# to avoid having permissions issues
# it's better to remove the file before,
# otherwise new permissions won't be written
if os.path.isfile(temp_filepath):
os.remove(temp_filepath)
filepath_dir = os.path.dirname(temp_filepath)
if not os.path.isdir(filepath_dir) and not \
os.path.lexists(filepath_dir):
os.makedirs(filepath_dir, 0o755)
const_setup_perms(filepath_dir, etpConst['entropygid'],
f_perms = 0o644)
try:
fetch_errors = (
UrlFetcher.GENERIC_FETCH_WARN,
UrlFetcher.TIMEOUT_FETCH_ERROR,
UrlFetcher.GENERIC_FETCH_ERROR)
fetcher = self._entropy._url_fetcher(
url,
temp_filepath,
resume = False,
disallow_redirect = disallow_redirect
)
rc = fetcher.download()
if rc in fetch_errors:
return False
try:
os.rename(temp_filepath, filepath)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
return False # not downloaded?
const_setup_file(filepath, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
return True
finally:
# cleanup temp file
try:
os.remove(temp_filepath)
except (OSError, IOError) as err:
if err.errno != errno.ENOENT:
raise
def _is_repository_unlocked(self):
"""
Returns whether the repository is remotely locked or not.
@return: repository being remotely locked
@rtype:bool
"""
rc = self._download_item("lock", disallow_redirect = True)
if rc: # cannot download database
return False
return True
def _standard_items_download(self):
repos_data = self._settings['repositories']
repo_data = repos_data['available'][self._repository_id]
notice_board = os.path.basename(repo_data['local_notice_board'])
db_meta_file = etpConst['etpdatabasemetafilesfile']
db_meta_file_gpg = etpConst['etpdatabasemetafilesfile'] + \
etpConst['etpgpgextension']
objects_to_unpack = ("meta_file",)
download_items = [
(
"meta_file",
db_meta_file,
False,
"%s %s %s" % (
red(_("Downloading repository metafile")),
darkgreen(db_meta_file),
red("..."),
)
),
(
"meta_file_gpg",
db_meta_file_gpg,
True,
"%s %s %s" % (
red(_("Downloading GPG signature of repository metafile")),
darkgreen(db_meta_file_gpg),
red("..."),
)
),
(
"notice_board",
notice_board,
True,
"%s %s %s" % (
red(_("Downloading Notice Board")),
darkgreen(notice_board),
red("..."),
)
),
]
def my_show_info(txt):
self._entropy.output(
txt,
importance = 0,
level = "info",
header = "\t",
back = True
)
def my_show_down_status(message, mytype):
self._entropy.output(
message,
importance = 0,
level = mytype,
header = "\t"
)
def my_show_file_unpack(fp):
self._entropy.output(
"%s: %s" % (darkgreen(_("unpacked meta file")), brown(fp),),
header = blue("\t << ")
)
def my_show_file_rm(fp):
self._entropy.output(
"%s: %s" % (darkgreen(_("removed meta file")), purple(fp),),
header = blue("\t << ")
)
downloaded_files = []
for item, myfile, ignorable, mytxt in download_items:
my_show_info(mytxt)
mystatus = self._download_item(item, disallow_redirect = True)
mytype = 'info'
myurl, mypath = self._construct_paths(item, None)
# download failed, is it critical?
if not mystatus:
if ignorable:
message = "%s: %s." % (blue(myfile),
red(_("not available, it's ok")))
else:
mytype = 'warning'
message = "%s: %s." % (blue(myfile),
darkred(_("not available, not very ok!")))
my_show_down_status(message, mytype)
# remove garbage
if os.path.isfile(mypath):
try:
os.remove(mypath)
except OSError:
continue
continue
message = "%s: %s." % (blue(myfile),
darkgreen(_("available, w00t!")))
my_show_down_status(message, mytype)
downloaded_files.append(mypath)
if item not in objects_to_unpack:
continue
if not (os.path.isfile(mypath) and os.access(mypath, os.R_OK)):
continue
tmpdir = tempfile.mkdtemp()
repo_dir = repo_data['dbpath']
enc = etpConst['conf_encoding']
try:
done = entropy.tools.universal_uncompress(mypath, tmpdir,
catch_empty = True)
if not done:
mytype = 'warning'
message = "%s: %s." % (blue(myfile),
darkred(_("cannot be unpacked, not very ok!")))
my_show_down_status(message, mytype)
continue
myfiles_to_move = set(os.listdir(tmpdir))
# exclude files not available by default
files_not_found_file = etpConst['etpdatabasemetafilesnotfound']
if files_not_found_file in myfiles_to_move:
myfiles_to_move.remove(files_not_found_file)
fnf_path = os.path.join(tmpdir, files_not_found_file)
if os.path.isfile(fnf_path) and \
os.access(fnf_path, os.R_OK):
with codecs.open(fnf_path, "r", encoding=enc) as f:
f_nf = [x.strip() for x in f.readlines()]
for myfile in f_nf:
myfile = os.path.basename(myfile) # avoid lamerz
myfpath = os.path.join(repo_dir, myfile)
if os.path.isfile(myfpath) and \
os.access(myfpath, os.W_OK):
try:
os.remove(myfpath)
my_show_file_rm(myfile)
except OSError:
continue
for myfile in sorted(myfiles_to_move):
from_mypath = os.path.join(tmpdir, myfile)
to_mypath = os.path.join(repo_dir, myfile)
try:
os.rename(from_mypath, to_mypath)
my_show_file_unpack(myfile)
except OSError:
# try non atomic way
try:
shutil.copy2(from_mypath, to_mypath)
my_show_file_unpack(myfile)
os.remove(from_mypath)
except (shutil.Error, IOError, OSError,):
continue
continue
const_setup_file(to_mypath, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
finally:
shutil.rmtree(tmpdir, True)
repo_r = AvailablePackagesRepository.revision(self._repository_id)
mytxt = "%s: %s" % (
red(_("Repository revision")),
bold(str(repo_r)),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = "\t"
)
return downloaded_files
def _check_downloaded_database(self, cmethod):
dbitem = 'dblight'
if self._repo_eapi == 2:
dbitem = 'dbdumplight'
elif self._developer_repo:
dbitem = 'db'
garbage, dbfilename = self._construct_paths(dbitem, cmethod)
# verify checksum
mytxt = "%s %s %s" % (
red(_("Checking downloaded repository")),
darkgreen(os.path.basename(dbfilename)),
red("..."),
)
self._entropy.output(
mytxt,
importance = 0,
back = True,
level = "info",
header = "\t"
)
db_status = self.__verify_database_checksum(cmethod)
if db_status == -1:
mytxt = "%s. %s !" % (
red(_("Cannot open digest")),
red(_("Cannot verify repository integrity")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "warning",
header = "\t"
)
elif db_status:
mytxt = "%s: %s" % (
red(_("Downloaded repository status")),
bold(_("OK")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = "\t"
)
else:
mytxt = "%s: %s" % (
red(_("Downloaded repository status")),
darkred(_("ERROR")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = "\t"
)
mytxt = "%s. %s" % (
red(_("An error occured while checking repository integrity")),
red(_("Giving up")),
)
self._entropy.output(
mytxt,
importance = 1,
level = "error",
header = "\t"
)
return 1
return 0
def _install_gpg_key_if_available(self):
my_repos = self._settings['repositories']
avail_data = my_repos['available']
repo_data = avail_data[self._repository_id]
gpg_path = repo_data['gpg_pubkey']
if not (os.path.isfile(gpg_path) and os.access(gpg_path, os.R_OK)):
return False # gpg key not available
def do_warn_user(fingerprint):
mytxt = purple(_("Make sure to verify the imported key and set an appropriate trust level"))
self._entropy.output(
mytxt + ":",
level = "warning",
header = "\t"
)
mytxt = brown("gpg --homedir '%s' --edit-key '%s'" % (
etpConst['etpclientgpgdir'], fingerprint,)
)
self._entropy.output(
"$ " + mytxt,
level = "warning",
header = "\t"
)
try:
repo_sec = self._entropy.RepositorySecurity()
except RepositorySecurity.GPGError:
mytxt = "%s," % (
purple(_("This repository suports GPG-signed packages")),
)
self._entropy.output(
mytxt,
level = "warning",
header = "\t"
)
mytxt = purple(_("you may want to install GnuPG to take advantage of this feature"))
self._entropy.output(
mytxt,
level = "warning",
header = "\t"
)
return False # GPG not available
pk_expired = False
try:
pk_avail = repo_sec.is_pubkey_available(self._repository_id)
except repo_sec.KeyExpired:
pk_avail = False
pk_expired = True
if pk_avail:
tmp_dir = tempfile.mkdtemp()
repo_tmp_sec = self._entropy.RepositorySecurity(
keystore_dir = tmp_dir)
# try to install and get fingerprint
try:
downloaded_key_fp = repo_tmp_sec.install_key(
self._repository_id, gpg_path)
except RepositorySecurity.GPGError:
downloaded_key_fp = None
fingerprint = repo_sec.get_key_metadata(
self._repository_id)['fingerprint']
shutil.rmtree(tmp_dir, True)
if downloaded_key_fp != fingerprint and \
(downloaded_key_fp is not None):
mytxt = "%s: %s !!!" % (
purple(_("GPG key changed for")),
bold(self._repository_id),
)
self._entropy.output(
mytxt,
level = "warning",
header = "\t"
)
mytxt = "[%s => %s]" % (
darkgreen(fingerprint),
purple(downloaded_key_fp),
)
self._entropy.output(
mytxt,
level = "warning",
header = "\t"
)
do_warn_user(downloaded_key_fp)
else:
mytxt = "%s: %s" % (
purple(_("GPG key already installed for")),
bold(self._repository_id),
)
self._entropy.output(
mytxt,
level = "info",
header = "\t"
)
do_warn_user(fingerprint)
return True # already installed
elif pk_expired:
mytxt = "%s: %s" % (
purple(_("GPG key EXPIRED for repository")),
bold(self._repository_id),
)
self._entropy.output(
mytxt,
level = "warning",
header = "\t"
)
# actually install
mytxt = "%s: %s" % (
purple(_("Installing GPG key for repository")),
brown(self._repository_id),
)
self._entropy.output(
mytxt,
level = "info",
header = "\t",
back = True
)
try_ignore = False
while True:
try:
fingerprint = repo_sec.install_key(self._repository_id,
gpg_path)
except RepositorySecurity.NothingImported as err:
if try_ignore:
mytxt = "%s: %s" % (
darkred(_("Error during GPG key installation")),
err,
)
self._entropy.output(
mytxt,
level = "error",
header = "\t"
)
return False
self._entropy.output(
purple(_("GPG key seems already installed but not properly recorded, resetting")),
level = "warning",
header = "\t"
)
target_fingerprint = repo_sec.get_key_fingerprint(gpg_path)
if target_fingerprint is not None:
# kill it, this is usually caused by shadow repos
# like sabayon-weekly
dead_repository_ids = set()
for _repository_id, key_meta in repo_sec.get_keys().items():
if key_meta['fingerprint'] == target_fingerprint:
dead_repository_ids.add(_repository_id)
for _repository_id in dead_repository_ids:
try:
repo_sec.delete_pubkey(_repository_id)
except KeyError:
# wtf, fault tolerance
pass
try_ignore = True
continue
except RepositorySecurity.GPGError as err:
mytxt = "%s: %s" % (
darkred(_("Error during GPG key installation")),
err,
)
self._entropy.output(
mytxt,
level = "error",
header = "\t"
)
return False
break
mytxt = "%s: %s" % (
purple(_("Successfully installed GPG key for repository")),
brown(self._repository_id),
)
self._entropy.output(
mytxt,
level = "info",
header = "\t"
)
mytxt = "%s: %s" % (
darkgreen(_("Fingerprint")),
bold(fingerprint),
)
self._entropy.output(
mytxt,
level = "info",
header = "\t"
)
do_warn_user(fingerprint)
return True
def _gpg_verify_downloaded_files(self, downloaded_files):
try:
repo_sec = self._entropy.RepositorySecurity()
except RepositorySecurity.GPGServiceNotAvailable:
# wtf! it was available a while ago!
return 0 # GPG not available
gpg_sign_ext = etpConst['etpgpgextension']
sign_files = [x for x in downloaded_files if \
x.endswith(gpg_sign_ext)]
sign_files = [x for x in sign_files if os.path.isfile(x) and \
os.access(x, os.R_OK)]
to_be_verified = []
for sign_path in sign_files:
target_path = sign_path[:-len(gpg_sign_ext)]
if os.path.isfile(target_path) and \
os.access(target_path, os.R_OK):
to_be_verified.append((target_path, sign_path,))
gpg_rc = 0
for target_path, sign_path in to_be_verified:
file_name = os.path.basename(target_path)
mytxt = "%s: %s ..." % (
darkgreen(_("Verifying GPG signature of")),
brown(file_name),
)
self._entropy.output(
mytxt,
level = "info",
header = blue("\t@@ "),
back = True
)
is_valid, err_msg = repo_sec.verify_file(self._repository_id,
target_path, sign_path)
if is_valid:
mytxt = "%s: %s" % (
darkgreen(_("Verified GPG signature of")),
brown(file_name),
)
self._entropy.output(
mytxt,
level = "info",
header = blue("\t@@ ")
)
else:
mytxt = "%s: %s" % (
darkred(_("Error during GPG verification of")),
file_name,
)
self._entropy.output(
mytxt,
level = "error",
header = "\t%s " % (bold("!!!"),)
)
mytxt = "%s: %s" % (
purple(_("It could mean a potential security risk")),
err_msg,
)
self._entropy.output(
mytxt,
level = "error",
header = "\t%s " % (bold("!!!"),)
)
gpg_rc = 1
return gpg_rc
def _handle_webserv_database_sync(self):
repo_db = None
try:
repo_db = self.__get_webserv_local_database()
if repo_db is None:
raise AttributeError()
return self.__handle_webserv_database_sync(repo_db)
except (DatabaseError, IntegrityError, OperationalError,
AttributeError,):
return False
finally:
if repo_db is not None:
repo_db.commit()
repo_db.close()
return False
def __handle_webserv_database_sync(self, mydbconn):
try:
webserv = self._webservice
except WebService.UnsupportedService as err:
const_debug_write(__name__,
"__handle_webserv_database_sync: error: %s" % (err,))
return False
try:
myidpackages = mydbconn.listAllPackageIds()
except (DatabaseError, IntegrityError, OperationalError,):
return False
added_ids, removed_ids = self.__get_webserv_database_differences(
webserv, myidpackages)
if (None in (added_ids, removed_ids)) or \
(not added_ids and not removed_ids and self.__force):
# nothing to sync, it seems, if force is True, fallback to EAPI2
return False
threshold = 500
# is it worth it?
if len(added_ids) > threshold:
mytxt = "%s: %s (%s: %s/%s)" % (
blue(_("Web Service")),
darkred(_("skipping differential sync")),
brown(_("threshold")),
blue(str(len(added_ids))),
darkred(str(threshold)),
)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # "),
)
return False
count = 0
chunk_size = RepositoryWebService.MAXIMUM_PACKAGE_REQUEST_SIZE
added_segments = []
mytmp = []
for package_id in added_ids:
count += 1
mytmp.append(package_id)
if count % chunk_size == 0:
added_segments.append(mytmp[:])
del mytmp[:]
if mytmp:
added_segments.append(mytmp[:])
def _do_fetch(fetch_sts_map, segment, count, maxcount):
try:
try:
pkg_meta = webserv.get_packages_metadata(segment)
except WebService.WebServiceException as err:
const_debug_write(__name__,
"__handle_webserv_database_sync: error: %s" % (err,))
mytxt = "%s: %s" % (
blue(_("Web Service communication error")),
err,
)
self._entropy.output(
mytxt, importance = 1, level = "info",
header = "\t", count = (count, maxcount,)
)
fetch_sts_map['error'] = True
return
except KeyboardInterrupt:
const_debug_write(__name__,
"__handle_webserv_database_sync: keyboard interrupt")
fetch_sts_map['error'] = True
return
if not pkg_meta:
const_debug_write(__name__,
"__handle_webserv_database_sync: empty data: %s" % (
pkg_meta,))
self._entropy.output(
_("Web Service data error"), importance = 1,
level = "info", header = "\t",
count = (count, maxcount,)
)
fetch_sts_map['error'] = True
return
try:
for package_id, pkg_data in pkg_meta.items():
dumpobj(
"%s%s" % (self.WEBSERV_CACHE_ID, package_id,),
pkg_data,
ignore_exceptions = False
)
except (IOError, EOFError, OSError,) as e:
mytxt = "%s: %s: %s." % (
blue(_("Local status")),
darkred("Error storing data"),
e,
)
self._entropy.output(
mytxt, importance = 1, level = "info",
header = "\t", count = (count, maxcount,)
)
fetch_sts_map['error'] = True
return
return
finally:
fetch_sts_map['sem'].release()
# do not exagerate or you're going to need a way to block
# further requests as long as some threads are still running
# to avoid timeout errors
max_threads = 4
fetch_sts_map = {
'sem': threading.Semaphore(max_threads),
'error': False,
}
# fetch and store
count = 0
maxcount = len(added_segments)
product = self._settings['repositories']['product']
segment = None
threads = []
for segment in added_segments:
count += 1
mytxt = "%s %s" % (blue(_("Fetching segments")), "...",)
self._entropy.output(
mytxt, importance = 0, level = "info",
header = "\t", back = True, count = (count, maxcount,)
)
fetch_sts_map['sem'].acquire()
if len(threads) >= max_threads:
const_debug_write(__name__,
purple("joining all the parallel threads"))
# give them the chance to complete
# since long delays on socket could cause timeouts
for th in threads:
th.join()
const_debug_write(__name__, purple("parallel threads joined"))
del threads[:]
if fetch_sts_map['error']:
return None
th = ParallelTask(_do_fetch, fetch_sts_map, segment, count,
maxcount)
th.daemon = True
th.start()
threads.append(th)
for th in threads:
th.join()
if fetch_sts_map['error']:
return None
del added_segments
# get repository metadata
repo_metadata = self.__get_webserv_repository_metadata()
# this gives us the "checksum" data too
if not repo_metadata:
mytxt = "%s: %s" % (
blue(_("Web Service status")),
darkred(_("cannot fetch repository metadata")),
)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # "),
)
return None
# update treeupdates
try:
mydbconn.setRepositoryUpdatesDigest(self._repository_id,
repo_metadata['treeupdates_digest'])
mydbconn.bumpTreeUpdatesActions(
repo_metadata['treeupdates_actions'])
except (Error,):
mytxt = "%s: %s" % (
blue(_("Web Service status")),
darkred(_("cannot update treeupdates data")),
)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # "),
)
return None
# update package sets
try:
mydbconn.clearPackageSets()
mydbconn.insertPackageSets(repo_metadata['sets'])
except (Error,):
mytxt = "%s: %s" % (
blue(_("Web Service status")),
darkred(_("cannot update package sets data")),
)
self._entropy.output(
mytxt,
importance = 0,
level = "info",
header = blue(" # "),
)
return None
# now that we have all stored, add
for idpackage in added_ids:
mydata = self._cacher.pop("%s%s" % (self.WEBSERV_CACHE_ID,
idpackage,))
if mydata is None:
mytxt = "%s: %s" % (
blue(_("Fetch error on segment while adding")),
darkred(str(segment)),
)
self._entropy.output(
mytxt, importance = 1, level = "warning",
header = " "
)
return False
mytxt = "%s %s" % (
darkgreen("++"),
teal(mydata['atom']),
)
self._entropy.output(
mytxt, importance = 0, level = "info",
header = " ", back = (not etpUi['verbose'])
)
try:
mydbconn.addPackage(
mydata, revision = mydata['revision'],
package_id = idpackage,
formatted_content = True
)
except (Error,) as err:
if etpUi['debug']:
entropy.tools.print_traceback()
self._entropy.output("%s: %s" % (
blue(_("repository error while adding packages")),
err,),
importance = 1, level = "warning",
header = " "
)
return False
# now remove
# preload atoms names to improve speed during removePackage
atoms_map = dict((x, mydbconn.retrieveAtom(x),) for x in removed_ids)
for idpackage in removed_ids:
myatom = atoms_map.get(idpackage)
mytxt = "%s %s" % (
darkred("--"),
purple(str(myatom)),)
self._entropy.output(
mytxt, importance = 0, level = "info",
header = " ", back = (not etpUi['verbose'])
)
try:
mydbconn.removePackage(idpackage)
except (Error,):
self._entropy.output(
blue(_("repository error while removing packages")),
importance = 1, level = "warning",
header = " "
)
return False
mydbconn.commit()
mydbconn.clearCache()
# now verify if both checksums match
result = False
mychecksum = mydbconn.checksum(do_order = True,
strict = False, strings = True, include_signatures = True)
if repo_metadata['checksum'] == mychecksum:
result = True
else:
self._entropy.output(
blue(_("Repository checksum doesn't match remote.")),
importance = 0, level = "info", header = "\t",
)
mytxt = "%s: %s" % (_('local'), mychecksum,)
self._entropy.output(
mytxt, importance = 0,
level = "info", header = "\t",
)
mytxt = "%s: %s" % (_('remote'), repo_metadata['checksum'],)
self._entropy.output(
mytxt, importance = 0,
level = "info", header = "\t",
)
return result
def remote_revision(self, _repo_uri = None):
if self._repo_eapi == 3 and _repo_uri is None:
# ask WebService then
revision = self.__get_webserv_repository_revision()
if revision is not None:
try:
revision = int(revision)
except ValueError:
revision = None
if revision is not None:
self._last_rev = revision
return revision
# otherwise, fallback to previous EAPI
self._repo_eapi -= 1
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
if _repo_uri is None:
_repo_uri = self._repo_uri
sep = const_convert_to_unicode("/")
url = _repo_uri + sep + etpConst['etpdatabaserevisionfile']
status = entropy.tools.get_remote_data(url,
timeout = self.__big_sock_timeout)
if status:
status = status[0].strip()
try:
status = int(status)
except ValueError:
status = -1
else:
status = -1
self._last_rev = status
return status
def update(self):
# disallow unprivileged update
if not entropy.tools.is_root():
raise PermissionDenied(
"cannot update repository as unprivileged user")
self.__show_repository_information()
# China loves us!
self.__select_database_mirror()
# this calls writes self._last_rev which is used to write back
# updated repository revision, do not remove!
updatable = self.__is_repository_updatable()
if not self.__force:
if not updatable:
mytxt = "%s: %s." % (bold(_("Attention")),
red(_("repository is already up to date")),)
self._entropy.output(
mytxt,
importance = 1,
level = "info",
header = "\t"
)
return EntropyRepositoryBase.REPOSITORY_ALREADY_UPTODATE
locked = self.__handle_repository_lock()
if locked:
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
# clear database interface cache belonging to this repository
self.__ensure_repository_path()
# dealing with EAPI
# setting some vars
db_checksum_down_status = False
do_db_update_transfer = False
rc = 0
my_repos = self._settings['repositories']
avail_data = my_repos['available']
repo_data = avail_data[self._repository_id]
# some variables
dumpfile = os.path.join(repo_data['dbpath'],
etpConst['etpdatabasedumplight'])
dbfile = os.path.join(repo_data['dbpath'],
etpConst['etpdatabasefile'])
dbfile_old = dbfile+".sync"
cmethod = self.__validate_compression_method()
while True:
downloaded_db_item = None
sig_down_status = False
db_checksum_down_status = False
if self._repo_eapi < 3:
down_status, sig_down_status, downloaded_db_item = \
self.__handle_database_download(cmethod)
if not down_status:
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
db_checksum_down_status = \
self.__handle_database_checksum_download(cmethod)
break
elif self._repo_eapi == 3 and not \
(os.path.isfile(dbfile) and os.access(dbfile, os.W_OK)):
do_db_update_transfer = None
self._repo_eapi -= 1
continue
elif self._repo_eapi == 3:
status = False
try:
status = self._handle_webserv_database_sync()
except:
# avoid broken entries, deal with every exception
self.__remove_repository_files()
raise
if status is None: # remote db not available anymore ?
time.sleep(5)
locked = self.__handle_repository_lock()
if locked:
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
do_db_update_transfer = None
self._repo_eapi -= 1
continue
elif not status: # (status == False)
# set to none and completely skip database alignment
do_db_update_transfer = None
self._repo_eapi -= 1
continue
break
downloaded_files = self._standard_items_download()
# also add db file to downloaded item
# and md5 check repository
if downloaded_db_item is not None:
durl, dpath = self._construct_paths(downloaded_db_item,
cmethod)
downloaded_files.append(dpath)
if sig_down_status:
d_sig_path = self.__append_gpg_signature_to_path(dpath)
downloaded_files.append(d_sig_path)
# 1. we're always in EAPI1 or 2 here
# 2. new policy, always deny repository if
# its database checksum cannot be fetched
if not db_checksum_down_status:
# delete all
self.__remove_repository_files()
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
rc = self._check_downloaded_database(cmethod)
if rc != 0:
# delete all
self.__remove_repository_files()
return EntropyRepositoryBase.REPOSITORY_CHECKSUM_ERROR
# GPG pubkey install hook
if self._gpg_feature:
gpg_available = self._install_gpg_key_if_available()
if gpg_available:
gpg_rc = self._gpg_verify_downloaded_files(downloaded_files)
# Now we can unpack
files_to_remove = []
if self._repo_eapi in (1, 2,):
# if do_db_update_transfer == False and not None
if (do_db_update_transfer is not None) and not \
do_db_update_transfer:
if os.access(dbfile, os.R_OK | os.W_OK) and \
os.path.isfile(dbfile):
try:
os.rename(dbfile, dbfile_old)
do_db_update_transfer = True
except OSError:
do_db_update_transfer = False
unpack_status, unpacked_item = \
self.__handle_downloaded_database_unpack(cmethod)
if not unpack_status:
# delete all
self.__remove_repository_files()
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
unpack_url, unpack_path = self._construct_paths(unpacked_item,
cmethod)
files_to_remove.append(unpack_path)
# re-validate
if not os.path.isfile(dbfile):
do_db_update_transfer = False
elif os.path.isfile(dbfile) and not do_db_update_transfer and \
(self._repo_eapi != 1):
os.remove(dbfile)
if self._repo_eapi == 2:
rc = self.__eapi2_inject_downloaded_dump(dumpfile,
dbfile, cmethod)
if do_db_update_transfer:
self.__eapi1_eapi2_databases_alignment(dbfile, dbfile_old)
if self._repo_eapi == 2:
# remove the dump
files_to_remove.append(dumpfile)
if rc != 0:
# delete all
self.__remove_repository_files()
files_to_remove.append(dbfile_old)
for path in files_to_remove:
try:
os.remove(path)
except OSError:
continue
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
# make sure that all the repository files are stored with proper
# permissions to avoid possible XSS and trust boundary problems.
downloaded_files.append(dbfile)
for downloaded_file in sorted(set(downloaded_files)):
if os.path.isfile(downloaded_file) and \
os.access(downloaded_file, os.W_OK | os.R_OK):
const_setup_file(downloaded_file, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
# remove garbage left around
for path in files_to_remove:
try:
os.remove(path)
except OSError:
continue
valid = self.__validate_database()
if not valid:
# repository failed validation
return EntropyRepositoryBase.REPOSITORY_GENERIC_ERROR
self.__update_repository_revision()
if self._entropy._indexing:
self.__database_indexing()
try:
spm_class = self._entropy.Spm_class()
spm_class.entropy_client_post_repository_update_hook(
self._entropy, self._repository_id)
except Exception as err:
entropy.tools.print_traceback()
mytxt = "%s: %s" % (
blue(_("Configuration files update error, not critical, continuing")),
err,
)
self._entropy.output(mytxt, importance = 0,
level = "info", header = blue(" # "),)
# remove garbage
if os.access(dbfile_old, os.R_OK) and os.path.isfile(dbfile_old):
os.remove(dbfile_old)
return EntropyRepositoryBase.REPOSITORY_UPDATED_OK
_CL_PLUGIN_ID = etpConst['system_settings_plugins_ids']['client_plugin']
class MaskableRepository(EntropyRepositoryBase):
"""
Objects inheriting from this class support package masking.
A masked package is a package that is not visible to user and thus not
selectable in dependency calculation and also not directly installable.
The only repositories that need to support the feature are those containing
installable packages, like AvailablePackagesRepository.
"""
_MASK_FILTER_CACHE_ID = EntropyCacher.CACHE_IDS['mask_filter']
def _mask_filter_fetch_cache(self, package_id):
if self._caching:
return loadobj("%s/%s/%s" % (
MaskableRepository._MASK_FILTER_CACHE_ID, self.name,
package_id,))
def _mask_filter_store_cache(self, package_id, value):
if self._caching:
dumpobj("%s/%s/%s" % (MaskableRepository._MASK_FILTER_CACHE_ID,
self.name, package_id,), value)
def _maskFilter_live(self, package_id):
ref = self._settings['pkg_masking_reference']
if (package_id, self.name) in \
self._settings['live_packagemasking']['mask_matches']:
# do not cache this
return -1, ref['user_live_mask']
elif (package_id, self.name) in \
self._settings['live_packagemasking']['unmask_matches']:
return package_id, ref['user_live_unmask']
def _maskFilter_user_package_mask(self, package_id, live):
with self._settings['mask']:
# thread-safe in here
cache_obj = self._settings['mask'].get()
if cache_obj is None:
cache_obj = {}
self._settings['mask'].set(cache_obj)
user_package_mask_ids = cache_obj.get(self.name)
if user_package_mask_ids is None:
user_package_mask_ids = set()
for atom in self._settings['mask']:
atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
if repository_ids is not None:
if self.name not in repository_ids:
# then the mask doesn't involve us
continue
# check if @repository is specified
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
user_package_mask_ids |= set(matches)
cache_obj[self.name] = user_package_mask_ids
if package_id in user_package_mask_ids:
# sorry, masked
ref = self._settings['pkg_masking_reference']
myr = ref['user_package_mask']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = -1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _maskFilter_user_package_unmask(self, package_id, live):
with self._settings['unmask']:
# thread-safe in here
cache_obj = self._settings['unmask'].get()
if cache_obj is None:
cache_obj = {}
self._settings['unmask'].set(cache_obj)
user_package_unmask_ids = cache_obj.get(self.name)
if user_package_unmask_ids is None:
user_package_unmask_ids = set()
for atom in self._settings['unmask']:
atom, repository_ids = entropy.dep.dep_get_match_in_repos(atom)
if repository_ids is not None:
if self.name not in repository_ids:
# then the mask doesn't involve us
continue
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
user_package_unmask_ids |= set(matches)
cache_obj[self.name] = user_package_unmask_ids
if package_id in user_package_unmask_ids:
ref = self._settings['pkg_masking_reference']
myr = ref['user_package_unmask']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
except KeyError: # system settings client plugin not found
pass
return package_id, myr
def _maskFilter_packages_db_mask(self, package_id, live):
# check if repository packages.db.mask needs it masked
repos_mask = {}
client_plg_id = etpConst['system_settings_plugins_ids']['client_plugin']
client_settings = self._settings.get(client_plg_id, {})
if client_settings:
repos_mask = client_settings['repositories']['mask']
repomask = repos_mask.get(self.name)
if isinstance(repomask, (list, set, frozenset)):
# first, seek into generic masking, all branches
# (below) avoid issues with repository names
mask_repo_id = "%s_ids@@:of:%s" % (self.name, self.name,)
repomask_ids = repos_mask.get(mask_repo_id)
if not isinstance(repomask_ids, set):
repomask_ids = set()
for atom in repomask:
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
repomask_ids |= set(matches)
repos_mask[mask_repo_id] = repomask_ids
if package_id in repomask_ids:
ref = self._settings['pkg_masking_reference']
myr = ref['repository_packages_db_mask']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
-1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _maskFilter_package_license_mask(self, package_id, live):
if not self._settings['license_mask']:
return
mylicenses = self.retrieveLicense(package_id)
mylicenses = mylicenses.strip().split()
lic_mask = self._settings['license_mask']
for mylicense in mylicenses:
if mylicense not in lic_mask:
continue
ref = self._settings['pkg_masking_reference']
myr = ref['user_license_mask']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = -1, myr
except KeyError: # system settings client plugin not found
pass
return -1, myr
def _maskFilter_keyword_mask(self, package_id, live):
# WORKAROUND for buggy entries
# ** is fine then
# TODO: remove this before 31-12-2011
mykeywords = self.retrieveKeywords(package_id)
if mykeywords == set([""]):
mykeywords = set(['**'])
mask_ref = self._settings['pkg_masking_reference']
# firstly, check if package keywords are in etpConst['keywords']
# (universal keywords have been merged from package.keywords)
same_keywords = etpConst['keywords'] & mykeywords
if same_keywords:
myr = mask_ref['system_keyword']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
except KeyError: # system settings client plugin not found
pass
return package_id, myr
# if we get here, it means we didn't find mykeywords
# in etpConst['keywords']
# we need to seek self._settings['keywords']
# seek in repository first
keyword_repo = self._settings['keywords']['repositories']
for keyword in tuple(keyword_repo.get(self.name, {}).keys()):
if keyword not in mykeywords:
continue
keyword_data = keyword_repo[self.name].get(keyword)
if not keyword_data:
continue
if "*" in keyword_data:
# all packages in this repo with keyword "keyword" are ok
myr = mask_ref['user_repo_package_keywords_all']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
except KeyError: # system settings client plugin not found
pass
return package_id, myr
kwd_key = "%s_ids" % (keyword,)
keyword_data_ids = keyword_repo[self.name].get(kwd_key)
if not isinstance(keyword_data_ids, set):
keyword_data_ids = set()
for atom in keyword_data:
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
keyword_data_ids |= matches
keyword_repo[self.name][kwd_key] = keyword_data_ids
if package_id in keyword_data_ids:
myr = mask_ref['user_repo_package_keywords']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
except KeyError: # system settings client plugin not found
pass
return package_id, myr
keyword_pkg = self._settings['keywords']['packages']
# if we get here, it means we didn't find a match in repositories
# so we scan packages, last chance
for keyword in keyword_pkg.keys():
# use .keys() because keyword_pkg gets modified during iteration
# first of all check if keyword is in mykeywords
if keyword not in mykeywords:
continue
keyword_data = keyword_pkg.get(keyword)
if not keyword_data:
continue
kwd_key = "%s_ids" % (keyword,)
keyword_data_ids = keyword_pkg.get(self.name+kwd_key)
if not isinstance(keyword_data_ids, (list, set)):
keyword_data_ids = set()
for atom in keyword_data:
# match atom
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
keyword_data_ids |= matches
keyword_pkg[self.name+kwd_key] = keyword_data_ids
if package_id in keyword_data_ids:
# valid!
myr = mask_ref['user_package_keywords']
try:
cl_data = self._settings[_CL_PLUGIN_ID]
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
except KeyError: # system settings client plugin not found
pass
return package_id, myr
## if we get here, it means that pkg it keyword masked
## and we should look at the very last resort, per-repository
## package keywords
# check if repository contains keyword unmasking data
cl_data = self._settings.get(_CL_PLUGIN_ID)
if cl_data is None:
# SystemSettings Entropy Client plugin not available
return
# let's see if something is available in repository config
repo_keywords = cl_data['repositories']['repos_keywords'].get(
self.name)
if repo_keywords is None:
# nopers, sorry!
return
# check universal keywords
same_keywords = repo_keywords.get('universal') & mykeywords
if same_keywords:
# universal keyword matches!
myr = mask_ref['repository_packages_db_keywords']
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
return package_id, myr
## if we get here, it means that even universal masking failed
## and we need to look at per-package settings
repo_settings = repo_keywords.get('packages')
if not repo_settings:
# it's empty, not worth checking
return
cached_key = "packages_ids"
keyword_data_ids = repo_keywords.get(cached_key)
if not isinstance(keyword_data_ids, dict):
# create cache
keyword_data_ids = {}
for atom, values in repo_settings.items():
matches, r = self.atomMatch(atom, multiMatch = True,
maskFilter = False)
if r != 0:
continue
for match in matches:
obj = keyword_data_ids.setdefault(match, set())
obj.update(values)
repo_keywords[cached_key] = keyword_data_ids
pkg_keywords = keyword_data_ids.get(package_id, set())
if "**" in pkg_keywords:
same_keywords = True
else:
same_keywords = pkg_keywords & etpConst['keywords']
if same_keywords:
# found! this pkg is not masked, yay!
myr = mask_ref['repository_packages_db_keywords']
validator_cache = cl_data['masking_validation']['cache']
validator_cache[(package_id, self.name, live)] = \
package_id, myr
return package_id, myr
def maskFilter(self, package_id, live = True):
"""
Reimplemented from EntropyRepositoryBase
"""
validator_cache = self._settings.get(_CL_PLUGIN_ID, {}).get(
'masking_validation', {}).get('cache', {})
cached = validator_cache.get((package_id, self.name, live))
if cached is not None:
return cached
# use on-disk cache?
cached = self._mask_filter_fetch_cache(package_id)
if cached is not None:
return cached
# avoid memleaks
if len(validator_cache) > 100000:
validator_cache.clear()
if live:
data = self._maskFilter_live(package_id)
if data:
return data
data = self._maskFilter_user_package_mask(package_id, live)
if data:
self._mask_filter_store_cache(package_id, data)
return data
data = self._maskFilter_user_package_unmask(package_id, live)
if data:
self._mask_filter_store_cache(package_id, data)
return data
data = self._maskFilter_packages_db_mask(package_id, live)
if data:
self._mask_filter_store_cache(package_id, data)
return data
data = self._maskFilter_package_license_mask(package_id, live)
if data:
self._mask_filter_store_cache(package_id, data)
return data
data = self._maskFilter_keyword_mask(package_id, live)
if data:
self._mask_filter_store_cache(package_id, data)
return data
# holy crap, can't validate
myr = self._settings['pkg_masking_reference']['completely_masked']
validator_cache[(package_id, self.name, live)] = -1, myr
self._mask_filter_store_cache(package_id, data)
return -1, myr
class AvailablePackagesRepository(CachedRepository, MaskableRepository):
"""
This class represents the available packages repository and is a direct
subclass of EntropyRepository. It implements the update() method in order
to make possible to update the repository.
"""
def __init__(self, *args, **kwargs):
EntropyRepository.__init__(self, *args, **kwargs)
# ensure proper repository file permissions
if entropy.tools.is_root() and os.path.isfile(self._db):
const_setup_file(self._db, etpConst['entropygid'], 0o644,
uid = etpConst['uid'])
@staticmethod
def update(entropy_client, repository_id, force, gpg):
"""
Reimplemented from EntropyRepositoryBase
"""
try:
return AvailablePackagesRepositoryUpdater(entropy_client, repository_id,
force, gpg).update()
except KeyError:
return EntropyRepositoryBase.REPOSITORY_NOT_AVAILABLE
@staticmethod
def revision(repository_id):
"""
Reimplemented from EntropyRepositoryBase
"""
db_data = SystemSettings()['repositories']['available'][repository_id]
fname = os.path.join(db_data['dbpath'],
etpConst['etpdatabaserevisionfile'])
revision = -1
enc = etpConst['conf_encoding']
if os.path.isfile(fname) and os.access(fname, os.R_OK):
with codecs.open(fname, "r", encoding=enc) as f:
try:
read_data = f.readline().strip()
revision = int(read_data)
except (OSError, IOError, ValueError,):
pass
return revision
@staticmethod
def remote_revision(repository_id):
"""
Reimplemented from EntropyRepositoryBase
"""
return AvailablePackagesRepositoryUpdater(TextInterface(),
repository_id, False, False).remote_revision()
def handlePackage(self, pkg_data, forcedRevision = -1,
formattedContent = False):
"""
Reimplemented from EntropyRepository
"""
raise PermissionDenied(
"cannot execute handlePackage on this repository")
def addPackage(self, pkg_data, revision = -1, package_id = None,
formatted_content = False):
"""
Reimplemented from EntropyRepository
"""
raise PermissionDenied(
"cannot execute addPackage on this repository")
def removePackage(self, package_id, from_add_package = False):
"""
Reimplemented from EntropyRepository
"""
raise PermissionDenied(
"cannot execute removePackage on this repository")
def clearCache(self):
# clear package masking filter
cl_data = self._settings.get(_CL_PLUGIN_ID, {})
cl_data.get('masking_validation', {}).get('cache', {}).clear()
EntropyRepository.clearCache(self)
class GenericRepository(CachedRepository, MaskableRepository):
"""
This class represents a generic packages repository and is a direct
subclass of EntropyRepository.
Even GenericRepository is a CachedRepository because its object could
get cached by 3rd party. Actually, we require this because our installed
packages repository could end up being a GenericRepository, when running
in fail-safe mode.
"""
def handlePackage(self, pkg_data, forcedRevision = -1,
formattedContent = False):
"""
Reimplemented from EntropyRepository.
It is supposed that a generic repository should not support
handlePackage. You can override this (at your own risk) by setting the
"override_handlePackage" property to True. In this case, a generic
addPackage() call is issued.
"""
override = getattr(self, 'override_handlePackage', False)
if not override:
raise PermissionDenied(
"cannot execute handlePackage on this repository")
return self.addPackage(pkg_data, revision = forcedRevision,
formatted_content = formattedContent)
def maskFilter(self, package_id, live = True):
"""
Reimplemented from EntropyRepository.
It is supposed that a generic repository doesn't support package
masking. You can override this by setting the "enable_mask_filter"
to True.
"""
enabled = getattr(self, 'enable_mask_filter', False)
if not enabled:
return package_id, 0
return MaskableRepository.maskFilter(self, package_id, live = live)
Reference in New Issue View Git Blame Copy Permalink