sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support to Basic Authentication and permit disable of SSL certificate validation on https protocol

Browse Source
This commit is contained in:
Geaaru
2015-05-08 11:20:43 +02:00
parent d92c7ca45c
commit 38d03dc200
5 changed files with 217 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/entropy/client/interfaces/db.py
+42 -7
View File
@@ -395,6 +395,9 @@ class AvailablePackagesRepositoryUpdater(object):
avail_data = repos_data['available']
repo_data = avail_data[self._repository_id]
database_uris = repo_data['databases']
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
ws_revision = self._remote_webservice_revision()
@@ -418,7 +421,11 @@ class AvailablePackagesRepositoryUpdater(object):
)
repo_uri = uri_meta['uri']
uri_revision = self._remote_revision(repo_uri)
uri_revision = self._remote_revision(repo_uri,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
if uri_revision != -1:
@@ -478,7 +485,11 @@ class AvailablePackagesRepositoryUpdater(object):
if url == uri:
# skip same URL
continue
revision = self._remote_revision(url)
revision = self._remote_revision(url,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
if revision != -1:
# found
self._entropy.output(
@@ -981,7 +992,15 @@ class AvailablePackagesRepositoryUpdater(object):
return url, path
def _download_item(self, uri, item, cmethod = None,
disallow_redirect = True, get_signature = False):
disallow_redirect = True, get_signature = False):
my_repos = self._settings['repositories']
avail_data = my_repos['available']
repo_data = avail_data[self._repository_id]
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
url, filepath = self._construct_paths(
uri, item, cmethod, get_signature = get_signature)
@@ -1011,7 +1030,10 @@ class AvailablePackagesRepositoryUpdater(object):
url,
temp_filepath,
resume = False,
disallow_redirect = disallow_redirect
disallow_redirect = disallow_redirect,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert
)
rc = fetcher.download()
@@ -1913,7 +1935,15 @@ class AvailablePackagesRepositoryUpdater(object):
rev = -1
return rev
rev = self._remote_revision(uri)
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
rev = self._remote_revision(url,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
return rev
def _remote_webservice_revision(self):
@@ -1939,7 +1969,9 @@ class AvailablePackagesRepositoryUpdater(object):
# otherwise, fallback to previous EAPI
self._repo_eapi -= 1
def _remote_revision(self, uri):
def _remote_revision(self, uri, http_basic_user = None,
http_basic_pwd = None,
https_validate_cert = True):
"""
Return the remote repository revision by downloading
the revision file from the given uri.
@@ -1953,7 +1985,10 @@ class AvailablePackagesRepositoryUpdater(object):
tmp_fd, tmp_path = const_mkstemp(
prefix = "AvailableEntropyRepository.remote_revision")
fetcher = self._entropy._url_fetcher(
url, tmp_path, resume = False)
url, tmp_path, resume = False,
http_basic_user = http_basic_user,
http_basic_pwd = http_basic_pwd,
https_validate_cert = https_validate_cert)
fetch_rc = fetcher.download()
if fetch_rc not in self.FETCH_ERRORS:
with codecs.open(tmp_path, "r") as tmp_f:
lib/entropy/client/interfaces/package/actions/fetch.py
+22 -2
View File
@@ -456,11 +456,21 @@ class _PackageFetchAction(PackageAction):
UrlFetcher.GENERIC_FETCH_WARN,
)
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
for delta_url, delta_save in download_plan:
delta_fetcher = self._entropy._url_fetcher(delta_url,
delta_save, resume = delta_resume,
abort_check_func = fetch_abort_function)
abort_check_func = fetch_abort_function,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
try:
# make sure that we don't need to abort already
@@ -550,9 +560,19 @@ class _PackageFetchAction(PackageAction):
if os.path.isfile(download_path) and os.path.exists(download_path):
existed_before = True
avail_data = self._settings['repositories']['available']
repo_data = avail_data[self._repository_id]
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
fetch_intf = self._entropy._url_fetcher(
url, download_path, resume = resume,
abort_check_func = fetch_abort_function)
abort_check_func = fetch_abort_function,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
if (package_id is not None) and (repository_id is not None):
self._setup_differential_download(
lib/entropy/client/interfaces/package/actions/multifetch.py
+18 -3
View File
@@ -382,7 +382,7 @@ class _PackageMultiFetchAction(_PackageFetchAction):
return fetched_url_data, data_transfer, 0
def _download_files(self, url_data, resume = True):
def _download_files(self, url_data, resume = True, repository_id = None):
"""
Effectively fetch the package files.
"""
@@ -449,6 +449,7 @@ class _PackageMultiFetchAction(_PackageFetchAction):
validated_download_ids.add(download_id)
url_path_list = []
last_repos_id = None
for pkg_id, repository_id, url, download_path, _cksum, _sig in url_data:
url_path_list.append((url, download_path))
@@ -463,10 +464,20 @@ class _PackageMultiFetchAction(_PackageFetchAction):
resume, download_path,
repository_id, pkg_id)
# This is horrible but probably
# MultipleUrlFetcher must be reimplemented.
last_repos_id = repository_id
finally:
if lock is not None:
lock.close()
avail_data = self._settings['repositories']['available']
repo_data = avail_data[last_repos_id]
basic_user = repo_data.get('username')
basic_pwd = repo_data.get('password')
https_validate_cert = not repo_data.get('https_validate_cert') == "false"
fetch_abort_function = self._meta.get('fetch_abort_function')
fetch_intf = self._entropy._multiple_url_fetcher(
url_path_list, resume = resume,
@@ -474,7 +485,10 @@ class _PackageMultiFetchAction(_PackageFetchAction):
url_fetcher_class = self._entropy._url_fetcher,
download_context_func = download_context,
pre_download_hook = pre_download_hook,
post_download_hook = post_download_hook)
post_download_hook = post_download_hook,
http_basic_user = basic_user,
http_basic_pwd = basic_pwd,
https_validate_cert = https_validate_cert)
try:
# make sure that we don't need to abort already
# doing the check here avoids timeouts
@@ -757,7 +771,8 @@ class _PackageMultiFetchAction(_PackageFetchAction):
(exit_st, failed_downloads,
data_transfer) = self._download_files(
updated_fetch_files_list,
resume = do_resume)
resume = do_resume,
repository_id = repository_id)
if exit_st == 0:
show_successful_download(
lib/entropy/core/settings/base.py
+85 -6
View File
@@ -83,9 +83,16 @@ class RepositoryConfigParser(BaseConfigParser):
that config files in /etc/entropy/repositories.conf.d/ starting
with "_" are considered to contain disabled repositories. This
is just provided for convienence.
- "username": if set, it used for HTTP Basic Authentication on retrieve
data from remote repository.
- "password": if set, it used for HTTP Basic Authentication on retrieve
data from remote repository.
- "https_validate_cert": if set to "false" disable ssl certificate
validation of the remote repository.
"""
_SUPPORTED_KEYS = ("desc", "repo", "pkg", "enabled")
_SUPPORTED_KEYS = ("desc", "repo", "pkg", "enabled", \
"username", "password", "https_validate_cert")
_DEFAULT_ENABLED_VALUE = True
@@ -114,7 +121,8 @@ class RepositoryConfigParser(BaseConfigParser):
return
return candidate
def add(self, repository_id, desc, repos, pkgs, enabled = True):
def add(self, repository_id, desc, repos, pkgs, enabled = True,
username = None, password = None, https_validate_cert = True):
"""
Add a repository to the repository configuration files directory.
Older repository configuration may get overwritten. This method
@@ -146,7 +154,8 @@ class RepositoryConfigParser(BaseConfigParser):
# while disabled config files start with _
disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name)
self.write(enabled_conf_file, repository_id, desc, repos, pkgs)
self.write(enabled_conf_file, repository_id, desc, repos, pkgs,
username, password, https_validate_cert)
# if any disabled entry file is around, kill it with fire!
try:
@@ -275,7 +284,8 @@ class RepositoryConfigParser(BaseConfigParser):
return accomplished
def write(self, path, repository_id, desc, repos, pkgs, enabled = True):
def write(self, path, repository_id, desc, repos, pkgs, enabled = True,
username = None, password = None, https_validate_cert = True):
"""
Write the repository configuration to the given file.
@@ -310,11 +320,17 @@ class RepositoryConfigParser(BaseConfigParser):
desc = %(desc)s
%(repos)s
enabled = %(enabled)s
%(username)s
%(password)s
%(https_validate_cert)s
""" % {
"repository_id": repository_id,
"desc": desc,
"repos": repos_str.rstrip(),
"enabled": enabled_str,
"username": ("", "username = %s" % username)[username],
"password": ("", "password = %s" % password)[password],
"https_validate_cert" : ("https_validate_cert = false", "")[https_validate_cert]
}
for pkg in pkgs:
config += "pkg = %s\n" % (pkg,)
@@ -397,6 +413,47 @@ enabled = %(enabled)s
except KeyError:
return self._DEFAULT_ENABLED_VALUE
def username(self, repository_id):
"""
Return the username to use with the repository.
@param repository_id: the repository identifier
@type repository_id: string
@raise KeyError: if repository_id is not found or
metadata is not available
@return: the repository username.
@rtype: string
"""
return self[repository_id]["username"][0]
def password(self, repository_id):
"""
Return the password to use with the repository.
@param repository_id: the repository identifier
@type repository_id: string
@raise KeyError: if repository_id is not found or
metadata is not available
@return: the repository password.
@rtype: string
"""
return self[repository_id]["password"][0]
def https_validate_cert(self, repository_id):
"""
Return whether SSL cert validation of remote repository
is enabled. It is used only for HTTPS.
@param repository_id: the repository identifier
@type repository_id: string
@return: status of ssl certificate validation.
@rtype: bool
"""
try:
https_validate_cert = self[repository_id]["https_validate_cert"][0]
return https_validate_cert.strip().lower() == "true"
except KeyError:
return True # Default is enabled
class SystemSettings(Singleton, EntropyPluginStore):
@@ -1682,7 +1739,8 @@ class SystemSettings(Singleton, EntropyPluginStore):
name, desc, packages, [database], product, branch)
def _generate_repository_metadata(self, name, desc, packages, databases,
product, branch):
product, branch, username = None,
password = None, https_validate_cert = True):
"""
Given a set of raw repository metadata information, like name,
description, a list of package urls and the database url, generate
@@ -1752,6 +1810,13 @@ class SystemSettings(Singleton, EntropyPluginStore):
data['packages'] = []
data['plain_packages'] = []
if username and password:
data['username'] = username
data['password'] = password
if not https_validate_cert:
data['https_validate_cert'] = "false"
data['dbpath'] = etpConst['etpdatabaseclientdir'] + os.path.sep + \
name + os.path.sep + product + os.path.sep + \
etpConst['currentarch'] + os.path.sep + branch
@@ -2035,10 +2100,24 @@ class SystemSettings(Singleton, EntropyPluginStore):
except KeyError:
ini_desc = _("No description")
try:
ini_username = ini_parser.username(ini_repository)
except KeyError:
ini_username = None
try:
ini_password = ini_parser.password(ini_repository)
except KeyError:
ini_password = None
ini_https_validate_cert = ini_parser.https_validate_cert(ini_repository)
ini_excluded = not ini_parser.enabled(ini_repository)
ini_data = self._generate_repository_metadata(
ini_repository, ini_desc, ini_pkgs, ini_dbs,
data['product'], data['branch'])
data['product'], data['branch'],
ini_username, ini_password,
ini_https_validate_cert)
if ini_excluded or ini_conf_excluded:
data['excluded'][ini_repository] = ini_data
else:
lib/entropy/fetchers.py
+50 -5
View File
@@ -24,6 +24,8 @@ import pty
import subprocess
import threading
import contextlib
import base64
import ssl
from entropy.const import const_is_python3, const_file_readable
@@ -77,7 +79,9 @@ class UrlFetcher(TextInterface):
abort_check_func = None, disallow_redirect = False,
thread_stop_func = None, speed_limit = None,
timeout = None, download_context_func = None,
pre_download_hook = None, post_download_hook = None):
pre_download_hook = None, post_download_hook = None,
http_basic_user = None, http_basic_pwd = None,
https_validate_cert = True):
"""
Entropy URL downloader constructor.
@@ -166,6 +170,12 @@ class UrlFetcher(TextInterface):
self.__disallow_redirect = disallow_redirect
self.__speedlimit = speed_limit # kbytes/sec
# HTTP Basic Authentication parameters
self.__http_basic_user = http_basic_user
self.__http_basic_pwd = http_basic_pwd
# SSL Context options
self.__https_validate_cert = https_validate_cert
self._init_vars()
self.__init_urllib()
@@ -583,8 +593,21 @@ class UrlFetcher(TextInterface):
)
if url_protocol in ("http", "https"):
headers = {'User-Agent': user_agent,}
# Handle HTTP Basic auth
if self.__http_basic_user and self.__http_basic_pwd:
basic_header = base64.encodestring('%s:%s' % (
self.__http_basic_user, self.__http_basic_pwd)).replace('\n', '')
headers = {
'User-Agent': user_agent,
'Authorization': 'Basic %s' % basic_header,
}
else:
headers = {'User-Agent': user_agent,}
req = urlmod.Request(url, headers = headers)
else:
req = url
@@ -594,7 +617,18 @@ class UrlFetcher(TextInterface):
# get file size if available
try:
self.__remotefile = urlmod.urlopen(req, None, self.__timeout)
if url_protocol in ("https") and \
not self.__https_validate_cert:
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
self.__remotefile = urlmod.urlopen(req, None, self.__timeout,
context=ctx)
else:
self.__remotefile = urlmod.urlopen(req, None, self.__timeout)
except KeyboardInterrupt:
self.__urllib_close(False)
raise
@@ -976,7 +1010,9 @@ class MultipleUrlFetcher(TextInterface):
abort_check_func = None, disallow_redirect = False,
url_fetcher_class = None, timeout = None,
download_context_func = None,
pre_download_hook = None, post_download_hook = None):
pre_download_hook = None, post_download_hook = None,
http_basic_user = None, http_basic_pwd = None,
https_validate_cert = True):
"""
@param url_path_list: list of tuples composed by url and
path to save, for eg. [(url,path_to_save,),...]
@@ -1045,6 +1081,12 @@ class MultipleUrlFetcher(TextInterface):
if self.__url_fetcher == None:
self.__url_fetcher = UrlFetcher
# HTTP Basic Authentication parameters
self.__http_basic_user = http_basic_user
self.__http_basic_pwd = http_basic_pwd
# SSL Context options
self.__https_validate_cert = https_validate_cert
def __handle_threads_stop(self):
if self.__stop_threads:
raise InterruptError("interrupted")
@@ -1128,7 +1170,10 @@ class MultipleUrlFetcher(TextInterface):
timeout = self.__timeout,
download_context_func = self.__download_context_func,
pre_download_hook = self.__pre_download_hook,
post_download_hook = self.__post_download_hook
post_download_hook = self.__post_download_hook,
http_basic_user = self.__http_basic_user,
http_basic_pwd = self.__http_basic_pwd,
https_validate_cert = self.__https_validate_cert
)
downloader.set_id(th_id)