diff --git a/lib/entropy/client/interfaces/db.py b/lib/entropy/client/interfaces/db.py index 91c422ed6..7e7856e65 100644 --- a/lib/entropy/client/interfaces/db.py +++ b/lib/entropy/client/interfaces/db.py @@ -395,6 +395,9 @@ class AvailablePackagesRepositoryUpdater(object): avail_data = repos_data['available'] repo_data = avail_data[self._repository_id] database_uris = repo_data['databases'] + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" ws_revision = self._remote_webservice_revision() @@ -418,7 +421,11 @@ class AvailablePackagesRepositoryUpdater(object): ) repo_uri = uri_meta['uri'] - uri_revision = self._remote_revision(repo_uri) + + uri_revision = self._remote_revision(repo_uri, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) if uri_revision != -1: @@ -478,7 +485,11 @@ class AvailablePackagesRepositoryUpdater(object): if url == uri: # skip same URL continue - revision = self._remote_revision(url) + + revision = self._remote_revision(url, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) if revision != -1: # found self._entropy.output( @@ -981,7 +992,15 @@ class AvailablePackagesRepositoryUpdater(object): return url, path def _download_item(self, uri, item, cmethod = None, - disallow_redirect = True, get_signature = False): + disallow_redirect = True, get_signature = False): + + my_repos = self._settings['repositories'] + avail_data = my_repos['available'] + repo_data = avail_data[self._repository_id] + + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" url, filepath = self._construct_paths( uri, item, cmethod, get_signature = get_signature) @@ -1011,7 +1030,10 @@ class AvailablePackagesRepositoryUpdater(object): url, temp_filepath, resume = False, - disallow_redirect = disallow_redirect + disallow_redirect = disallow_redirect, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert ) rc = fetcher.download() @@ -1913,7 +1935,15 @@ class AvailablePackagesRepositoryUpdater(object): rev = -1 return rev - rev = self._remote_revision(uri) + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" + + rev = self._remote_revision(url, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) + return rev def _remote_webservice_revision(self): @@ -1939,7 +1969,9 @@ class AvailablePackagesRepositoryUpdater(object): # otherwise, fallback to previous EAPI self._repo_eapi -= 1 - def _remote_revision(self, uri): + def _remote_revision(self, uri, http_basic_user = None, + http_basic_pwd = None, + https_validate_cert = True): """ Return the remote repository revision by downloading the revision file from the given uri. @@ -1953,7 +1985,10 @@ class AvailablePackagesRepositoryUpdater(object): tmp_fd, tmp_path = const_mkstemp( prefix = "AvailableEntropyRepository.remote_revision") fetcher = self._entropy._url_fetcher( - url, tmp_path, resume = False) + url, tmp_path, resume = False, + http_basic_user = http_basic_user, + http_basic_pwd = http_basic_pwd, + https_validate_cert = https_validate_cert) fetch_rc = fetcher.download() if fetch_rc not in self.FETCH_ERRORS: with codecs.open(tmp_path, "r") as tmp_f: diff --git a/lib/entropy/client/interfaces/package/actions/fetch.py b/lib/entropy/client/interfaces/package/actions/fetch.py index 6befb86cb..1be24523e 100644 --- a/lib/entropy/client/interfaces/package/actions/fetch.py +++ b/lib/entropy/client/interfaces/package/actions/fetch.py @@ -456,11 +456,21 @@ class _PackageFetchAction(PackageAction): UrlFetcher.GENERIC_FETCH_WARN, ) + avail_data = self._settings['repositories']['available'] + repo_data = avail_data[self._repository_id] + + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" + for delta_url, delta_save in download_plan: delta_fetcher = self._entropy._url_fetcher(delta_url, delta_save, resume = delta_resume, - abort_check_func = fetch_abort_function) + abort_check_func = fetch_abort_function, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) try: # make sure that we don't need to abort already @@ -550,9 +560,19 @@ class _PackageFetchAction(PackageAction): if os.path.isfile(download_path) and os.path.exists(download_path): existed_before = True + avail_data = self._settings['repositories']['available'] + repo_data = avail_data[self._repository_id] + + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" + fetch_intf = self._entropy._url_fetcher( url, download_path, resume = resume, - abort_check_func = fetch_abort_function) + abort_check_func = fetch_abort_function, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) if (package_id is not None) and (repository_id is not None): self._setup_differential_download( diff --git a/lib/entropy/client/interfaces/package/actions/multifetch.py b/lib/entropy/client/interfaces/package/actions/multifetch.py index 58369c1a0..015bd1a8f 100644 --- a/lib/entropy/client/interfaces/package/actions/multifetch.py +++ b/lib/entropy/client/interfaces/package/actions/multifetch.py @@ -382,7 +382,7 @@ class _PackageMultiFetchAction(_PackageFetchAction): return fetched_url_data, data_transfer, 0 - def _download_files(self, url_data, resume = True): + def _download_files(self, url_data, resume = True, repository_id = None): """ Effectively fetch the package files. """ @@ -449,6 +449,7 @@ class _PackageMultiFetchAction(_PackageFetchAction): validated_download_ids.add(download_id) url_path_list = [] + last_repos_id = None for pkg_id, repository_id, url, download_path, _cksum, _sig in url_data: url_path_list.append((url, download_path)) @@ -463,10 +464,20 @@ class _PackageMultiFetchAction(_PackageFetchAction): resume, download_path, repository_id, pkg_id) + # This is horrible but probably + # MultipleUrlFetcher must be reimplemented. + last_repos_id = repository_id + finally: if lock is not None: lock.close() + avail_data = self._settings['repositories']['available'] + repo_data = avail_data[last_repos_id] + basic_user = repo_data.get('username') + basic_pwd = repo_data.get('password') + https_validate_cert = not repo_data.get('https_validate_cert') == "false" + fetch_abort_function = self._meta.get('fetch_abort_function') fetch_intf = self._entropy._multiple_url_fetcher( url_path_list, resume = resume, @@ -474,7 +485,10 @@ class _PackageMultiFetchAction(_PackageFetchAction): url_fetcher_class = self._entropy._url_fetcher, download_context_func = download_context, pre_download_hook = pre_download_hook, - post_download_hook = post_download_hook) + post_download_hook = post_download_hook, + http_basic_user = basic_user, + http_basic_pwd = basic_pwd, + https_validate_cert = https_validate_cert) try: # make sure that we don't need to abort already # doing the check here avoids timeouts @@ -757,7 +771,8 @@ class _PackageMultiFetchAction(_PackageFetchAction): (exit_st, failed_downloads, data_transfer) = self._download_files( updated_fetch_files_list, - resume = do_resume) + resume = do_resume, + repository_id = repository_id) if exit_st == 0: show_successful_download( diff --git a/lib/entropy/core/settings/base.py b/lib/entropy/core/settings/base.py index 8a6fc01aa..4c9d9f34d 100644 --- a/lib/entropy/core/settings/base.py +++ b/lib/entropy/core/settings/base.py @@ -83,9 +83,16 @@ class RepositoryConfigParser(BaseConfigParser): that config files in /etc/entropy/repositories.conf.d/ starting with "_" are considered to contain disabled repositories. This is just provided for convienence. + - "username": if set, it used for HTTP Basic Authentication on retrieve + data from remote repository. + - "password": if set, it used for HTTP Basic Authentication on retrieve + data from remote repository. + - "https_validate_cert": if set to "false" disable ssl certificate + validation of the remote repository. """ - _SUPPORTED_KEYS = ("desc", "repo", "pkg", "enabled") + _SUPPORTED_KEYS = ("desc", "repo", "pkg", "enabled", \ + "username", "password", "https_validate_cert") _DEFAULT_ENABLED_VALUE = True @@ -114,7 +121,8 @@ class RepositoryConfigParser(BaseConfigParser): return return candidate - def add(self, repository_id, desc, repos, pkgs, enabled = True): + def add(self, repository_id, desc, repos, pkgs, enabled = True, + username = None, password = None, https_validate_cert = True): """ Add a repository to the repository configuration files directory. Older repository configuration may get overwritten. This method @@ -146,7 +154,8 @@ class RepositoryConfigParser(BaseConfigParser): # while disabled config files start with _ disabled_conf_file = os.path.join(conf_d_dir, "_" + base_name) - self.write(enabled_conf_file, repository_id, desc, repos, pkgs) + self.write(enabled_conf_file, repository_id, desc, repos, pkgs, + username, password, https_validate_cert) # if any disabled entry file is around, kill it with fire! try: @@ -275,7 +284,8 @@ class RepositoryConfigParser(BaseConfigParser): return accomplished - def write(self, path, repository_id, desc, repos, pkgs, enabled = True): + def write(self, path, repository_id, desc, repos, pkgs, enabled = True, + username = None, password = None, https_validate_cert = True): """ Write the repository configuration to the given file. @@ -310,11 +320,17 @@ class RepositoryConfigParser(BaseConfigParser): desc = %(desc)s %(repos)s enabled = %(enabled)s +%(username)s +%(password)s +%(https_validate_cert)s """ % { "repository_id": repository_id, "desc": desc, "repos": repos_str.rstrip(), "enabled": enabled_str, + "username": ("", "username = %s" % username)[username], + "password": ("", "password = %s" % password)[password], + "https_validate_cert" : ("https_validate_cert = false", "")[https_validate_cert] } for pkg in pkgs: config += "pkg = %s\n" % (pkg,) @@ -397,6 +413,47 @@ enabled = %(enabled)s except KeyError: return self._DEFAULT_ENABLED_VALUE + def username(self, repository_id): + """ + Return the username to use with the repository. + + @param repository_id: the repository identifier + @type repository_id: string + @raise KeyError: if repository_id is not found or + metadata is not available + @return: the repository username. + @rtype: string + """ + return self[repository_id]["username"][0] + + def password(self, repository_id): + """ + Return the password to use with the repository. + + @param repository_id: the repository identifier + @type repository_id: string + @raise KeyError: if repository_id is not found or + metadata is not available + @return: the repository password. + @rtype: string + """ + return self[repository_id]["password"][0] + + def https_validate_cert(self, repository_id): + """ + Return whether SSL cert validation of remote repository + is enabled. It is used only for HTTPS. + + @param repository_id: the repository identifier + @type repository_id: string + @return: status of ssl certificate validation. + @rtype: bool + """ + try: + https_validate_cert = self[repository_id]["https_validate_cert"][0] + return https_validate_cert.strip().lower() == "true" + except KeyError: + return True # Default is enabled class SystemSettings(Singleton, EntropyPluginStore): @@ -1682,7 +1739,8 @@ class SystemSettings(Singleton, EntropyPluginStore): name, desc, packages, [database], product, branch) def _generate_repository_metadata(self, name, desc, packages, databases, - product, branch): + product, branch, username = None, + password = None, https_validate_cert = True): """ Given a set of raw repository metadata information, like name, description, a list of package urls and the database url, generate @@ -1752,6 +1810,13 @@ class SystemSettings(Singleton, EntropyPluginStore): data['packages'] = [] data['plain_packages'] = [] + if username and password: + data['username'] = username + data['password'] = password + + if not https_validate_cert: + data['https_validate_cert'] = "false" + data['dbpath'] = etpConst['etpdatabaseclientdir'] + os.path.sep + \ name + os.path.sep + product + os.path.sep + \ etpConst['currentarch'] + os.path.sep + branch @@ -2035,10 +2100,24 @@ class SystemSettings(Singleton, EntropyPluginStore): except KeyError: ini_desc = _("No description") + try: + ini_username = ini_parser.username(ini_repository) + except KeyError: + ini_username = None + + try: + ini_password = ini_parser.password(ini_repository) + except KeyError: + ini_password = None + + ini_https_validate_cert = ini_parser.https_validate_cert(ini_repository) + ini_excluded = not ini_parser.enabled(ini_repository) ini_data = self._generate_repository_metadata( ini_repository, ini_desc, ini_pkgs, ini_dbs, - data['product'], data['branch']) + data['product'], data['branch'], + ini_username, ini_password, + ini_https_validate_cert) if ini_excluded or ini_conf_excluded: data['excluded'][ini_repository] = ini_data else: diff --git a/lib/entropy/fetchers.py b/lib/entropy/fetchers.py index 10953b06f..27ce1f069 100644 --- a/lib/entropy/fetchers.py +++ b/lib/entropy/fetchers.py @@ -24,6 +24,8 @@ import pty import subprocess import threading import contextlib +import base64 +import ssl from entropy.const import const_is_python3, const_file_readable @@ -77,7 +79,9 @@ class UrlFetcher(TextInterface): abort_check_func = None, disallow_redirect = False, thread_stop_func = None, speed_limit = None, timeout = None, download_context_func = None, - pre_download_hook = None, post_download_hook = None): + pre_download_hook = None, post_download_hook = None, + http_basic_user = None, http_basic_pwd = None, + https_validate_cert = True): """ Entropy URL downloader constructor. @@ -166,6 +170,12 @@ class UrlFetcher(TextInterface): self.__disallow_redirect = disallow_redirect self.__speedlimit = speed_limit # kbytes/sec + # HTTP Basic Authentication parameters + self.__http_basic_user = http_basic_user + self.__http_basic_pwd = http_basic_pwd + # SSL Context options + self.__https_validate_cert = https_validate_cert + self._init_vars() self.__init_urllib() @@ -583,8 +593,21 @@ class UrlFetcher(TextInterface): ) if url_protocol in ("http", "https"): - headers = {'User-Agent': user_agent,} + + # Handle HTTP Basic auth + if self.__http_basic_user and self.__http_basic_pwd: + basic_header = base64.encodestring('%s:%s' % ( + self.__http_basic_user, self.__http_basic_pwd)).replace('\n', '') + + headers = { + 'User-Agent': user_agent, + 'Authorization': 'Basic %s' % basic_header, + } + else: + headers = {'User-Agent': user_agent,} + req = urlmod.Request(url, headers = headers) + else: req = url @@ -594,7 +617,18 @@ class UrlFetcher(TextInterface): # get file size if available try: - self.__remotefile = urlmod.urlopen(req, None, self.__timeout) + if url_protocol in ("https") and \ + not self.__https_validate_cert: + + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + self.__remotefile = urlmod.urlopen(req, None, self.__timeout, + context=ctx) + + else: + self.__remotefile = urlmod.urlopen(req, None, self.__timeout) except KeyboardInterrupt: self.__urllib_close(False) raise @@ -976,7 +1010,9 @@ class MultipleUrlFetcher(TextInterface): abort_check_func = None, disallow_redirect = False, url_fetcher_class = None, timeout = None, download_context_func = None, - pre_download_hook = None, post_download_hook = None): + pre_download_hook = None, post_download_hook = None, + http_basic_user = None, http_basic_pwd = None, + https_validate_cert = True): """ @param url_path_list: list of tuples composed by url and path to save, for eg. [(url,path_to_save,),...] @@ -1045,6 +1081,12 @@ class MultipleUrlFetcher(TextInterface): if self.__url_fetcher == None: self.__url_fetcher = UrlFetcher + # HTTP Basic Authentication parameters + self.__http_basic_user = http_basic_user + self.__http_basic_pwd = http_basic_pwd + # SSL Context options + self.__https_validate_cert = https_validate_cert + def __handle_threads_stop(self): if self.__stop_threads: raise InterruptError("interrupted") @@ -1128,7 +1170,10 @@ class MultipleUrlFetcher(TextInterface): timeout = self.__timeout, download_context_func = self.__download_context_func, pre_download_hook = self.__pre_download_hook, - post_download_hook = self.__post_download_hook + post_download_hook = self.__post_download_hook, + http_basic_user = self.__http_basic_user, + http_basic_pwd = self.__http_basic_pwd, + https_validate_cert = self.__https_validate_cert ) downloader.set_id(th_id)