From 94867af2fe8a36cff22a85c041e9b9d401dcd879 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Sat, 21 Nov 2009 13:41:38 +0100 Subject: [PATCH] [entropy.server] remove etpConst['officialserverrepositoryid'] and handlers support (there is ssh URI support now) --- conf/server.conf.example | 19 +- libraries/entropy/client/interfaces/qa.py | 2 +- libraries/entropy/const.py | 17 +- libraries/entropy/server/interfaces/main.py | 183 +++++++----------- .../entropy/server/interfaces/mirrors.py | 4 +- libraries/entropy/server/transceivers.py | 99 +--------- libraries/tests/server.py | 2 +- 7 files changed, 92 insertions(+), 234 deletions(-) diff --git a/conf/server.conf.example b/conf/server.conf.example index e5432837f..306326126 100644 --- a/conf/server.conf.example +++ b/conf/server.conf.example @@ -15,22 +15,25 @@ # -# syntax for officialserverrepositoryid: +# syntax for default-repository: # -# officialserverrepositoryid: Repository identifier for the official main distribution repository -# officialserverrepositoryid| +# default-repository: Currently working repository identifier (if you have +# more than one server-side repository, this indicates the one which +# Entropy Server is authorized to work on) +# default-repository| # -officialserverrepositoryid|sabayonlinux.org +default-repository|sabayonlinux.org # -# syntax for repositories: +# syntax for repository: # # repository: server-side repositories configuration -# note: default repository setting is in repositories.conf (officialrepositoryid) -#| repository|repository_id|repository description| <2>||#, +# Currently supported upload URI protocols are ftp:// and ssh:// +# => repository||| <2>[|#,] +# (EAPI3 part is optional) # # example: -#| repository|sabayonlinux.org|Sabayon Linux Official Repository|ftp://user:pass@111.111.111.111/|http://svn.sabayonlinux.org/entropy|svn.sabayonlinux.org#1026,1027 +# => repository|myserverrepo|My Server Repository|ftp://user:pass@111.111.111.111/ ssh://username@host:~user/path:port|hostname.tld#1026,1027 # # repository| diff --git a/libraries/entropy/client/interfaces/qa.py b/libraries/entropy/client/interfaces/qa.py index c35351bbf..a4ef2c2e5 100644 --- a/libraries/entropy/client/interfaces/qa.py +++ b/libraries/entropy/client/interfaces/qa.py @@ -50,7 +50,7 @@ class UGCErrorReportInterface(ErrorReportInterface): self.__system_settings = SystemSettings() if repository_id == None: - repository_id = etpConst['officialserverrepositoryid'] + repository_id = etpConst['officialrepositoryid'] self.entropy = Client() self.__repository_id = repository_id diff --git a/libraries/entropy/const.py b/libraries/entropy/const.py index 98e56fb4a..c73550280 100644 --- a/libraries/entropy/const.py +++ b/libraries/entropy/const.py @@ -455,8 +455,7 @@ def const_default_settings(rootdir): # and handle the removal through expiration (using creation date) 'expiration_based_scope': False, # our official repository name - 'officialserverrepositoryid': "sabayonlinux.org", - # our official repository name + 'defaultserverrepositoryid': None, 'officialrepositoryid': "sabayonlinux.org", 'conntestlink': "http://www.sabayonlinux.org", # tag to append to .tbz2 file before entropy database (must be 32bytes) @@ -708,12 +707,6 @@ def const_default_settings(rootdir): 'ugc_accessfile': default_etp_ugc_confdir+"/access.xml", 'ugc_voterange': list(range(1, 6)), - # handler settings - 'handlers': { - # md5sum handler, - 'md5sum': "md5sum.php?arch="+etpSys['arch']+"&package=", - }, - } # set current nice level @@ -1196,13 +1189,12 @@ def const_extract_srv_repo_params(repostring, product = None): repoid = repostring.split("|")[1].strip() repodesc = repostring.split("|")[2].strip() repouris = repostring.split("|")[3].strip() - repohandlers = repostring.split("|")[4].strip() service_url = None eapi3_port = int(etpConst['socket_service']['port']) eapi3_ssl_port = int(etpConst['socket_service']['ssl_port']) - if len(repostring.split("|")) > 5: - service_url = repostring.split("|")[5].strip() + if len(repostring.split("|")) > 4: + service_url = repostring.split("|")[4].strip() eapi3_formatcolon = service_url.rfind("#") if eapi3_formatcolon != -1: @@ -1224,9 +1216,6 @@ def const_extract_srv_repo_params(repostring, product = None): mydata['service_url'] = service_url mydata['service_port'] = eapi3_port mydata['ssl_service_port'] = eapi3_ssl_port - if repohandlers: - repohandlers = os.path.join(repohandlers, product, repoid, "handlers") - mydata['handler'] = repohandlers uris = repouris.split() for uri in uris: mydata['mirrors'].append(uri) diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py index 6ed17d770..0dd1c94fb 100644 --- a/libraries/entropy/server/interfaces/main.py +++ b/libraries/entropy/server/interfaces/main.py @@ -358,7 +358,7 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin): 'repositories': etpConst['server_repositories'].copy(), 'sync_speed_limit': None, 'qa_langs': ["en_US", "C"], - 'default_repository_id': etpConst['officialserverrepositoryid'], + 'default_repository_id': etpConst['defaultserverrepositoryid'], 'packages_expiration_days': etpConst['packagesexpirationdays'], 'database_file_format': etpConst['etpdatabasefileformat'], 'disabled_eapis': set(), @@ -382,16 +382,30 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin): with open(etpConst['serverconf'], "r") as server_f: serverconf = [x.strip() for x in server_f.readlines() if x.strip()] + default_repo_changed = False + for line in serverconf: split_line = line.split("|") split_line_len = len(split_line) + # TODO: remove this in future, supported for backward compat. if (line.find("officialserverrepositoryid|") != -1) and \ (not line.startswith("#")) and (split_line_len == 2): + # TODO: added for backward and mixed compat. + if default_repo_changed: + continue + + if not fake_instance: + data['default_repository_id'] = split_line[1].strip() + + elif (line.find("default-repository|") != -1) and \ + (not line.startswith("#")) and (split_line_len == 2): + if not fake_instance: data['default_repository_id'] = split_line[1].strip() + default_repo_changed = True elif line.startswith("expiration-days|") and (split_line_len == 2): @@ -963,12 +977,12 @@ class Server(Singleton, TextInterface): found = False new_content = [] for line in content: - if line.strip().startswith("officialserverrepositoryid|"): - line = "officialserverrepositoryid|%s" % (repoid,) + if line.strip().startswith("default-repository|"): + line = "default-repository|%s" % (repoid,) found = True new_content.append(line) if not found: - new_content.append("officialserverrepositoryid|%s" % (repoid,)) + new_content.append("default-repository|%s" % (repoid,)) f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp", "w") for line in new_content: f_srv_t.write(line+"\n") @@ -978,7 +992,7 @@ class Server(Singleton, TextInterface): etpConst['serverconf']) else: f_srv = open(etpConst['serverconf'], "w") - f_srv.write("officialserverrepositoryid|%s\n" % (repoid,)) + f_srv.write("default-repository|%s\n" % (repoid,)) f_srv.flush() f_srv.close() @@ -1144,7 +1158,7 @@ class Server(Singleton, TextInterface): etpConst['server_repositories'][repoid] = repodata if set_as_default: - etpConst['officialserverrepositoryid'] = repoid + etpConst['defaultserverrepositoryid'] = repoid sys_set = SystemSettings() sys_set.clear() @@ -2038,7 +2052,6 @@ class Server(Singleton, TextInterface): [uri], downloader_queue, critical_files = downloader_queue, - use_handlers = True, txc_basedir = basedir, local_basedir = tmp_down_dir, download = True, @@ -3639,50 +3652,6 @@ class Server(Singleton, TextInterface): ) return idpackages, False - def get_remote_package_checksum(self, repo, filename, branch): - - if sys.hexversion >= 0x3000000: - import urllib.request as urlmod - import urllib.error as urlmod_error - else: - import urllib2 as urlmod - import urllib2 as urlmod_error - - srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server'] - if 'handler' not in srv_set['repositories'][repo]: - return None - url = srv_set['repositories'][repo]['handler'] - - # does the package has "#" (== tag) ? hackish thing that works - filename = filename.replace("#", "%23") - # "+" - filename = filename.replace("+", "%2b") - request = os.path.join(url, etpConst['handlers']['md5sum']) - request += filename+"&branch="+branch - - proxy_settings = self.SystemSettings['system']['proxy'] - try: - mydict = {} - if proxy_settings['ftp']: - mydict['ftp'] = proxy_settings['ftp'] - if proxy_settings['http']: - mydict['http'] = proxy_settings['http'] - if mydict: - mydict['username'] = proxy_settings['username'] - mydict['password'] = proxy_settings['password'] - self.entropyTools.add_proxy_opener(urlmod, mydict) - else: - # unset - urlmod._opener = None - item = urlmod.urlopen(request) - result = item.readline().strip() - item.close() - del item - return result - except (urlmod_error.URLError, urlmod_error.HTTPError,): - # no HTTP support? - return None - def verify_remote_packages(self, packages, ask = True, repo = None): if repo is None: @@ -3758,79 +3727,71 @@ class Server(Singleton, TextInterface): totalcounter = len(idpackages) currentcounter = 0 - for idpackage in idpackages: - currentcounter += 1 - pkgfile = dbconn.retrieveDownloadURL(idpackage) - orig_branch = self.get_branch_from_download_relative_uri( - pkgfile) + txc = self.Transceiver(uri) + txc.set_verbosity(False) + with txc as handler: - self.updateProgress( - "[%s] %s: %s" % ( - brown(crippled_uri), - blue(_("checking hash")), - darkgreen(pkgfile), - ), - importance = 1, - type = "info", - header = blue(" @@ "), - back = True, - count = (currentcounter, totalcounter,) - ) + for idpackage in idpackages: - checksum_ok = False - ck_remote = self.get_remote_package_checksum(repo, - os.path.basename(pkgfile), orig_branch) - if ck_remote is None: - self.updateProgress( - "[%s] %s: %s %s" % ( - brown(crippled_uri), - blue(_("digest verification of")), - bold(pkgfile), - blue(_("not supported")), - ), - importance = 1, - type = "info", - header = blue(" @@ "), - count = (currentcounter, totalcounter,) - ) - elif len(ck_remote) == 32: + currentcounter += 1 + pkgfile = dbconn.retrieveDownloadURL(idpackage) pkghash = dbconn.retrieveDigest(idpackage) - if ck_remote == pkghash: - checksum_ok = True - else: + self.updateProgress( - "[%s] %s: %s %s" % ( + "[%s] %s: %s" % ( brown(crippled_uri), - blue(_("digest verification of")), - bold(pkgfile), - blue(_("failed for unknown reasons")), + blue(_("checking hash")), + darkgreen(pkgfile), ), importance = 1, type = "info", header = blue(" @@ "), + back = True, count = (currentcounter, totalcounter,) ) - if checksum_ok: - match.add(idpackage) - else: - not_match.add(idpackage) - self.updateProgress( - "[%s] %s: %s %s" % ( - brown(crippled_uri), - blue(_("package")), - bold(pkgfile), - red(_("NOT healthy")), - ), - importance = 1, - type = "warning", - header = darkred(" !!! "), - count = (currentcounter, totalcounter,) - ) - if crippled_uri not in broken_packages: - broken_packages[crippled_uri] = [] - broken_packages[crippled_uri].append(pkgfile) + pkg_rel_path = '/'.join(pkgfile.split("/")[2:]) + + pkgfile = os.path.join( + self.get_remote_packages_relative_path(repo), + pkg_rel_path) + + ck_remote = handler.get_md5(pkgfile) + if ck_remote is None: + self.updateProgress( + "[%s] %s: %s %s" % ( + brown(crippled_uri), + blue(_("digest verification of")), + bold(pkgfile), + blue(_("not supported")), + ), + importance = 1, + type = "info", + header = blue(" @@ "), + count = (currentcounter, totalcounter,) + ) + continue + + if ck_remote == pkghash: + match.add(idpackage) + else: + not_match.add(idpackage) + self.updateProgress( + "[%s] %s: %s %s" % ( + brown(crippled_uri), + blue(_("package")), + bold(pkgfile), + red(_("NOT healthy")), + ), + importance = 1, + type = "warning", + header = darkred(" !!! "), + count = (currentcounter, totalcounter,) + ) + if crippled_uri not in broken_packages: + broken_packages[crippled_uri] = [] + broken_packages[crippled_uri].append(pkgfile) if broken_packages: mytxt = blue("%s:") % ( diff --git a/libraries/entropy/server/interfaces/mirrors.py b/libraries/entropy/server/interfaces/mirrors.py index 5ecc799f1..4b122f1ff 100644 --- a/libraries/entropy/server/interfaces/mirrors.py +++ b/libraries/entropy/server/interfaces/mirrors.py @@ -2640,7 +2640,7 @@ class Server: uploader = self.TransceiverServerHandler(self.Entropy, [uri], myqueue, critical_files = myqueue, - use_handlers = True, txc_basedir = remote_dir, + txc_basedir = remote_dir, handlers_data = {'branch': branch }, repo = repo) errors, m_fine_uris, m_broken_uris = uploader.go() @@ -2695,7 +2695,7 @@ class Server: self.Entropy.get_local_packages_directory(repo), branch) downloader = self.TransceiverServerHandler( self.Entropy, [uri], myqueue, - critical_files = myqueue, use_handlers = True, + critical_files = myqueue, txc_basedir = remote_dir, local_basedir = local_basedir, handlers_data = {'branch': branch }, download = True, repo = repo) diff --git a/libraries/entropy/server/transceivers.py b/libraries/entropy/server/transceivers.py index 58c04bb4e..f847b3bef 100644 --- a/libraries/entropy/server/transceivers.py +++ b/libraries/entropy/server/transceivers.py @@ -24,7 +24,7 @@ class TransceiverServerHandler: def __init__(self, entropy_interface, uris, files_to_upload, download = False, remove = False, txc_basedir = None, - local_basedir = None, critical_files = None, use_handlers = False, + local_basedir = None, critical_files = None, handlers_data = None, repo = None): if critical_files is None: @@ -61,10 +61,8 @@ class TransceiverServerHandler: self.repo = repo if self.repo == None: self.repo = self.Entropy.default_repository - self.use_handlers = use_handlers if self.remove: self.download = False - self.use_handlers = False if not txc_basedir: # default to database directory @@ -162,98 +160,7 @@ class TransceiverServerHandler: ) valid_remote_md5 = False - if not self.use_handlers: - # handlers usage is disabled - return valid_remote_md5 # always valid - - checksum = self.Entropy.get_remote_package_checksum( - self.repo, - os.path.basename(local_filepath), - self.handlers_data['branch'] - ) - if checksum == None: - self.Entropy.updateProgress( - "[%s|#%s|(%s/%s)] %s: %s: %s" % ( - blue(crippled_uri), - darkgreen(str(tries)), - blue(str(counter)), - bold(str(maxcount)), - blue(_("digest verification")), - os.path.basename(local_filepath), - darkred(_("not supported")), - ), - importance = 0, - type = "info", - header = red(" @@ ") - ) - return valid_remote_md5 - elif isinstance(checksum, bool) and not checksum: - self.Entropy.updateProgress( - "[%s|#%s|(%s/%s)] %s: %s: %s" % ( - blue(crippled_uri), - darkgreen(str(tries)), - blue(str(counter)), - bold(str(maxcount)), - blue(_("digest verification")), - os.path.basename(local_filepath), - bold(_("file not found")), - ), - importance = 0, - type = "warning", - header = brown(" @@ ") - ) - return False - elif is_valid_md5(checksum): - # valid? checking - ckres = compare_md5(local_filepath, checksum) - if ckres: - self.Entropy.updateProgress( - "[%s|#%s|(%s/%s)] %s: %s: %s" % ( - blue(crippled_uri), - darkgreen(str(tries)), - blue(str(counter)), - bold(str(maxcount)), - blue(_("digest verification")), - os.path.basename(local_filepath), - darkgreen(_("so far, so good!")), - ), - importance = 0, - type = "info", - header = red(" @@ ") - ) - return True - else: - self.Entropy.updateProgress( - "[%s|#%s|(%s/%s)] %s: %s: %s" % ( - blue(crippled_uri), - darkgreen(str(tries)), - blue(str(counter)), - bold(str(maxcount)), - blue(_("digest verification")), - os.path.basename(local_filepath), - darkred(_("invalid checksum")), - ), - importance = 0, - type = "warning", - header = brown(" @@ ") - ) - return False - else: - self.Entropy.updateProgress( - "[%s|#%s|(%s/%s)] %s: %s: %s" % ( - blue(crippled_uri), - darkgreen(str(tries)), - blue(str(counter)), - bold(str(maxcount)), - blue(_("digest verification")), - os.path.basename(local_filepath), - darkred(_("unknown data returned")), - ), - importance = 0, - type = "warning", - header = brown(" @@ ") - ) - return valid_remote_md5 + return valid_remote_md5 # always valid def _transceive(self, uri): @@ -329,8 +236,6 @@ class TransceiverServerHandler: ) rc = syncer(*myargs) if rc and not (self.download or self.remove): - # try with "SITE MD5 command first" - # proftpd's mod_md5 supports it remote_md5 = handler.get_md5(remote_path) rc = self.handler_verify_upload(mypath, uri, counter, maxcount, tries, remote_md5 = remote_md5) diff --git a/libraries/tests/server.py b/libraries/tests/server.py index 12e2af2c2..ccee704e3 100644 --- a/libraries/tests/server.py +++ b/libraries/tests/server.py @@ -18,7 +18,7 @@ class EntropyRepositoryTest(unittest.TestCase): sys.stdout.write("%s called\n" % (self,)) sys.stdout.flush() self.default_repo = "foo" - etpConst['officialserverrepositoryid'] = self.default_repo + etpConst['defaultserverrepositoryid'] = self.default_repo etpConst['uid'] = 0 self.SystemSettings = SystemSettings()