sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[entropy.server] remove etpConst['officialserverrepositoryid'] and handlers support (there is ssh URI support now)

Browse Source
This commit is contained in:
Fabio Erculiani
2009-11-21 13:41:38 +01:00
parent 758890b06b
commit 94867af2fe
7 changed files with 92 additions and 234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conf/server.conf.example
+11 -8
View File
@@ -15,22 +15,25 @@
#
# syntax for officialserverrepositoryid:
# syntax for default-repository:
#
# officialserverrepositoryid: Repository identifier for the official main distribution repository
# officialserverrepositoryid|<repository identifier>
# default-repository: Currently working repository identifier (if you have
# more than one server-side repository, this indicates the one which
# Entropy Server is authorized to work on)
# default-repository|<repository identifier>
#
officialserverrepositoryid|sabayonlinux.org
default-repository|sabayonlinux.org
#
# syntax for repositories:
# syntax for repository:
#
# repository: server-side repositories configuration
# note: default repository setting is in repositories.conf (officialrepositoryid)
#| repository|repository_id|repository description|<ftp upload URI 1> <2>|<http handlers uri>|<remote server hostname>#<service port>,<ssl service port>
# Currently supported upload URI protocols are ftp:// and ssh://
# => repository|<repository_id>|<repository description>|<upload URI 1> <2>[|<EAPI3 remote server hostname>#<service port>,<ssl service port>]
# (EAPI3 part is optional)
#
# example:
#| repository|sabayonlinux.org|Sabayon Linux Official Repository|ftp://user:pass@111.111.111.111/|http://svn.sabayonlinux.org/entropy|svn.sabayonlinux.org#1026,1027
# => repository|myserverrepo|My Server Repository|ftp://user:pass@111.111.111.111/ ssh://username@host:~user/path:port|hostname.tld#1026,1027
#
# repository|
libraries/entropy/client/interfaces/qa.py
+1 -1
View File
@@ -50,7 +50,7 @@ class UGCErrorReportInterface(ErrorReportInterface):
self.__system_settings = SystemSettings()
if repository_id == None:
repository_id = etpConst['officialserverrepositoryid']
repository_id = etpConst['officialrepositoryid']
self.entropy = Client()
self.__repository_id = repository_id
libraries/entropy/const.py
+3 -14
View File
@@ -455,8 +455,7 @@ def const_default_settings(rootdir):
# and handle the removal through expiration (using creation date)
'expiration_based_scope': False,
# our official repository name
'officialserverrepositoryid': "sabayonlinux.org",
# our official repository name
'defaultserverrepositoryid': None,
'officialrepositoryid': "sabayonlinux.org",
'conntestlink': "http://www.sabayonlinux.org",
# tag to append to .tbz2 file before entropy database (must be 32bytes)
@@ -708,12 +707,6 @@ def const_default_settings(rootdir):
'ugc_accessfile': default_etp_ugc_confdir+"/access.xml",
'ugc_voterange': list(range(1, 6)),
# handler settings
'handlers': {
# md5sum handler,
'md5sum': "md5sum.php?arch="+etpSys['arch']+"&package=",
},
}
# set current nice level
@@ -1196,13 +1189,12 @@ def const_extract_srv_repo_params(repostring, product = None):
repoid = repostring.split("|")[1].strip()
repodesc = repostring.split("|")[2].strip()
repouris = repostring.split("|")[3].strip()
repohandlers = repostring.split("|")[4].strip()
service_url = None
eapi3_port = int(etpConst['socket_service']['port'])
eapi3_ssl_port = int(etpConst['socket_service']['ssl_port'])
if len(repostring.split("|")) > 5:
service_url = repostring.split("|")[5].strip()
if len(repostring.split("|")) > 4:
service_url = repostring.split("|")[4].strip()
eapi3_formatcolon = service_url.rfind("#")
if eapi3_formatcolon != -1:
@@ -1224,9 +1216,6 @@ def const_extract_srv_repo_params(repostring, product = None):
mydata['service_url'] = service_url
mydata['service_port'] = eapi3_port
mydata['ssl_service_port'] = eapi3_ssl_port
if repohandlers:
repohandlers = os.path.join(repohandlers, product, repoid, "handlers")
mydata['handler'] = repohandlers
uris = repouris.split()
for uri in uris:
mydata['mirrors'].append(uri)
libraries/entropy/server/interfaces/main.py
+72 -111
View File
@@ -358,7 +358,7 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin):
'repositories': etpConst['server_repositories'].copy(),
'sync_speed_limit': None,
'qa_langs': ["en_US", "C"],
'default_repository_id': etpConst['officialserverrepositoryid'],
'default_repository_id': etpConst['defaultserverrepositoryid'],
'packages_expiration_days': etpConst['packagesexpirationdays'],
'database_file_format': etpConst['etpdatabasefileformat'],
'disabled_eapis': set(),
@@ -382,16 +382,30 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin):
with open(etpConst['serverconf'], "r") as server_f:
serverconf = [x.strip() for x in server_f.readlines() if x.strip()]
default_repo_changed = False
for line in serverconf:
split_line = line.split("|")
split_line_len = len(split_line)
# TODO: remove this in future, supported for backward compat.
if (line.find("officialserverrepositoryid|") != -1) and \
(not line.startswith("#")) and (split_line_len == 2):
# TODO: added for backward and mixed compat.
if default_repo_changed:
continue
if not fake_instance:
data['default_repository_id'] = split_line[1].strip()
elif (line.find("default-repository|") != -1) and \
(not line.startswith("#")) and (split_line_len == 2):
if not fake_instance:
data['default_repository_id'] = split_line[1].strip()
default_repo_changed = True
elif line.startswith("expiration-days|") and (split_line_len == 2):
@@ -963,12 +977,12 @@ class Server(Singleton, TextInterface):
found = False
new_content = []
for line in content:
if line.strip().startswith("officialserverrepositoryid|"):
line = "officialserverrepositoryid|%s" % (repoid,)
if line.strip().startswith("default-repository|"):
line = "default-repository|%s" % (repoid,)
found = True
new_content.append(line)
if not found:
new_content.append("officialserverrepositoryid|%s" % (repoid,))
new_content.append("default-repository|%s" % (repoid,))
f_srv_t = open(etpConst['serverconf']+".save_default_repo_tmp", "w")
for line in new_content:
f_srv_t.write(line+"\n")
@@ -978,7 +992,7 @@ class Server(Singleton, TextInterface):
etpConst['serverconf'])
else:
f_srv = open(etpConst['serverconf'], "w")
f_srv.write("officialserverrepositoryid|%s\n" % (repoid,))
f_srv.write("default-repository|%s\n" % (repoid,))
f_srv.flush()
f_srv.close()
@@ -1144,7 +1158,7 @@ class Server(Singleton, TextInterface):
etpConst['server_repositories'][repoid] = repodata
if set_as_default:
etpConst['officialserverrepositoryid'] = repoid
etpConst['defaultserverrepositoryid'] = repoid
sys_set = SystemSettings()
sys_set.clear()
@@ -2038,7 +2052,6 @@ class Server(Singleton, TextInterface):
[uri],
downloader_queue,
critical_files = downloader_queue,
use_handlers = True,
txc_basedir = basedir,
local_basedir = tmp_down_dir,
download = True,
@@ -3639,50 +3652,6 @@ class Server(Singleton, TextInterface):
)
return idpackages, False
def get_remote_package_checksum(self, repo, filename, branch):
if sys.hexversion >= 0x3000000:
import urllib.request as urlmod
import urllib.error as urlmod_error
else:
import urllib2 as urlmod
import urllib2 as urlmod_error
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if 'handler' not in srv_set['repositories'][repo]:
return None
url = srv_set['repositories'][repo]['handler']
# does the package has "#" (== tag) ? hackish thing that works
filename = filename.replace("#", "%23")
# "+"
filename = filename.replace("+", "%2b")
request = os.path.join(url, etpConst['handlers']['md5sum'])
request += filename+"&branch="+branch
proxy_settings = self.SystemSettings['system']['proxy']
try:
mydict = {}
if proxy_settings['ftp']:
mydict['ftp'] = proxy_settings['ftp']
if proxy_settings['http']:
mydict['http'] = proxy_settings['http']
if mydict:
mydict['username'] = proxy_settings['username']
mydict['password'] = proxy_settings['password']
self.entropyTools.add_proxy_opener(urlmod, mydict)
else:
# unset
urlmod._opener = None
item = urlmod.urlopen(request)
result = item.readline().strip()
item.close()
del item
return result
except (urlmod_error.URLError, urlmod_error.HTTPError,):
# no HTTP support?
return None
def verify_remote_packages(self, packages, ask = True, repo = None):
if repo is None:
@@ -3758,79 +3727,71 @@ class Server(Singleton, TextInterface):
totalcounter = len(idpackages)
currentcounter = 0
for idpackage in idpackages:
currentcounter += 1
pkgfile = dbconn.retrieveDownloadURL(idpackage)
orig_branch = self.get_branch_from_download_relative_uri(
pkgfile)
txc = self.Transceiver(uri)
txc.set_verbosity(False)
with txc as handler:
self.updateProgress(
"[%s] %s: %s" % (
brown(crippled_uri),
blue(_("checking hash")),
darkgreen(pkgfile),
),
importance = 1,
type = "info",
header = blue(" @@ "),
back = True,
count = (currentcounter, totalcounter,)
)
for idpackage in idpackages:
checksum_ok = False
ck_remote = self.get_remote_package_checksum(repo,
os.path.basename(pkgfile), orig_branch)
if ck_remote is None:
self.updateProgress(
"[%s] %s: %s %s" % (
brown(crippled_uri),
blue(_("digest verification of")),
bold(pkgfile),
blue(_("not supported")),
),
importance = 1,
type = "info",
header = blue(" @@ "),
count = (currentcounter, totalcounter,)
)
elif len(ck_remote) == 32:
currentcounter += 1
pkgfile = dbconn.retrieveDownloadURL(idpackage)
pkghash = dbconn.retrieveDigest(idpackage)
if ck_remote == pkghash:
checksum_ok = True
else:
self.updateProgress(
"[%s] %s: %s %s" % (
"[%s] %s: %s" % (
brown(crippled_uri),
blue(_("digest verification of")),
bold(pkgfile),
blue(_("failed for unknown reasons")),
blue(_("checking hash")),
darkgreen(pkgfile),
),
importance = 1,
type = "info",
header = blue(" @@ "),
back = True,
count = (currentcounter, totalcounter,)
)
if checksum_ok:
match.add(idpackage)
else:
not_match.add(idpackage)
self.updateProgress(
"[%s] %s: %s %s" % (
brown(crippled_uri),
blue(_("package")),
bold(pkgfile),
red(_("NOT healthy")),
),
importance = 1,
type = "warning",
header = darkred(" !!! "),
count = (currentcounter, totalcounter,)
)
if crippled_uri not in broken_packages:
broken_packages[crippled_uri] = []
broken_packages[crippled_uri].append(pkgfile)
pkg_rel_path = '/'.join(pkgfile.split("/")[2:])
pkgfile = os.path.join(
self.get_remote_packages_relative_path(repo),
pkg_rel_path)
ck_remote = handler.get_md5(pkgfile)
if ck_remote is None:
self.updateProgress(
"[%s] %s: %s %s" % (
brown(crippled_uri),
blue(_("digest verification of")),
bold(pkgfile),
blue(_("not supported")),
),
importance = 1,
type = "info",
header = blue(" @@ "),
count = (currentcounter, totalcounter,)
)
continue
if ck_remote == pkghash:
match.add(idpackage)
else:
not_match.add(idpackage)
self.updateProgress(
"[%s] %s: %s %s" % (
brown(crippled_uri),
blue(_("package")),
bold(pkgfile),
red(_("NOT healthy")),
),
importance = 1,
type = "warning",
header = darkred(" !!! "),
count = (currentcounter, totalcounter,)
)
if crippled_uri not in broken_packages:
broken_packages[crippled_uri] = []
broken_packages[crippled_uri].append(pkgfile)
if broken_packages:
mytxt = blue("%s:") % (
libraries/entropy/server/interfaces/mirrors.py
+2 -2
View File
@@ -2640,7 +2640,7 @@ class Server:
uploader = self.TransceiverServerHandler(self.Entropy, [uri],
myqueue, critical_files = myqueue,
use_handlers = True, txc_basedir = remote_dir,
txc_basedir = remote_dir,
handlers_data = {'branch': branch }, repo = repo)
errors, m_fine_uris, m_broken_uris = uploader.go()
@@ -2695,7 +2695,7 @@ class Server:
self.Entropy.get_local_packages_directory(repo), branch)
downloader = self.TransceiverServerHandler(
self.Entropy, [uri], myqueue,
critical_files = myqueue, use_handlers = True,
critical_files = myqueue,
txc_basedir = remote_dir, local_basedir = local_basedir,
handlers_data = {'branch': branch }, download = True, repo = repo)
libraries/entropy/server/transceivers.py
+2 -97
View File
@@ -24,7 +24,7 @@ class TransceiverServerHandler:
def __init__(self, entropy_interface, uris, files_to_upload,
download = False, remove = False, txc_basedir = None,
local_basedir = None, critical_files = None, use_handlers = False,
local_basedir = None, critical_files = None,
handlers_data = None, repo = None):
if critical_files is None:
@@ -61,10 +61,8 @@ class TransceiverServerHandler:
self.repo = repo
if self.repo == None:
self.repo = self.Entropy.default_repository
self.use_handlers = use_handlers
if self.remove:
self.download = False
self.use_handlers = False
if not txc_basedir:
# default to database directory
@@ -162,98 +160,7 @@ class TransceiverServerHandler:
)
valid_remote_md5 = False
if not self.use_handlers:
# handlers usage is disabled
return valid_remote_md5 # always valid
checksum = self.Entropy.get_remote_package_checksum(
self.repo,
os.path.basename(local_filepath),
self.handlers_data['branch']
)
if checksum == None:
self.Entropy.updateProgress(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkred(_("not supported")),
),
importance = 0,
type = "info",
header = red(" @@ ")
)
return valid_remote_md5
elif isinstance(checksum, bool) and not checksum:
self.Entropy.updateProgress(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
bold(_("file not found")),
),
importance = 0,
type = "warning",
header = brown(" @@ ")
)
return False
elif is_valid_md5(checksum):
# valid? checking
ckres = compare_md5(local_filepath, checksum)
if ckres:
self.Entropy.updateProgress(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkgreen(_("so far, so good!")),
),
importance = 0,
type = "info",
header = red(" @@ ")
)
return True
else:
self.Entropy.updateProgress(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkred(_("invalid checksum")),
),
importance = 0,
type = "warning",
header = brown(" @@ ")
)
return False
else:
self.Entropy.updateProgress(
"[%s|#%s|(%s/%s)] %s: %s: %s" % (
blue(crippled_uri),
darkgreen(str(tries)),
blue(str(counter)),
bold(str(maxcount)),
blue(_("digest verification")),
os.path.basename(local_filepath),
darkred(_("unknown data returned")),
),
importance = 0,
type = "warning",
header = brown(" @@ ")
)
return valid_remote_md5
return valid_remote_md5 # always valid
def _transceive(self, uri):
@@ -329,8 +236,6 @@ class TransceiverServerHandler:
)
rc = syncer(*myargs)
if rc and not (self.download or self.remove):
# try with "SITE MD5 command first"
# proftpd's mod_md5 supports it
remote_md5 = handler.get_md5(remote_path)
rc = self.handler_verify_upload(mypath, uri,
counter, maxcount, tries, remote_md5 = remote_md5)
libraries/tests/server.py
+1 -1
View File
@@ -18,7 +18,7 @@ class EntropyRepositoryTest(unittest.TestCase):
sys.stdout.write("%s called\n" % (self,))
sys.stdout.flush()
self.default_repo = "foo"
etpConst['officialserverrepositoryid'] = self.default_repo
etpConst['defaultserverrepositoryid'] = self.default_repo
etpConst['uid'] = 0
self.SystemSettings = SystemSettings()