Entropy/Server Interface:
- complete code migration from activatorTools.py to entropy.py let the testing begins git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1566 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
@@ -7,12 +7,10 @@
|
||||
#
|
||||
# example:
|
||||
# mirror-upload|ftp://username:password@address
|
||||
# mirror-download|ftp://address
|
||||
#
|
||||
# note: the packages/%ARCH% part is added by Entropy
|
||||
#
|
||||
mirror-upload|ftp://username:password@address
|
||||
mirror-download|ftp://address
|
||||
|
||||
# Database format:
|
||||
# bz2 or gz
|
||||
|
||||
+10
-12
@@ -2766,18 +2766,16 @@ class etpDatabase:
|
||||
self.cursor.execute('SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies WHERE dependencies.idpackage = (?) AND dependenciesreference.iddependency = dependencies.iddependency', (idpackage,))
|
||||
return set(self.cursor.fetchall())
|
||||
|
||||
def listBranchPackagesTbz2(self, branch):
|
||||
result = set()
|
||||
pkglist = self.listBranchPackages(branch)
|
||||
for pkg in pkglist:
|
||||
idpackage = pkg[1]
|
||||
url = self.retrieveDownloadURL(idpackage)
|
||||
if url:
|
||||
result.add(os.path.basename(url))
|
||||
if (result):
|
||||
result = list(result)
|
||||
result.sort()
|
||||
return result
|
||||
def listBranchPackagesTbz2(self, branch, do_sort = True):
|
||||
self.cursor.execute('SELECT extrainfo.download FROM baseinfo,extrainfo WHERE baseinfo.branch = (?) AND baseinfo.idpackage = extrainfo.idpackage', (branch,))
|
||||
result = self.fetchall2set(self.cursor.fetchall())
|
||||
sorted_result = []
|
||||
for package in result:
|
||||
if package:
|
||||
sorted_result.append(os.path.basename(package))
|
||||
if do_sort:
|
||||
sorted_result.sort()
|
||||
return sorted_result
|
||||
|
||||
def listBranchPackages(self, branch):
|
||||
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE branch = (?)', (branch,))
|
||||
|
||||
+261
-57
@@ -10778,7 +10778,7 @@ class ServerInterface(TextInterface):
|
||||
|
||||
if default_repository == None:
|
||||
self.updateProgress(
|
||||
blue("Entropy Server Interface Instance on repository: %s" % (self.default_repository,) ),
|
||||
blue("Entropy Server Interface Instance on repository: %s" % (red(self.default_repository),) ),
|
||||
importance = 2,
|
||||
type = "info",
|
||||
header = red(" @@ ")
|
||||
@@ -10790,13 +10790,16 @@ class ServerInterface(TextInterface):
|
||||
def setup_services(self):
|
||||
self.setup_entropy_settings()
|
||||
self.ClientService = EquoInterface(indexing = self.indexing, xcache = self.xcache, repo_validation = False, noclientdb = 1)
|
||||
self.backup_entropy_settings()
|
||||
self.ClientService.FtpInterface = self.FtpInterface
|
||||
self.databaseTools = self.ClientService.databaseTools
|
||||
self.entropyTools = self.ClientService.entropyTools
|
||||
self.dumpTools = self.ClientService.dumpTools
|
||||
self.backup_entropy_settings()
|
||||
self.SpmService = self.ClientService.Spm()
|
||||
|
||||
self.MirrorsService = ServerMirrorsInterface(self)
|
||||
self.MirrorsService.FtpInterface = self.FtpInterface
|
||||
self.MirrorsService.rssFeed = self.rssFeed
|
||||
|
||||
def setup_entropy_settings(self):
|
||||
self.settings_to_backup.extend([
|
||||
@@ -10805,7 +10808,6 @@ class ServerInterface(TextInterface):
|
||||
'officialrepositoryid',
|
||||
'dbrepodir',
|
||||
'activatoruploaduris',
|
||||
'activatordownloaduris',
|
||||
'binaryurirelativepath',
|
||||
'etpurirelativepath',
|
||||
'handlers'
|
||||
@@ -11390,13 +11392,14 @@ class ServerInterface(TextInterface):
|
||||
dbconn.addTreeUpdatesActions(treeupdates_actions)
|
||||
|
||||
# now fill the database
|
||||
pkgbranches = list(self.list_all_branches())
|
||||
pkgbranches = etpConst['branches']
|
||||
|
||||
for mybranch in pkgbranches:
|
||||
|
||||
pkglist = os.listdir(os.path.join(etpConst['packagesserverbindir'],mybranch))
|
||||
pkg_branch_dir = os.path.join(etpConst['packagesserverbindir'],mybranch)
|
||||
pkglist = os.listdir(pkg_branch_dir)
|
||||
# filter .md5 and .expired packages
|
||||
pkglist = [x for x in pkglist if x[-5:] == etpConst['packagesext'] and not os.path.isfile(etpConst['packagesserverbindir']+"/"+mybranch+"/"+x+etpConst['packagesexpirationfileext'])]
|
||||
pkglist = [x for x in pkglist if x[-5:] == etpConst['packagesext'] and not os.path.isfile(os.path.join(pkg_branch_dir,x+etpConst['packagesexpirationfileext']))]
|
||||
|
||||
if not pkglist:
|
||||
continue
|
||||
@@ -12012,32 +12015,21 @@ class ServerInterface(TextInterface):
|
||||
|
||||
class ServerMirrorsInterface:
|
||||
|
||||
def __init__(self, ServerInstance,
|
||||
upload_mirrors = etpConst['activatoruploaduris'],
|
||||
download_mirrors = etpConst['activatordownloaduris']):
|
||||
import entropyTools, dumpTools
|
||||
def __init__(self, ServerInstance, mirrors = etpConst['activatoruploaduris']):
|
||||
|
||||
if not isinstance(ServerInstance,ServerInterface):
|
||||
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid ServerInterface instance is needed")
|
||||
|
||||
self.Entropy = ServerInterface
|
||||
self.entropyTools = self.Entropy.entropyTools
|
||||
self.dumpTools = self.Entropy.dumpTools
|
||||
self.upload_mirrors = upload_mirrors[:]
|
||||
self.download_mirrors = download_mirrors[:]
|
||||
self.FtpInterface = self.Entropy.FtpInterface
|
||||
self.rssFeed = self.Entropy.rssFeed
|
||||
self.Mirrors = mirrors[:]
|
||||
self.repository_directory = etpConst['etpdatabasedir']
|
||||
|
||||
|
||||
def set_upload_mirrors(self, mirrors):
|
||||
def set_mirrors(self, mirrors):
|
||||
if not isinstance(mirrors,list):
|
||||
raise exceptionTools.InvalidDataType("InvalidDataType: set_upload_mirrors needs a list type")
|
||||
self.upload_mirrors = mirrors[:]
|
||||
|
||||
def set_download_mirrors(self, mirrors):
|
||||
if not isinstance(mirrors,list):
|
||||
raise exceptionTools.InvalidDataType("InvalidDataType: set_download_mirrors needs a list type")
|
||||
self.download_mirrors = mirrors[:]
|
||||
self.Mirrors = mirrors[:]
|
||||
|
||||
def lock_mirrors(self, lock = True, mirrors = []):
|
||||
|
||||
@@ -12344,7 +12336,7 @@ class ServerMirrorsInterface:
|
||||
def get_remote_databases_status(self):
|
||||
|
||||
data = []
|
||||
for uri in etpConst['activatoruploaduris']:
|
||||
for uri in self.Mirrors:
|
||||
|
||||
ftp = self.FtpInterface(uri, self.Entropy)
|
||||
ftp.setCWD(etpConst['etpurirelativepath'])
|
||||
@@ -12389,7 +12381,7 @@ class ServerMirrorsInterface:
|
||||
def get_mirrors_lock(self):
|
||||
|
||||
dbstatus = []
|
||||
for uri in etpConst['activatoruploaduris']:
|
||||
for uri in self.Mirrors:
|
||||
data = [uri,False,False]
|
||||
ftp = FtpInterface(uri, self.Entropy)
|
||||
ftp.setCWD(etpConst['etpurirelativepath'])
|
||||
@@ -12535,7 +12527,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
return data, critical
|
||||
|
||||
class FileTransmitter:
|
||||
class FileTransceiver:
|
||||
|
||||
def __init__( self,
|
||||
ftp_interface,
|
||||
@@ -12543,6 +12535,7 @@ class ServerMirrorsInterface:
|
||||
uris,
|
||||
files_to_upload,
|
||||
download = False,
|
||||
remove = False,
|
||||
ftp_basedir = None,
|
||||
local_basedir = None,
|
||||
critical_files = [],
|
||||
@@ -12563,16 +12556,22 @@ class ServerMirrorsInterface:
|
||||
self.myfiles = [x for x in files_to_upload]
|
||||
self.myfiles.sort()
|
||||
self.download = download
|
||||
self.remove = remove
|
||||
self.use_handlers = use_handlers
|
||||
if self.remove:
|
||||
self.download = False
|
||||
self.use_handlers = False
|
||||
if not ftp_basedir:
|
||||
self.ftp_basedir = str(etpConst['etpurirelativepath']) # default to database directory
|
||||
# default to database directory
|
||||
self.ftp_basedir = str(etpConst['etpurirelativepath'])
|
||||
else:
|
||||
self.ftp_basedir = str(ftp_basedir)
|
||||
if not local_basedir:
|
||||
self.local_basedir = os.path.dirname(etpConst['etpdatabasefilepath']) # default to database directory
|
||||
# default to database directory
|
||||
self.local_basedir = os.path.dirname(etpConst['etpdatabasefilepath'])
|
||||
else:
|
||||
self.local_basedir = str(local_basedir)
|
||||
self.critical_files = critical_files
|
||||
self.use_handlers = use_handlers
|
||||
self.handlers_data = handlers_data.copy()
|
||||
|
||||
def handler_verify_upload(self, local_filepath, uri, ftp_connection, counter, maxcount):
|
||||
@@ -12595,7 +12594,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
|
||||
checksum = self.Entropy.ClientService.get_remote_package_checksum(
|
||||
self.Entropy.entropyTools.extractFTPHostFromUri(uri),
|
||||
self.entropyTools.extractFTPHostFromUri(uri),
|
||||
os.path.basename(local_filepath),
|
||||
self.handlers_data['branch']
|
||||
)
|
||||
@@ -12637,7 +12636,7 @@ class ServerMirrorsInterface:
|
||||
return False
|
||||
elif len(checksum) == 32:
|
||||
# valid? checking
|
||||
ckres = self.Entropy.entropyTools.compareMd5(local_filepath,checksum)
|
||||
ckres = self.entropyTools.compareMd5(local_filepath,checksum)
|
||||
if ckres:
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|mirror:%s|%s|#%s|(%s/%s)] digest verification: %s: %s" % (
|
||||
@@ -12701,10 +12700,12 @@ class ServerMirrorsInterface:
|
||||
action = 'upload'
|
||||
if self.download:
|
||||
action = 'download'
|
||||
elif self.remove:
|
||||
action = 'remove'
|
||||
|
||||
for uri in self.uris:
|
||||
|
||||
crippled_uri = self.Entropy.entropyTools.extractFTPHostFromUri(uri)
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|mirror:%s|%s] connecting to mirror..." % (etpConst['officialrepositoryid'],crippled_uri,action,)),
|
||||
importance = 0,
|
||||
@@ -12731,6 +12732,8 @@ class ServerMirrorsInterface:
|
||||
if self.download:
|
||||
syncer = ftp.downloadFile
|
||||
myargs = [os.path.basename(mypath),self.local_basedir]
|
||||
elif self.remove:
|
||||
syncer = ftp.deleteFile
|
||||
|
||||
counter += 1
|
||||
tries = 0
|
||||
@@ -13027,7 +13030,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
if not pretend:
|
||||
# upload
|
||||
uploader = self.FileTransmitter(self.FtpInterface, self.Entropy, [uri], upload_data, critical_files = critical)
|
||||
uploader = self.FileTransceiver(self.FtpInterface, self.Entropy, [uri], upload_data, critical_files = critical)
|
||||
errors, m_fine_uris, m_broken_uris = uploader.go()
|
||||
if errors:
|
||||
my_fine_uris = [self.entropyTools.extractFTPHostFromUri(x) for x in m_fine_uris]
|
||||
@@ -13061,7 +13064,6 @@ class ServerMirrorsInterface:
|
||||
return upload_errors, broken_uris, fine_uris
|
||||
|
||||
|
||||
|
||||
def download_database(self, uris, lock_check = False, pretend = False):
|
||||
|
||||
import gzip
|
||||
@@ -13081,7 +13083,7 @@ class ServerMirrorsInterface:
|
||||
database_path = etpConst['etpdatabasefilepath']
|
||||
database_dir_path = os.path.dirname(etpConst['etpdatabasefilepath'])
|
||||
download_data, critical = self.get_files_to_sync(cmethod, download = True)
|
||||
mytmpdir = self.Entropy.entropyTools.getRandomTempFile()
|
||||
mytmpdir = self.entropyTools.getRandomTempFile()
|
||||
os.makedirs(mytmpdir)
|
||||
|
||||
self.Entropy.updateProgress(
|
||||
@@ -13112,7 +13114,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
if not pretend:
|
||||
# download
|
||||
downloader = self.FileTransmitter(self.FtpInterface, self.Entropy, [uri], download_data, download = True, local_basedir = mytmpdir, critical_files = critical)
|
||||
downloader = self.FileTransceiver(self.FtpInterface, self.Entropy, [uri], download_data, download = True, local_basedir = mytmpdir, critical_files = critical)
|
||||
errors, m_fine_uris, m_broken_uris = downloader.go()
|
||||
if errors:
|
||||
my_fine_uris = [self.entropyTools.extractFTPHostFromUri(x) for x in m_fine_uris]
|
||||
@@ -13255,9 +13257,6 @@ class ServerMirrorsInterface:
|
||||
return 0, set(), set()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def calculate_local_upload_files(self, branch):
|
||||
upload_files = 0
|
||||
upload_packages = set()
|
||||
@@ -13325,7 +13324,7 @@ class ServerMirrorsInterface:
|
||||
# show stats
|
||||
for itemdata in upload:
|
||||
package = darkgreen(os.path.basename(itemdata[0]))
|
||||
size = blue(self.Entropy.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s|%s] %s [%s]" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
@@ -13340,7 +13339,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
for itemdata in download:
|
||||
package = darkred(os.path.basename(itemdata[0]))
|
||||
size = blue(self.Entropy.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s|%s] %s [%s]" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
@@ -13356,7 +13355,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
for itemdata in copy:
|
||||
package = darkblue(os.path.basename(itemdata[0]))
|
||||
size = blue(self.Entropy.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s|%s] %s [%s]" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
@@ -13372,7 +13371,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
for itemdata in removal:
|
||||
package = brown(os.path.basename(itemdata[0]))
|
||||
size = blue(self.Entropy.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
size = blue(self.entropyTools.bytesIntoHuman(itemdata[1]))
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s|%s] %s [%s]" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
@@ -13429,7 +13428,7 @@ class ServerMirrorsInterface:
|
||||
etpConst['officialrepositoryid'],
|
||||
branch,
|
||||
blue("total removal size"),
|
||||
self.Entropy.entropyTools.bytesIntoHuman(metainfo['removal']),
|
||||
self.entropyTools.bytesIntoHuman(metainfo['removal']),
|
||||
)
|
||||
),
|
||||
importance = 0,
|
||||
@@ -13442,7 +13441,7 @@ class ServerMirrorsInterface:
|
||||
etpConst['officialrepositoryid'],
|
||||
branch,
|
||||
bold("total upload size"),
|
||||
self.Entropy.entropyTools.bytesIntoHuman(metainfo['upload']),
|
||||
self.entropyTools.bytesIntoHuman(metainfo['upload']),
|
||||
)
|
||||
),
|
||||
importance = 0,
|
||||
@@ -13454,7 +13453,7 @@ class ServerMirrorsInterface:
|
||||
etpConst['officialrepositoryid'],
|
||||
branch,
|
||||
darkred("total download size"),
|
||||
self.Entropy.entropyTools.bytesIntoHuman(metainfo['download']),
|
||||
self.entropyTools.bytesIntoHuman(metainfo['download']),
|
||||
)
|
||||
),
|
||||
importance = 0,
|
||||
@@ -13500,7 +13499,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
def calculate_packages_to_sync(self, uri, branch):
|
||||
|
||||
crippled_uri = self.Entropy.entropyTools.extractFTPHostFromUri(uri)
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
upload_files, upload_packages = self.calculate_local_upload_files(branch)
|
||||
local_files, local_packages = self.calculate_local_package_files(branch)
|
||||
self._show_local_sync_stats(crippled_uri, branch, upload_files, local_files)
|
||||
@@ -13688,7 +13687,7 @@ class ServerMirrorsInterface:
|
||||
etpConst['officialrepositoryid'],
|
||||
branch,
|
||||
remove_filename,
|
||||
self.Entropy.entropyTools.bytesIntoHuman(itemdata[1]),
|
||||
self.entropyTools.bytesIntoHuman(itemdata[1]),
|
||||
)
|
||||
),
|
||||
importance = 0,
|
||||
@@ -13749,18 +13748,18 @@ class ServerMirrorsInterface:
|
||||
|
||||
def _sync_run_upload_queue(self, uri, upload_queue, branch):
|
||||
|
||||
crippled_uri = self.Entropy.entropyTools.extractFTPHostFromUri(uri)
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
myqueue = [x[0] for x in upload_queue]
|
||||
for itemdata in upload_queue:
|
||||
x = itemdata[0]
|
||||
hash_file = x+etpConst['packageshashfileext']
|
||||
if not os.path.isfile(hash_file):
|
||||
self.Entropy.entropyTools.createHashFile(x)
|
||||
self.entropyTools.createHashFile(x)
|
||||
myqueue.append(hash_file)
|
||||
myqueue.append(x)
|
||||
|
||||
ftp_basedir = os.path.join(etpConst['binaryurirelativepath'],branch)
|
||||
uploader = self.FileTransmitter( self.FtpInterface,
|
||||
uploader = self.FileTransceiver( self.FtpInterface,
|
||||
self.Entropy,
|
||||
[uri],
|
||||
myqueue,
|
||||
@@ -13771,7 +13770,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
errors, m_fine_uris, m_broken_uris = uploader.go()
|
||||
if errors:
|
||||
my_broken_uris = [(self.Entropy.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
|
||||
my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
|
||||
reason = my_broken_uris[0][1]
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s] upload errors: %s, reason: %s" % (
|
||||
@@ -13803,7 +13802,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
def _sync_run_download_queue(self, uri, download_queue, branch):
|
||||
|
||||
crippled_uri = self.Entropy.entropyTools.extractFTPHostFromUri(uri)
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
myqueue = [x[0] for x in download_queue]
|
||||
for itemdata in download_queue:
|
||||
x = itemdata[0]
|
||||
@@ -13813,7 +13812,7 @@ class ServerMirrorsInterface:
|
||||
|
||||
ftp_basedir = os.path.join(etpConst['binaryurirelativepath'],branch)
|
||||
local_basedir = os.path.join(etpConst['packagesserverbindir'],branch)
|
||||
downloader = self.FileTransmitter( self.FtpInterface,
|
||||
downloader = self.FileTransceiver( self.FtpInterface,
|
||||
self.Entropy,
|
||||
[uri],
|
||||
myqueue,
|
||||
@@ -13826,7 +13825,7 @@ class ServerMirrorsInterface:
|
||||
)
|
||||
errors, m_fine_uris, m_broken_uris = downloader.go()
|
||||
if errors:
|
||||
my_broken_uris = [(self.Entropy.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
|
||||
my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
|
||||
reason = my_broken_uris[0][1]
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s] download errors: %s, reason: %s" % (
|
||||
@@ -13866,7 +13865,7 @@ class ServerMirrorsInterface:
|
||||
back = True
|
||||
)
|
||||
|
||||
pkgbranches = list(self.Entropy.list_all_branches())
|
||||
pkgbranches = etpConst['branches']
|
||||
successfull_mirrors = set()
|
||||
broken_mirrors = set()
|
||||
check_data = ()
|
||||
@@ -13874,9 +13873,9 @@ class ServerMirrorsInterface:
|
||||
mirror_errors = False
|
||||
mirrors_errors = False
|
||||
|
||||
for uri in etpConst['activatoruploaduris']:
|
||||
for uri in self.Mirrors:
|
||||
|
||||
crippled_uri = self.Entropy.entropyTools.extractFTPHostFromUri(uri)
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
mirror_errors = False
|
||||
|
||||
for mybranch in pkgbranches:
|
||||
@@ -14040,3 +14039,208 @@ class ServerMirrorsInterface:
|
||||
return mirrors_tainted, mirrors_errors, successfull_mirrors, broken_mirrors, check_data
|
||||
|
||||
|
||||
def is_package_expired(self, package_file, branch):
|
||||
pkg_path = os.path.join(etpConst['packagesserverbindir'],branch,package_file)
|
||||
pkg_path += etpConst['packagesexpirationfileext']
|
||||
if not os.path.isfile(pkg_path):
|
||||
return False
|
||||
mtime = self.entropyTools.getFileUnixMtime(pkg_path)
|
||||
delta = int(etpConst['packagesexpirationdays'])*24*3600
|
||||
currmtime = time.time()
|
||||
if currmtime - mtime > delta:
|
||||
return True
|
||||
return False
|
||||
|
||||
def create_expiration_file(self, package_file, branch):
|
||||
pkg_path = os.path.join(etpConst['packagesserverbindir'],branch,package_file)
|
||||
pkg_path += etpConst['packagesexpirationfileext']
|
||||
f = open(pkg_path,"w")
|
||||
f.flush()
|
||||
f.close()
|
||||
|
||||
|
||||
def collect_expiring_packages(self, branch):
|
||||
dbconn = self.Entropy.openServerDatabase(just_reading = True)
|
||||
database_bins = set(dbconn.listBranchPackagesTbz2(branch, do_sort = False))
|
||||
bins_dir = os.path.join(etpConst['packagesserverbindir'],branch)
|
||||
repo_bins = []
|
||||
if os.path.isdir(bins_dir):
|
||||
repo_bins = os.listdir(bins_dir)
|
||||
repo_bins = set([x for x in repo_bins if x.endswith(etpConst['packagesext'])])
|
||||
repo_bins -= database_bins
|
||||
return repo_bins
|
||||
|
||||
|
||||
|
||||
def tidy_mirrors(self, ask = True, pretend = False):
|
||||
|
||||
pkgbranches = etpConst['branches']
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branches:%s] collecting expired packages" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
blue(str(pkgbranches)),
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
|
||||
branch_data = {}
|
||||
errors = False
|
||||
|
||||
for mybranch in pkgbranches:
|
||||
|
||||
branch_data[mybranch] = {}
|
||||
branch_data[mybranch]['errors'] = False
|
||||
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branch:%s] collecting expired packages in the selected branches" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
|
||||
# collect removed packages
|
||||
expiring_packages = self.collect_expiring_packages(mybranch)
|
||||
|
||||
removal = []
|
||||
for package in expiring_packages:
|
||||
expired = self.is_package_expired(package, mybranch)
|
||||
if expired:
|
||||
removal.append(package)
|
||||
else:
|
||||
self.create_expiration_file(package, mybranch)
|
||||
|
||||
# fill returning data
|
||||
branch_data[mybranch]['removal'] = removal[:]
|
||||
|
||||
if not removal:
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branch:%s] nothing to remove on this branch" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
continue
|
||||
else:
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branch:%s] these are the expired packages:" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
for package in removal:
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|branch:%s] remove: %s" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
package,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = brown(" # ")
|
||||
)
|
||||
|
||||
if pretend:
|
||||
continue
|
||||
|
||||
if ask:
|
||||
rc = self.Entropy.askQuestion("Would you like to continue ?")
|
||||
if rc == "No":
|
||||
continue
|
||||
|
||||
myqueue = []
|
||||
for package in removal:
|
||||
myqueue.append(package+etpConst['packageshashfileext'])
|
||||
myqueue.append(package)
|
||||
ftp_basedir = os.path.join(etpConst['binaryurirelativepath'],mybranch)
|
||||
for uri in self.Mirrors:
|
||||
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branch:%s] removing packages remotely..." % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
|
||||
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
|
||||
destroyer = self.FileTransceiver(
|
||||
self.FtpInterface,
|
||||
self.Entropy,
|
||||
[uri],
|
||||
myqueue,
|
||||
critical_files = [],
|
||||
ftp_basedir = ftp_basedir,
|
||||
remove = True
|
||||
)
|
||||
errors, m_fine_uris, m_broken_uris = destroyer.go()
|
||||
if errors:
|
||||
my_broken_uris = [(self.entropyTools.extractFTPHostFromUri(x[0]),x[1]) for x in m_broken_uris]
|
||||
reason = my_broken_uris[0][1]
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|sync|branch:%s] remove errors: %s, reason: %s" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
crippled_uri,
|
||||
reason,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "warning",
|
||||
header = brown(" !!! ")
|
||||
)
|
||||
branch_data[mybranch]['errors'] = True
|
||||
errors = True
|
||||
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|tidy|branch:%s] removing packages locally..." % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = darkgreen(" * ")
|
||||
)
|
||||
|
||||
branch_data[mybranch]['removed'] = set()
|
||||
for package in removal:
|
||||
package_path = os.path.join(etpConst['packagesserverbindir'],mybranch,package)
|
||||
package_path_hash = package_path+etpConst['packageshashfileext']
|
||||
package_path_expired = package_path+etpConst['packagesexpirationfileext']
|
||||
for myfile in [package_path_hash,package_path,package_path_expired]:
|
||||
if os.path.isfile(myfile):
|
||||
self.Entropy.updateProgress(
|
||||
red("[repo:%s|branch:%s] removing: %s" % (
|
||||
etpConst['officialrepositoryid'],
|
||||
mybranch,
|
||||
blue(myfile),
|
||||
)
|
||||
),
|
||||
importance = 1,
|
||||
type = "info",
|
||||
header = brown(" @@ ")
|
||||
)
|
||||
os.remove(myfile)
|
||||
branch_data[mybranch]['removed'].add(myfile)
|
||||
|
||||
|
||||
return errors, branch_data
|
||||
|
||||
@@ -531,7 +531,6 @@ def const_defaultSettings(rootdir):
|
||||
'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file
|
||||
'socketconf': ETP_CONF_DIR+"/socket.conf", # socket.conf file
|
||||
'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf)
|
||||
'activatordownloaduris': [], # list of URIs that activator can use to fetch data
|
||||
'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository.
|
||||
'packagesrelativepath': "packages/"+ETP_ARCH_CONST+"/", # user by client interfaces
|
||||
'etpurirelativepath': "database/"+ETP_ARCH_CONST+"/", # database relative path
|
||||
@@ -775,11 +774,6 @@ def const_setupServerClientRepository():
|
||||
etpConst['dbrepodir'],
|
||||
etpConst['officialrepositoryid']
|
||||
)
|
||||
for x in range(len(etpConst['activatordownloaduris'])):
|
||||
etpConst['activatordownloaduris'][x] = etpConst['activatordownloaduris'][x].replace(
|
||||
etpConst['dbrepodir'],
|
||||
etpConst['officialrepositoryid']
|
||||
)
|
||||
|
||||
# align etpConst['binaryurirelativepath'] and etpConst['etpurirelativepath'] with etpConst['product']
|
||||
etpConst['binaryurirelativepath'] = etpConst['product']+"/"+etpConst['officialrepositoryid']+"/"+etpConst['binaryurirelativepath']
|
||||
@@ -1199,11 +1193,6 @@ def const_readActivatorSettings():
|
||||
if uri.endswith("/"):
|
||||
uri = uri[:-1]
|
||||
etpConst['activatoruploaduris'].append(uri+"/"+etpConst['dbrepodir'])
|
||||
elif line.startswith("mirror-download|") and (len(line.split("mirror-download|")) == 2):
|
||||
uri = line.split("mirror-download|")[1]
|
||||
if uri.endswith("/"):
|
||||
uri = uri[:-1]
|
||||
etpConst['activatordownloaduris'].append(uri+"/"+etpConst['dbrepodir'])
|
||||
elif line.startswith("database-format|") and (len(line.split("database-format|")) == 2):
|
||||
format = line.split("database-format|")[1]
|
||||
if format in etpConst['etpdatabasesupportedcformats']:
|
||||
|
||||
+26
-117
@@ -1,7 +1,7 @@
|
||||
#!/usr/bin/python
|
||||
'''
|
||||
# DESCRIPTION:
|
||||
# generic tools for enzyme application
|
||||
# activator textual interface
|
||||
|
||||
Copyright (C) 2007-2008 Fabio Erculiani
|
||||
|
||||
@@ -20,143 +20,49 @@
|
||||
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
'''
|
||||
|
||||
import time
|
||||
from entropyConstants import *
|
||||
from outputTools import *
|
||||
from entropy import FtpInterface, ServerInterface
|
||||
Entropy = ServerInterface(noclientdb = 2)
|
||||
from entropy import ServerInterface
|
||||
Entropy = ServerInterface()
|
||||
|
||||
def sync(options, justTidy = False):
|
||||
|
||||
activatorRequestNoAsk = False
|
||||
do_noask = False
|
||||
myopts = []
|
||||
for i in options:
|
||||
if ( i == "--noask" ):
|
||||
activatorRequestNoAsk = True
|
||||
for opt in options:
|
||||
if opt == "--noask":
|
||||
do_noask = True
|
||||
else:
|
||||
myopts.append(i)
|
||||
myopts.append(opt)
|
||||
options = myopts
|
||||
|
||||
print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ..."))
|
||||
|
||||
if (not justTidy):
|
||||
# firstly sync the packages
|
||||
if (activatorRequestNoAsk):
|
||||
rc = packages(["sync"])
|
||||
else:
|
||||
ask = etpUi['ask']
|
||||
etpUi['ask'] = True
|
||||
rc = packages(["sync"])
|
||||
etpUi['ask'] = ask
|
||||
# then sync the database, if the packages sync completed successfully
|
||||
if rc:
|
||||
if (not activatorRequestNoAsk) and etpConst['rss-feed']:
|
||||
if not justTidy:
|
||||
|
||||
mirrors_tainted, mirrors_errors, successfull_mirrors, broken_mirrors, check_data = Entropy.MirrorsService.sync_packages(ask = not do_noask, pretend = etpUi['pretend'])
|
||||
if not mirrors_errors:
|
||||
|
||||
if (not do_noask) and etpConst['rss-feed']:
|
||||
etpRSSMessages['commitmessage'] = readtext(">> Please insert a commit message: ")
|
||||
elif etpConst['rss-feed']:
|
||||
etpRSSMessages['commitmessage'] = "Autodriven Update"
|
||||
# if packages are ok, we can sync the database
|
||||
database(["sync"])
|
||||
if not activatorRequestNoAsk:
|
||||
# ask question
|
||||
rc = Entropy.askQuestion(" Should I continue with the tidy procedure ?")
|
||||
rc = database(["sync"])
|
||||
if not rc and not do_noask:
|
||||
rc = Entropy.askQuestion("Should I continue with the tidy procedure ?")
|
||||
if rc == "No":
|
||||
sys.exit(0)
|
||||
elif rc:
|
||||
print_error(darkred(" !!! ")+red("Aborting !"))
|
||||
sys.exit(rc)
|
||||
|
||||
print_info(green(" * ")+red("Starting to collect packages that would be removed from the repository ..."))
|
||||
for mybranch in etpConst['branches']:
|
||||
|
||||
print_info(red(" * ")+blue("Switching to branch: ")+bold(mybranch))
|
||||
|
||||
# now it's time to do some tidy
|
||||
# collect all the binaries in the database
|
||||
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
|
||||
dbBinaries = dbconn.listBranchPackagesTbz2(mybranch)
|
||||
dbconn.closeDB()
|
||||
|
||||
# list packages in the packages directory
|
||||
repoBinaries = os.listdir(etpConst['packagesbindir']+"/"+mybranch)
|
||||
|
||||
removeList = []
|
||||
# select packages
|
||||
for repoBin in repoBinaries:
|
||||
if (repoBin.endswith(".tbz2")):
|
||||
if repoBin not in dbBinaries:
|
||||
# check if it's expired
|
||||
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+repoBin
|
||||
if os.path.isfile(filepath+etpConst['packagesexpirationfileext']):
|
||||
# check mtime
|
||||
mtime = Entropy.entropyTools.getFileUnixMtime(filepath+etpConst['packagesexpirationfileext'])
|
||||
delta = int(etpConst['packagesexpirationdays'])*24*3600
|
||||
currmtime = time.time()
|
||||
if currmtime - mtime > delta: # if it's expired
|
||||
removeList.append(repoBin)
|
||||
else:
|
||||
# create expiration file
|
||||
f = open(filepath+etpConst['packagesexpirationfileext'],"w")
|
||||
f.write("\n")
|
||||
f.flush()
|
||||
f.close()
|
||||
# not expired.
|
||||
|
||||
if (not removeList):
|
||||
print_info(green(" * ")+red("No packages to remove from the mirrors."))
|
||||
print_info(green(" * ")+red("Syncronization across mirrors completed."))
|
||||
continue
|
||||
|
||||
print_info(green(" * ")+red("This is the list of files that would be removed from the mirrors: "))
|
||||
for xfile in removeList:
|
||||
print_info(green("\t* ")+brown(xfile))
|
||||
|
||||
# ask question
|
||||
if (not activatorRequestNoAsk):
|
||||
rc = Entropy.askQuestion(" Would you like to continue ?")
|
||||
if rc == "No":
|
||||
sys.exit(0)
|
||||
|
||||
# remove them!
|
||||
for uri in etpConst['activatoruploaduris']:
|
||||
print_info(green(" * ")+red("Connecting to: ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
||||
ftp = FtpInterface(uri, Entropy)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch)
|
||||
for xfile in removeList:
|
||||
|
||||
print_info(green(" * ")+red("Removing file: ")+bold(xfile), back = True)
|
||||
# remove remotely
|
||||
if (ftp.isFileAvailable(xfile)):
|
||||
rc = ftp.deleteFile(xfile)
|
||||
if (rc):
|
||||
print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
||||
else:
|
||||
print_warning(brown(" * ")+red("ATTENTION: remote file ")+bold(xfile)+red(" cannot be removed."))
|
||||
# checksum
|
||||
if (ftp.isFileAvailable(xfile+etpConst['packageshashfileext'])):
|
||||
rc = ftp.deleteFile(xfile+etpConst['packageshashfileext'])
|
||||
if (rc):
|
||||
print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
||||
else:
|
||||
print_warning(brown(" * ")+red("ATTENTION: remote checksum file ")+bold(xfile)+red(" cannot be removed."))
|
||||
# remove locally
|
||||
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile):
|
||||
print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
||||
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile)
|
||||
try:
|
||||
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packagesexpirationfileext'])
|
||||
except OSError:
|
||||
pass
|
||||
# checksum
|
||||
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext']):
|
||||
print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
||||
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext'])
|
||||
ftp.closeConnection()
|
||||
|
||||
print_info(green(" * ")+red("Syncronization across mirrors completed."))
|
||||
Entropy.MirrorsService.tidy_mirrors(ask = not do_noask, pretend = etpUi['pretend'])
|
||||
|
||||
|
||||
def packages(options):
|
||||
|
||||
myopts = options[1:]
|
||||
do_pkg_check = False
|
||||
for opt in myopts:
|
||||
for opt in options:
|
||||
if (opt == "--do-packages-check"):
|
||||
do_pkg_check = True
|
||||
|
||||
@@ -164,7 +70,10 @@ def packages(options):
|
||||
return
|
||||
|
||||
if options[0] == "sync":
|
||||
Entropy.MirrorsService.sync_packages(ask = etpUi['ask'], pretend = etpUi['pretend'], packages_check = do_pkg_check)
|
||||
return Entropy.MirrorsService.sync_packages( ask = etpUi['ask'],
|
||||
pretend = etpUi['pretend'],
|
||||
packages_check = do_pkg_check
|
||||
)
|
||||
|
||||
|
||||
def database(options):
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
|
||||
from entropyConstants import *
|
||||
from outputTools import *
|
||||
from entropy import ServerInterface, FtpInterface
|
||||
from entropy import ServerInterface
|
||||
Entropy = ServerInterface()
|
||||
|
||||
def inject(options):
|
||||
|
||||
Reference in New Issue
Block a user