6b82b00341
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1035 cd1c1023-2f26-0410-ae45-c471fc1f0318
1356 lines
66 KiB
Python
1356 lines
66 KiB
Python
#!/usr/bin/python
|
|
'''
|
|
# DESCRIPTION:
|
|
# generic tools for enzyme application
|
|
|
|
Copyright (C) 2007-2008 Fabio Erculiani
|
|
|
|
This program is free software; you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation; either version 2 of the License, or
|
|
(at your option) any later version.
|
|
|
|
This program is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with this program; if not, write to the Free Software
|
|
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
'''
|
|
|
|
import time
|
|
import shutil
|
|
from entropyConstants import *
|
|
from serverConstants import *
|
|
from outputTools import *
|
|
from entropy import FtpInterface, EquoInterface, rssFeed
|
|
import exceptionTools
|
|
Entropy = EquoInterface(noclientdb = True)
|
|
|
|
# Logging initialization
|
|
import logTools
|
|
activatorLog = logTools.LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]")
|
|
|
|
|
|
def sync(options, justTidy = False):
|
|
|
|
activatorRequestNoAsk = False
|
|
myopts = []
|
|
for i in options:
|
|
if ( i == "--noask" ):
|
|
activatorRequestNoAsk = True
|
|
else:
|
|
myopts.append(i)
|
|
options = myopts
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: called with justTidy -> "+str(justTidy))
|
|
print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ..."))
|
|
|
|
if (not justTidy):
|
|
# firstly sync the packages
|
|
if (activatorRequestNoAsk):
|
|
rc = packages(["sync"])
|
|
else:
|
|
ask = etpUi['ask']
|
|
etpUi['ask'] = True
|
|
rc = packages(["sync"])
|
|
etpUi['ask'] = ask
|
|
# then sync the database, if the packages sync completed successfully
|
|
if rc:
|
|
if (not activatorRequestNoAsk) and etpConst['rss-feed']:
|
|
etpRSSMessages['commitmessage'] = readtext(">> Please insert a commit message: ")
|
|
elif etpConst['rss-feed']:
|
|
etpRSSMessages['commitmessage'] = "Autodriven Update"
|
|
# if packages are ok, we can sync the database
|
|
database(["sync"])
|
|
if (not activatorRequestNoAsk):
|
|
# ask question
|
|
rc = Entropy.askQuestion(" Should I continue with the tidy procedure ?")
|
|
if rc == "No":
|
|
sys.exit(0)
|
|
|
|
print_info(green(" * ")+red("Starting to collect packages that would be removed from the repository ..."), back = True)
|
|
|
|
|
|
for mybranch in etpConst['branches']:
|
|
|
|
print_info(red(" * ")+blue("Switching to branch: ")+bold(mybranch))
|
|
|
|
# now it's time to do some tidy
|
|
# collect all the binaries in the database
|
|
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
|
|
dbBinaries = dbconn.listBranchPackagesTbz2(mybranch)
|
|
dbconn.closeDB()
|
|
|
|
# list packages in the packages directory
|
|
repoBinaries = os.listdir(etpConst['packagesbindir']+"/"+mybranch)
|
|
|
|
removeList = []
|
|
# select packages
|
|
for repoBin in repoBinaries:
|
|
if (repoBin.endswith(".tbz2")):
|
|
if repoBin not in dbBinaries:
|
|
# check if it's expired
|
|
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+repoBin
|
|
if os.path.isfile(filepath+etpConst['packagesexpirationfileext']):
|
|
# check mtime
|
|
mtime = Entropy.entropyTools.getFileUnixMtime(filepath+etpConst['packagesexpirationfileext'])
|
|
delta = int(etpConst['packagesexpirationdays'])*24*3600
|
|
currmtime = time.time()
|
|
if currmtime - mtime > delta: # if it's expired
|
|
removeList.append(repoBin)
|
|
else:
|
|
# create expiration file
|
|
f = open(filepath+etpConst['packagesexpirationfileext'],"w")
|
|
f.write("\n")
|
|
f.flush()
|
|
f.close()
|
|
# not expired.
|
|
|
|
if (not removeList):
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: no packages to remove from the lirrors.")
|
|
print_info(green(" * ")+red("No packages to remove from the mirrors."))
|
|
print_info(green(" * ")+red("Syncronization across mirrors completed."))
|
|
continue
|
|
|
|
print_info(green(" * ")+red("This is the list of files that would be removed from the mirrors: "))
|
|
for xfile in removeList:
|
|
print_info(green("\t* ")+brown(xfile))
|
|
|
|
# ask question
|
|
if (not activatorRequestNoAsk):
|
|
rc = Entropy.askQuestion(" Would you like to continue ?")
|
|
if rc == "No":
|
|
sys.exit(0)
|
|
|
|
# remove them!
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: starting to remove packages from mirrors.")
|
|
for uri in etpConst['activatoruploaduris']:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: connecting to mirror "+Entropy.entropyTools.extractFTPHostFromUri(uri))
|
|
print_info(green(" * ")+red("Connecting to: ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch)
|
|
for xfile in removeList:
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (remote) file "+xfile)
|
|
print_info(green(" * ")+red("Removing file: ")+bold(xfile), back = True)
|
|
# remove remotely
|
|
if (ftp.isFileAvailable(xfile)):
|
|
rc = ftp.deleteFile(xfile)
|
|
if (rc):
|
|
print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
else:
|
|
print_warning(brown(" * ")+red("ATTENTION: remote file ")+bold(xfile)+red(" cannot be removed."))
|
|
# checksum
|
|
if (ftp.isFileAvailable(xfile+etpConst['packageshashfileext'])):
|
|
rc = ftp.deleteFile(xfile+etpConst['packageshashfileext'])
|
|
if (rc):
|
|
print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
else:
|
|
print_warning(brown(" * ")+red("ATTENTION: remote checksum file ")+bold(xfile)+red(" cannot be removed."))
|
|
# remove locally
|
|
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile):
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile)
|
|
print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
|
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile)
|
|
try:
|
|
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packagesexpirationfileext'])
|
|
except OSError:
|
|
pass
|
|
# checksum
|
|
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext']):
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile+etpConst['packageshashfileext'])
|
|
print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
|
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext'])
|
|
ftp.closeConnection()
|
|
|
|
print_info(green(" * ")+red("Syncronization across mirrors completed."))
|
|
|
|
|
|
def packages(options):
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called with options -> "+str(options))
|
|
|
|
# Options available for all the packages submodules
|
|
myopts = options[1:]
|
|
activatorRequestPackagesCheck = False
|
|
for opt in myopts:
|
|
if (opt == "--do-packages-check"):
|
|
activatorRequestPackagesCheck = True
|
|
|
|
if not options:
|
|
return
|
|
|
|
if (options[0] == "sync"):
|
|
|
|
print_info(green(" * ")+red("Starting ")+bold("binary")+brown(" packages")+red(" syncronization across servers ..."))
|
|
|
|
totalUris = len(etpConst['activatoruploaduris'])
|
|
currentUri = 0
|
|
totalSuccessfulUri = 0
|
|
mirrorsTainted = False
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called sync.")
|
|
|
|
for uri in etpConst['activatoruploaduris']:
|
|
|
|
uriSuccessfulSync = 0
|
|
currentUri += 1
|
|
try:
|
|
print_info(green(" * ")+brown("Working on ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)+red(" mirror.")))
|
|
|
|
pkgbranches = os.listdir(etpConst['packagessuploaddir'])
|
|
pkgbranches = [x for x in pkgbranches if os.path.isdir(etpConst['packagessuploaddir']+"/"+x)]
|
|
|
|
for mybranch in pkgbranches:
|
|
|
|
print_info(red(" * ")+blue("Switching to branch: ")+bold(mybranch))
|
|
print_info(green(" * ")+brown("Local Statistics: "))
|
|
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir']+"/"+mybranch)+red(" ..."), back = True)
|
|
|
|
uploadCounter = 0
|
|
toBeUploaded = set() # parse etpConst['packagessuploaddir']
|
|
for tbz2 in os.listdir(etpConst['packagessuploaddir']+"/"+mybranch):
|
|
if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']):
|
|
toBeUploaded.add(tbz2)
|
|
if tbz2.endswith(".tbz2"):
|
|
uploadCounter += 1
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: upload directory stats -> files: "+str(uploadCounter)+" | upload packages list: "+str(toBeUploaded))
|
|
|
|
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
|
|
localPackagesRepository = set() # parse etpConst['packagesbindir']
|
|
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir']+"/"+mybranch)+red(" ..."), back = True)
|
|
packageCounter = 0
|
|
for tbz2 in os.listdir(etpConst['packagesbindir']+"/"+mybranch):
|
|
if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']):
|
|
localPackagesRepository.add(tbz2)
|
|
if tbz2.endswith(".tbz2"):
|
|
packageCounter += 1
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: packages directory stats -> files: "+str(packageCounter)+" | download packages list (including md5): "+str(localPackagesRepository))
|
|
|
|
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
|
|
|
|
print_info(green(" * ")+brown("Fetching remote statistics..."), back = True)
|
|
ftp = FtpInterface(uri, Entropy)
|
|
try:
|
|
ftp.setCWD(etpConst['binaryurirelativepath'])
|
|
except:
|
|
bdir = ""
|
|
for mydir in etpConst['binaryurirelativepath'].split("/"):
|
|
bdir += "/"+mydir
|
|
if (not ftp.isFileAvailable(bdir)):
|
|
try:
|
|
ftp.mkdir(bdir)
|
|
except Exception, e:
|
|
if str(e).find("550") != -1:
|
|
pass
|
|
else:
|
|
raise
|
|
ftp.setCWD(etpConst['binaryurirelativepath'])
|
|
|
|
if (not ftp.isFileAvailable(mybranch)):
|
|
ftp.mkdir(mybranch)
|
|
ftp.setCWD(mybranch)
|
|
remotePackages = ftp.listDir()
|
|
remotePackagesInfo = ftp.getRoughList()
|
|
ftp.closeConnection()
|
|
|
|
print_info(green(" * ")+brown("Remote statistics"))
|
|
remoteCounter = 0
|
|
for tbz2 in remotePackages:
|
|
if tbz2.endswith(".tbz2"):
|
|
remoteCounter += 1
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: remote packages stats -> files: "+str(remoteCounter))
|
|
|
|
print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
|
|
|
|
print_info(green(" * ")+brown("Calculating..."))
|
|
uploadQueue = set()
|
|
downloadQueue = set()
|
|
removalQueue = set()
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: starting packages calculation...")
|
|
|
|
finePackages = set()
|
|
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
|
|
for localPackage in toBeUploaded:
|
|
if localPackage in remotePackages:
|
|
# it's already on the mirror, but... is its size correct??
|
|
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+mybranch+"/"+localPackage)[6])
|
|
remoteSize = 0
|
|
for data in remotePackagesInfo:
|
|
if data.split()[8] == localPackage:
|
|
remoteSize = int(data.split()[4])
|
|
#print localPackage,"==>",localSize, remoteSize
|
|
if (localSize != remoteSize):
|
|
# size does not match, adding to the upload queue
|
|
uploadQueue.add(localPackage)
|
|
else:
|
|
finePackages.add(localPackage) # just move from upload to packages
|
|
else:
|
|
# always force upload of packages in uploaddir
|
|
uploadQueue.add(localPackage)
|
|
|
|
# if a package is in the packages directory but not online, we have to upload it
|
|
# we have localPackagesRepository and remotePackages
|
|
for localPackage in localPackagesRepository:
|
|
if localPackage in remotePackages:
|
|
# it's already on the mirror, but... is its size correct??
|
|
localSize = int(os.stat(etpConst['packagesbindir']+"/"+mybranch+"/"+localPackage)[6])
|
|
remoteSize = 0
|
|
for xfile in remotePackagesInfo:
|
|
if xfile.split()[8] == localPackage:
|
|
remoteSize = int(xfile.split()[4])
|
|
if (localSize != remoteSize) and (localSize != 0):
|
|
# size does not match, adding to the upload queue
|
|
if localPackage not in finePackages:
|
|
uploadQueue.add(localPackage)
|
|
else:
|
|
# this means that the local package does not exist
|
|
# so, we need to download it
|
|
uploadQueue.add(localPackage)
|
|
|
|
# Fill downloadQueue and removalQueue
|
|
for remotePackage in remotePackages:
|
|
if remotePackage in localPackagesRepository:
|
|
# it's already on the mirror, but... is its size correct??
|
|
localSize = int(os.stat(etpConst['packagesbindir']+"/"+mybranch+"/"+remotePackage)[6])
|
|
remoteSize = 0
|
|
for xfile in remotePackagesInfo:
|
|
if xfile.split()[8] == remotePackage:
|
|
remoteSize = int(xfile.split()[4])
|
|
if (localSize != remoteSize) and (localSize != 0):
|
|
# size does not match, remove first
|
|
#print "removal of "+localPackage+" because its size differ"
|
|
if remotePackage not in uploadQueue: # do it only if the package has not been added to the uploadQueue
|
|
removalQueue.add(remotePackage) # remotePackage == localPackage # just remove something that differs from the content of the mirror
|
|
# then add to the download queue
|
|
downloadQueue.add(remotePackage)
|
|
else:
|
|
# this means that the local package does not exist
|
|
# so, we need to download it
|
|
if not remotePackage.endswith(".tmp"): # ignore .tmp files
|
|
downloadQueue.add(remotePackage)
|
|
|
|
# Collect packages that don't exist anymore in the database
|
|
# so we can filter them out from the download queue
|
|
# Why downloading something that will be removed??
|
|
# the same thing for the uploadQueue...
|
|
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
|
|
dbFiles = dbconn.listBranchPackagesTbz2(mybranch)
|
|
dbconn.closeDB()
|
|
|
|
exclude = set()
|
|
for dlFile in downloadQueue:
|
|
if dlFile.endswith(".tbz2"):
|
|
if dlFile not in dbFiles:
|
|
exclude.add(dlFile)
|
|
downloadQueue.difference_update(exclude)
|
|
|
|
exclude = set()
|
|
for upFile in uploadQueue:
|
|
if upFile.endswith(".tbz2"):
|
|
if upFile not in dbFiles:
|
|
exclude.add(upFile)
|
|
uploadQueue.difference_update(exclude)
|
|
|
|
# now filter things
|
|
# packages in uploadQueue should be removed, if found, from downloadQueue
|
|
exclude = set()
|
|
for p in downloadQueue:
|
|
# search inside uploadQueue
|
|
if p in uploadQueue:
|
|
exclude.add(p)
|
|
downloadQueue.difference_update(exclude)
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: removal queue -> "+str(removalQueue))
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: download queue -> "+str(downloadQueue))
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: upload queue -> "+str(uploadQueue))
|
|
|
|
if (not uploadQueue) and (not downloadQueue) and (not removalQueue):
|
|
print_info(green(" * ")+red("Nothing to syncronize for ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)+red(". Queue empty.")))
|
|
uriSuccessfulSync += 1
|
|
if (uriSuccessfulSync == len(pkgbranches)):
|
|
totalSuccessfulUri += 1
|
|
continue
|
|
|
|
totalRemovalSize = 0
|
|
totalDownloadSize = 0
|
|
totalUploadSize = 0
|
|
|
|
print_info(green(" * ")+brown("Queue tasks:"))
|
|
detailedRemovalQueue = []
|
|
detailedDownloadQueue = []
|
|
detailedUploadQueue = []
|
|
# this below is used when a package has been already uploaded
|
|
# but something weird happened and it hasn't been moved to the packages dir
|
|
simpleCopyQueue = []
|
|
|
|
for item in removalQueue:
|
|
fileSize = os.stat(etpConst['packagesbindir']+"/"+mybranch+"/"+item)[6]
|
|
totalRemovalSize += int(fileSize)
|
|
print_info(bold("\t[") + red("LOCAL REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(Entropy.entropyTools.bytesIntoHuman(fileSize)))
|
|
detailedRemovalQueue.append([item,fileSize])
|
|
|
|
for item in downloadQueue:
|
|
# if the package is already in the upload directory, do not add the size
|
|
if not os.path.isfile(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item):
|
|
fileSize = "0"
|
|
for remotePackage in remotePackagesInfo:
|
|
if remotePackage.split()[8] == item:
|
|
fileSize = remotePackage.split()[4]
|
|
break
|
|
if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload
|
|
totalDownloadSize += int(fileSize)
|
|
print_info(bold("\t[") + brown("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(Entropy.entropyTools.bytesIntoHuman(fileSize)))
|
|
detailedDownloadQueue.append([item,fileSize])
|
|
else:
|
|
if (not item.endswith(etpConst['packageshashfileext'])):
|
|
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item)[6]
|
|
print_info(bold("\t[") + green("LOCAL COPY") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(Entropy.entropyTools.bytesIntoHuman(fileSize)))
|
|
# file exists locally and remotely (where is fine == fully uploaded)
|
|
simpleCopyQueue.append(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item)
|
|
|
|
for item in uploadQueue:
|
|
# if it is in the upload dir
|
|
if os.path.isfile(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item):
|
|
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item)[6]
|
|
else: # otherwise it is in the packages dir
|
|
fileSize = os.stat(etpConst['packagesbindir']+"/"+mybranch+"/"+item)[6]
|
|
if not item.endswith(etpConst['packageshashfileext']): # do not show .md5 to upload
|
|
print_info(bold("\t[") + red("REMOTE UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(Entropy.entropyTools.bytesIntoHuman(fileSize)))
|
|
totalUploadSize += int(fileSize)
|
|
detailedUploadQueue.append([item,fileSize])
|
|
|
|
# queue length info
|
|
removalQueueLength = 0
|
|
for i in removalQueue:
|
|
if not i.endswith(etpConst['packageshashfileext']):
|
|
removalQueueLength += 1
|
|
downloadQueueLength = 0
|
|
for i in downloadQueue:
|
|
if not i.endswith(etpConst['packageshashfileext']):
|
|
downloadQueueLength += 1
|
|
uploadQueueLength = 0
|
|
for i in uploadQueue:
|
|
if not i.endswith(etpConst['packageshashfileext']):
|
|
uploadQueueLength += 1
|
|
|
|
print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t\t")+bold(str(removalQueueLength)))
|
|
print_info(red(" * ")+blue("Packages that would be ")+brown("downloaded/moved locally:\t")+bold(str(downloadQueueLength)))
|
|
print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t\t")+bold(str(uploadQueueLength)))
|
|
print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t\t")+bold(Entropy.entropyTools.bytesIntoHuman(str(totalRemovalSize))))
|
|
print_info(red(" * ")+blue("Total download ")+brown("size:\t\t\t\t")+bold(Entropy.entropyTools.bytesIntoHuman(str(totalDownloadSize))))
|
|
print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t\t")+bold(Entropy.entropyTools.bytesIntoHuman(str(totalUploadSize))))
|
|
|
|
if (etpUi['pretend']):
|
|
continue
|
|
if (etpUi['ask']):
|
|
rc = Entropy.askQuestion("\n Would you like to run the steps above ?")
|
|
if rc == "No":
|
|
print "\n"
|
|
continue
|
|
|
|
# queues management
|
|
successfulUploadCounter = 0
|
|
successfulDownloadCounter = 0
|
|
uploadCounter = "0"
|
|
downloadCounter = "0"
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: detailed removal queue -> "+str(detailedRemovalQueue))
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: detailed simple copy queue -> "+str(simpleCopyQueue))
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: detailed download queue -> "+str(detailedDownloadQueue))
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: detailed upload queue -> "+str(detailedUploadQueue))
|
|
|
|
|
|
# removal queue
|
|
if (detailedRemovalQueue):
|
|
for item in detailedRemovalQueue:
|
|
print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(Entropy.entropyTools.bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']+"/"+mybranch)+red(" ..."))
|
|
try:
|
|
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+item[0])
|
|
except OSError:
|
|
pass
|
|
try:
|
|
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+item[0]+etpConst['packageshashfileext'])
|
|
except OSError:
|
|
pass
|
|
print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
|
|
|
# simple copy queue
|
|
if (simpleCopyQueue):
|
|
for item in simpleCopyQueue:
|
|
print_info(red(" * Copying file from ") + bold(item) + red(" to ")+bold(etpConst['packagesbindir']+"/"+mybranch))
|
|
toitem = etpConst['packagesbindir']+"/"+mybranch+"/"+os.path.basename(item)
|
|
if not os.path.isdir(os.path.dirname(toitem)):
|
|
os.makedirs(os.path.dirname(toitem))
|
|
# md5 copy not needed, already in simpleCopyQueue
|
|
shutil.copy2(item,toitem)
|
|
if os.path.isfile(toitem+etpConst['packagesexpirationfileext']): # clear expiration file
|
|
os.remove(toitem+etpConst['packagesexpirationfileext'])
|
|
|
|
|
|
# upload queue
|
|
if (detailedUploadQueue):
|
|
mirrorsTainted = True
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch)
|
|
uploadCounter = str(len(detailedUploadQueue))
|
|
currentCounter = 0
|
|
for item in detailedUploadQueue:
|
|
currentCounter += 1
|
|
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")")
|
|
print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(Entropy.entropyTools.bytesIntoHuman(item[1]))+red("] to ")+ bold(Entropy.entropyTools.extractFTPHostFromUri(uri)) +red(" ..."))
|
|
# is the package in the upload queue?
|
|
if os.path.isfile(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item[0]):
|
|
uploadItem = etpConst['packagessuploaddir']+"/"+mybranch+"/"+item[0]
|
|
else:
|
|
uploadItem = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0]
|
|
|
|
ckOk = False
|
|
while not ckOk:
|
|
rc = ftp.uploadFile(uploadItem)
|
|
print_info(" "+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
|
|
ck = Entropy.get_remote_package_checksum(Entropy.entropyTools.extractFTPHostFromUri(uri),item[0], mybranch)
|
|
if (ck == None):
|
|
print_warning(" "+red(" -> Digest verification of ")+green(item[0])+bold(" not supported"))
|
|
ckOk = True
|
|
else:
|
|
if (ck == False):
|
|
# file does not exist???
|
|
print_warning(" "+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading..."))
|
|
else:
|
|
if len(ck) == 32:
|
|
# valid checksum, checking
|
|
ckres = Entropy.entropyTools.compareMd5(uploadItem,ck)
|
|
if (ckres):
|
|
print_info(" "+red(" -> Package ")+bold(item[0])+red(" has been uploaded correctly."))
|
|
ckOk = True
|
|
else:
|
|
print_warning(" "+red(" -> Package ")+bold(item[0])+brown(" has NOT been uploaded correctly. Reuploading..."))
|
|
else:
|
|
# hum, what the hell is this checksum!?!?!?!
|
|
print_warning(" "+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+". Reuploading..."))
|
|
|
|
if not os.path.isfile(uploadItem+etpConst['packageshashfileext']):
|
|
hashfile = Entropy.entropyTools.createHashFile(uploadItem)
|
|
else:
|
|
hashfile = uploadItem+etpConst['packageshashfileext']
|
|
|
|
# upload md5 hash
|
|
print_info(" "+red(" -> Uploading checksum of ")+bold(item[0]) + red(" to ") + bold(Entropy.entropyTools.extractFTPHostFromUri(uri)) +red(" ..."))
|
|
ckOk = False
|
|
while not ckOk:
|
|
rcmd5 = ftp.uploadFile(hashfile,ascii = True)
|
|
print_info(" "+red(" -> Verifying ")+green(item[0]+etpConst['packageshashfileext'])+bold(" checksum")+red(" (if supported)"), back = True)
|
|
ck = Entropy.get_remote_package_checksum(Entropy.entropyTools.extractFTPHostFromUri(uri),item[0]+etpConst['packageshashfileext'], mybranch)
|
|
if (ck == None):
|
|
print_warning(" "+red(" -> Digest verification of ")+green(item[0]+etpConst['packageshashfileext'])+bold(" not supported"))
|
|
ckOk = True
|
|
else:
|
|
if (ck == False):
|
|
# file does not exist???
|
|
print_warning(" "+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not exist remotely. Reuploading..."))
|
|
else:
|
|
if len(ck) == 32:
|
|
# valid checksum, checking
|
|
ckres = Entropy.entropyTools.compareMd5(hashfile,ck)
|
|
if (ckres):
|
|
print_info(" "+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" has been uploaded correctly."))
|
|
ckOk = True
|
|
else:
|
|
print_warning(" "+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+brown(" has NOT been uploaded correctly. Reuploading..."))
|
|
else:
|
|
# hum, what the hell is this checksum!?!?!?!
|
|
print_warning(" "+red(" -> Package ")+bold(item[0]+etpConst['packageshashfileext'])+red(" does not have a proper checksum: "+str(ck)+" Reuploading..."))
|
|
|
|
# now check
|
|
if (rc) and (rcmd5):
|
|
successfulUploadCounter += 1
|
|
print_info(red(" * Upload completed for ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
ftp.closeConnection()
|
|
|
|
# download queue
|
|
if (detailedDownloadQueue):
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch)
|
|
downloadCounter = str(len(detailedDownloadQueue))
|
|
currentCounter = 0
|
|
for item in detailedDownloadQueue:
|
|
currentCounter += 1
|
|
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")")
|
|
if os.path.isfile(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item[0]):
|
|
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+mybranch+"/"+item[0])[6])
|
|
remoteSize = int(item[1])
|
|
if localSize == remoteSize:
|
|
# skip that, we'll move at the end of the mirrors sync
|
|
continue
|
|
print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(Entropy.entropyTools.bytesIntoHuman(item[1]))+red("] from ")+ bold(Entropy.entropyTools.extractFTPHostFromUri(uri)) +red(" ..."))
|
|
|
|
ckOk = False
|
|
while not ckOk:
|
|
|
|
if item[0].endswith(etpConst['packageshashfileext']):
|
|
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/"+mybranch+"/", ascii = True)
|
|
else:
|
|
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/"+mybranch+"/")
|
|
|
|
if not item[0].endswith(etpConst['packageshashfileext']):
|
|
print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
|
|
ck = Entropy.get_remote_package_checksum(Entropy.entropyTools.extractFTPHostFromUri(uri),item[0], mybranch)
|
|
if (ck == None):
|
|
print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported"))
|
|
ckOk = True
|
|
else:
|
|
if (ck == False):
|
|
# file does not exist???
|
|
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Skipping ..."))
|
|
ckOk = True
|
|
else:
|
|
if len(ck) == 32:
|
|
# valid checksum, checking
|
|
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0]
|
|
ckres = Entropy.entropyTools.compareMd5(filepath,ck)
|
|
if (ckres):
|
|
print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been downloaded correctly."))
|
|
ckOk = True
|
|
else:
|
|
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+brown(" has NOT been downloaded correctly. Redownloading..."))
|
|
else:
|
|
# hum, what the hell is this checksum!?!?!?!
|
|
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+" Redownloading..."))
|
|
else: # skip checking for .md5 files
|
|
ckOk = True
|
|
|
|
if (rc):
|
|
successfulDownloadCounter += 1
|
|
|
|
print_info(red(" * Download completed for ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
ftp.closeConnection()
|
|
|
|
uploadCounter = int(uploadCounter)
|
|
downloadCounter = int(downloadCounter)
|
|
|
|
if (successfulUploadCounter == uploadCounter) and (successfulDownloadCounter == downloadCounter):
|
|
uriSuccessfulSync += 1
|
|
|
|
if (uriSuccessfulSync == len(pkgbranches)):
|
|
totalSuccessfulUri += 1
|
|
|
|
# trap exceptions, failed to upload/download someting?
|
|
except Exception, e: # FIXME: only trap proper ftp exceptions
|
|
|
|
print_error(brown(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Showing traceback:"))
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
# trap CTRL+C
|
|
if (str(e) == "100"):
|
|
sys.exit(0)
|
|
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.")
|
|
activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: cannot properly syncronize "+Entropy.entropyTools.extractFTPHostFromUri(uri)+". Trying to continue if possible.")
|
|
|
|
# print warning cannot sync uri
|
|
print_warning(brown(" * ")+red("ATTENTION: cannot properly syncronize ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Continuing if possible..."))
|
|
|
|
# decide what to do
|
|
if (totalSuccessfulUri > 0) or (etpUi['pretend']):
|
|
# we're safe
|
|
activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: at least one mirror has been synced properly. I'm fine.")
|
|
print_info(green(" * ")+red("At least one mirror has been synced properly. I'm fine."))
|
|
continue
|
|
else:
|
|
if (currentUri < totalUris):
|
|
# we have another mirror to try
|
|
continue
|
|
else:
|
|
# no mirrors were synced properly
|
|
# show error and return, do not move files from the upload dir
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.")
|
|
print_error(brown(" * ")+red("ERROR: no mirrors have been properly syncronized. Check network status and retry. Cannot continue."))
|
|
return False
|
|
|
|
|
|
# if at least one server has been synced successfully, move files
|
|
if (totalSuccessfulUri > 0) and (not etpUi['pretend']):
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"packages: all done. Now it's time to move packages to "+etpConst['packagesbindir'])
|
|
pkgbranches = os.listdir(etpConst['packagessuploaddir'])
|
|
pkgbranches = [x for x in pkgbranches if os.path.isdir(etpConst['packagessuploaddir']+"/"+x)]
|
|
for branch in pkgbranches:
|
|
branchcontent = os.listdir(etpConst['packagessuploaddir']+"/"+branch)
|
|
for xfile in branchcontent:
|
|
source = etpConst['packagessuploaddir']+"/"+branch+"/"+xfile
|
|
destdir = etpConst['packagesbindir']+"/"+branch
|
|
if not os.path.isdir(destdir):
|
|
os.makedirs(destdir)
|
|
dest = destdir+"/"+xfile
|
|
shutil.move(source,dest)
|
|
if os.path.isfile(dest+etpConst['packagesexpirationfileext']): # clear expiration file
|
|
os.remove(dest+etpConst['packagesexpirationfileext'])
|
|
return mirrorsTainted
|
|
else:
|
|
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: neither a mirror has been properly sync'd.")
|
|
|
|
# Now we should start to check all the packages in the packages directory
|
|
if (activatorRequestPackagesCheck):
|
|
import reagentTools
|
|
reagentTools.database(['md5check'])
|
|
|
|
|
|
def database(options):
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: called with -> "+str(options))
|
|
|
|
# lock tool
|
|
if (options[0] == "lock"):
|
|
print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
|
|
rc = lockDatabases(lock = True)
|
|
if (rc):
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !")
|
|
print_info(green(" * ")+red("A problem occured on at least one mirror !"))
|
|
else:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
|
|
print_info(green(" * ")+green("Databases lock complete"))
|
|
|
|
# unlock tool
|
|
elif (options[0] == "unlock"):
|
|
print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
|
|
rc = lockDatabases(lock = False)
|
|
if (rc):
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !")
|
|
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
|
|
else:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
|
|
print_info(green(" * ")+green("Databases unlock complete"))
|
|
|
|
# download lock tool
|
|
elif (options[0] == "download-lock"):
|
|
print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
|
|
rc = downloadLockDatabases(lock = True)
|
|
if (rc):
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !")
|
|
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
|
|
else:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.")
|
|
print_info(green(" * ")+green("Download mirrors lock complete"))
|
|
|
|
# download unlock tool
|
|
elif (options[0] == "download-unlock"):
|
|
print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
|
|
rc = downloadLockDatabases(lock = False)
|
|
if (rc):
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !")
|
|
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
|
|
else:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.")
|
|
print_info(green(" * ")+green("Download mirrors unlock complete"))
|
|
|
|
# lock status tool
|
|
elif (options[0] == "lock-status"):
|
|
print_info(brown(" * ")+green("Mirrors status table:"))
|
|
dbstatus = getMirrorsLock()
|
|
for db in dbstatus:
|
|
if (db[1]):
|
|
db[1] = red("Locked")
|
|
else:
|
|
db[1] = green("Unlocked")
|
|
if (db[2]):
|
|
db[2] = red("Locked")
|
|
else:
|
|
db[2] = green("Unlocked")
|
|
print_info(bold("\t"+Entropy.entropyTools.extractFTPHostFromUri(db[0])+": ")+red("[")+brown("DATABASE: ")+db[1]+red("] [")+brown("DOWNLOAD: ")+db[2]+red("]"))
|
|
|
|
# database sync tool
|
|
elif (options[0] == "sync"):
|
|
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.")
|
|
|
|
print_info(green(" * ")+red("Checking database status ..."), back = True)
|
|
|
|
dbLockFile = False
|
|
# does the taint file exist?
|
|
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
|
|
dbLockFile = True
|
|
|
|
# are online mirrors locked?
|
|
mirrorsLocked = False
|
|
for uri in etpConst['activatoruploaduris']:
|
|
ftp = FtpInterface(uri, Entropy)
|
|
try:
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
except:
|
|
bdir = ""
|
|
for mydir in etpConst['etpurirelativepath'].split("/"):
|
|
bdir += "/"+mydir
|
|
if (not ftp.isFileAvailable(bdir)):
|
|
try:
|
|
ftp.mkdir(bdir)
|
|
except Exception, e:
|
|
if str(e).find("550") != -1:
|
|
pass
|
|
else:
|
|
raise
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
|
|
mirrorsLocked = True
|
|
ftp.closeConnection()
|
|
break
|
|
|
|
if (mirrorsLocked):
|
|
# if the mirrors are locked, we need to check if we have
|
|
# the taint file in place. Because in this case, the one
|
|
# that tainted the db, was me.
|
|
if (dbLockFile):
|
|
print_info(green(" * ")+red("Updating mirrors with new information ..."))
|
|
# it's safe to sync
|
|
syncRemoteDatabases()
|
|
# remove the online lock file
|
|
lockDatabases(False)
|
|
# remove the taint file
|
|
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
|
|
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
|
|
else:
|
|
print
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...")
|
|
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: At the moment, mirrors are locked, someone is working on their databases, try again later...")
|
|
|
|
else:
|
|
if (dbLockFile):
|
|
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
|
|
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
|
|
else:
|
|
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...")
|
|
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
|
|
|
|
syncRemoteDatabases()
|
|
|
|
else:
|
|
print_error(red(" * ")+green("No valid tool specified."))
|
|
|
|
########################################################
|
|
####
|
|
## Database handling functions
|
|
#
|
|
|
|
def syncRemoteDatabases(noUpload = False, justStats = False):
|
|
|
|
remoteDbsStatus = getEtpRemoteDatabaseStatus()
|
|
print_info(green(" * ")+red("Remote Entropy Database Repository Status:"))
|
|
for dbstat in remoteDbsStatus:
|
|
print_info(green("\t Host:\t")+bold(Entropy.entropyTools.extractFTPHostFromUri(dbstat[0])))
|
|
print_info(red("\t * Database revision: ")+blue(str(dbstat[1])))
|
|
|
|
# check if the local DB or the revision file exist
|
|
# in this way we can regenerate the db without messing the --initialize function with a new unwanted download
|
|
etpDbLocalRevision = 0
|
|
if os.path.isfile(etpConst['etpdatabasefilepath']):
|
|
# file exist, get revision if we can
|
|
if os.path.isfile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile']):
|
|
f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],"r")
|
|
etpDbLocalRevision = int(f.readline().strip())
|
|
f.close()
|
|
|
|
print_info(red("\t * Database local revision currently at: ")+blue(str(etpDbLocalRevision)))
|
|
|
|
if (justStats):
|
|
return
|
|
|
|
downloadLatest = []
|
|
uploadLatest = False
|
|
uploadList = []
|
|
|
|
# if the local DB does not exist, get the latest
|
|
if (etpDbLocalRevision == 0):
|
|
# seek mirrors
|
|
latestRemoteDb = []
|
|
etpDbRemotePaths = []
|
|
for dbstat in remoteDbsStatus:
|
|
if ( dbstat[1] != 0 ):
|
|
# collect
|
|
etpDbRemotePaths.append(dbstat)
|
|
if etpDbRemotePaths == []:
|
|
#print "DEBUG: generate and upload"
|
|
# (to all!)
|
|
uploadLatest = True
|
|
uploadList = remoteDbsStatus
|
|
else:
|
|
#print "DEBUG: get the latest ?"
|
|
revisions = []
|
|
for dbstat in etpDbRemotePaths:
|
|
revisions.append(dbstat[1])
|
|
if len(revisions) > 1:
|
|
latestrevision = Entropy.entropyTools.alphaSorter(revisions)[len(revisions)-1]
|
|
else:
|
|
latestrevision = revisions[0]
|
|
for dbstat in etpDbRemotePaths:
|
|
if dbstat[1] == latestrevision:
|
|
# found !
|
|
downloadLatest = dbstat
|
|
break
|
|
# Now check if we need to upload back the files to the other mirrors
|
|
#print "DEBUG: check the others, if they're also updated, quit"
|
|
for dbstat in remoteDbsStatus:
|
|
if (downloadLatest[1] != dbstat[1]):
|
|
uploadLatest = True
|
|
uploadList.append(dbstat)
|
|
else:
|
|
# while if it exists
|
|
# seek mirrors
|
|
latestRemoteDb = []
|
|
etpDbRemotePaths = []
|
|
for dbstat in remoteDbsStatus:
|
|
if ( dbstat[1] != 0 ):
|
|
# collect
|
|
etpDbRemotePaths.append(dbstat)
|
|
if etpDbRemotePaths == []:
|
|
#print "DEBUG: upload our version"
|
|
uploadLatest = True
|
|
# upload to all !
|
|
uploadList = remoteDbsStatus
|
|
else:
|
|
#print "DEBUG: get the latest?"
|
|
revisions = []
|
|
for dbstat in etpDbRemotePaths:
|
|
revisions.append(str(dbstat[1]))
|
|
|
|
latestrevision = int(Entropy.entropyTools.alphaSorter(revisions)[len(revisions)-1])
|
|
for dbstat in etpDbRemotePaths:
|
|
if dbstat[1] == latestrevision:
|
|
# found !
|
|
latestRemoteDb = dbstat
|
|
break
|
|
# now compare downloadLatest with our local db revision
|
|
if (etpDbLocalRevision < latestRemoteDb[1]):
|
|
# download !
|
|
#print "appending a download"
|
|
downloadLatest = latestRemoteDb
|
|
elif (etpDbLocalRevision > latestRemoteDb[1]):
|
|
# upload to all !
|
|
uploadLatest = True
|
|
uploadList = remoteDbsStatus
|
|
|
|
# If the uploadList is not filled, this means that the other mirror might need an update
|
|
if (not uploadLatest):
|
|
for dbstat in remoteDbsStatus:
|
|
if (latestRemoteDb[1] != dbstat[1]):
|
|
uploadLatest = True
|
|
uploadList.append(dbstat)
|
|
|
|
if (downloadLatest == []) and (not uploadLatest):
|
|
print_info(green(" * ")+red("Online database does not need to be updated."))
|
|
return
|
|
|
|
# now run the selected task!
|
|
if (downloadLatest != []):
|
|
# match the proper URI
|
|
for uri in etpConst['activatoruploaduris']:
|
|
if downloadLatest[0].startswith(uri):
|
|
downloadLatest[0] = uri
|
|
downloadDatabase(downloadLatest[0])
|
|
|
|
if (uploadLatest) and (not noUpload):
|
|
print_info(green(" * ")+red("Starting to update the needed mirrors ..."))
|
|
_uploadList = []
|
|
for uri in etpConst['activatoruploaduris']:
|
|
for xlist in uploadList:
|
|
if xlist[0].startswith(uri):
|
|
_uploadList.append(uri)
|
|
break
|
|
|
|
uploadDatabase(_uploadList)
|
|
print_info(green(" * ")+red("All the mirrors have been updated."))
|
|
|
|
remoteDbsStatus = getEtpRemoteDatabaseStatus()
|
|
print_info(green(" * ")+red("Remote Entropy Database Repository Status:"))
|
|
for dbstat in remoteDbsStatus:
|
|
print_info(green("\t Host:\t")+bold(Entropy.entropyTools.extractFTPHostFromUri(dbstat[0])))
|
|
print_info(red("\t * Database revision: ")+blue(str(dbstat[1])))
|
|
|
|
|
|
def uploadDatabase(uris):
|
|
|
|
import gzip
|
|
import bz2
|
|
|
|
### PREPARE RSS FEED
|
|
if etpConst['rss-feed']:
|
|
rssClass = rssFeed(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'], maxentries = etpConst['rss-max-entries'])
|
|
# load dump
|
|
db_actions = Entropy.dumpTools.loadobj(etpConst['rss-dump-name'])
|
|
if db_actions:
|
|
try:
|
|
f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])
|
|
revision = f.readline().strip()
|
|
f.close()
|
|
except:
|
|
revision = "N/A"
|
|
pass
|
|
commitmessage = ''
|
|
if etpRSSMessages['commitmessage']:
|
|
commitmessage = ' :: '+etpRSSMessages['commitmessage']
|
|
title = ": "+etpConst['systemname']+" "+etpConst['product'][0].upper()+etpConst['product'][1:]+" "+etpConst['branch']+" :: Revision: "+revision+commitmessage
|
|
link = etpConst['rss-base-url']
|
|
# create description
|
|
added_items = db_actions.get("added")
|
|
if added_items:
|
|
for atom in added_items:
|
|
mylink = link+"?search="+atom.split("~")[0]+"&arch="+etpConst['currentarch']+"&product="+etpConst['product']
|
|
description = atom+": "+added_items[atom]['description']
|
|
rssClass.addItem(title = "Added/Updated"+title, link = mylink, description = description)
|
|
removed_items = db_actions.get("removed")
|
|
if removed_items:
|
|
for atom in removed_items:
|
|
description = atom+": "+removed_items[atom]['description']
|
|
rssClass.addItem(title = "Removed"+title, link = link, description = description)
|
|
|
|
rssClass.writeChanges()
|
|
# clean global vars
|
|
etpRSSMessages.clear()
|
|
Entropy.dumpTools.removeobj(etpConst['rss-dump-name'])
|
|
|
|
for uri in uris:
|
|
downloadLockDatabases(True,[uri])
|
|
|
|
print_info(green(" * ")+red("Uploading database to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" ..."))
|
|
print_info(green(" * ")+red("Connecting to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" ..."), back = True)
|
|
ftp = FtpInterface(uri, Entropy)
|
|
print_info(green(" * ")+red("Changing directory to ")+bold(etpConst['etpurirelativepath'])+red(" ..."), back = True)
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
|
|
cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
|
|
if cmethod == None: raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
|
|
print_info(green(" * ")+red("Uploading file ")+bold(etpConst[cmethod[2]])+red(" ..."), back = True)
|
|
|
|
dbfilec = eval(cmethod[0])(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]], "wb")
|
|
# compress the database file first
|
|
dbfile = open(etpConst['etpdatabasefilepath'],"rb")
|
|
dbcont = dbfile.readlines()
|
|
dbfile.close()
|
|
for i in dbcont:
|
|
dbfilec.write(i)
|
|
dbfilec.close()
|
|
del dbcont
|
|
|
|
# uploading database file
|
|
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]])
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Upload of ")+bold(etpConst[cmethod[2]])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# remove the compressed file
|
|
os.remove(etpConst['etpdatabasedir'] + "/" + etpConst[cmethod[2]])
|
|
|
|
# generate digest
|
|
hexdigest = Entropy.entropyTools.md5sum(etpConst['etpdatabasefilepath'])
|
|
f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],"w")
|
|
f.write(hexdigest+" "+etpConst['etpdatabasefile']+"\n")
|
|
f.flush()
|
|
f.close()
|
|
|
|
# upload digest
|
|
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" ..."), back = True)
|
|
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'],True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasehashfile'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# uploading revision file
|
|
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
|
|
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# uploading rss file (if enabled)
|
|
if etpConst['rss-feed'] and os.path.isfile(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name']):
|
|
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'])+red(" ..."), back = True)
|
|
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'],True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# close connection
|
|
ftp.closeConnection()
|
|
# unlock database
|
|
downloadLockDatabases(False,[uri])
|
|
|
|
def downloadDatabase(uri):
|
|
|
|
import gzip
|
|
import bz2
|
|
|
|
print_info(green(" * ")+red("Downloading database from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" ..."))
|
|
print_info(green(" * ")+red("Connecting to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" ..."), back = True)
|
|
ftp = FtpInterface(uri, Entropy)
|
|
print_info(green(" * ")+red("Changing directory to ")+bold(etpConst['etpurirelativepath'])+red(" ..."), back = True)
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
|
|
cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
|
|
if cmethod == None: raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
|
|
|
|
unpackFunction = cmethod[1]
|
|
dbfilename = etpConst[cmethod[2]]
|
|
|
|
# downloading database file
|
|
print_info(green(" * ")+red("Downloading file to ")+bold(dbfilename)+red(" ..."), back = True)
|
|
rc = ftp.downloadFile(etpConst['etpdatabasedir'] + "/" + dbfilename,os.path.dirname(etpConst['etpdatabasefilepath']))
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Download of ")+bold(dbfilename)+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly download to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# On the fly decompression
|
|
print_info(green(" * ")+red("Decompressing ")+bold(dbfilename)+red(" ..."), back = True)
|
|
|
|
eval(unpackFunction)(etpConst['etpdatabasedir'] + "/" + dbfilename)
|
|
|
|
print_info(green(" * ")+red("Decompression of ")+bold(dbfilename)+red(" completed."))
|
|
|
|
# downloading revision file
|
|
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
|
|
rc = ftp.downloadFile(etpConst['etpdatabaserevisionfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabaserevisionfile'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly download to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# downlading digest -> FIXME: add digest comparation
|
|
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasehashfile'])+red(" ..."), back = True)
|
|
rc = ftp.downloadFile(etpConst['etpdatabasehashfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabasehashfile'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
# download RSS
|
|
if etpConst['rss-feed']:
|
|
|
|
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['rss-name'])+red(" ..."), back = True)
|
|
try:
|
|
rc = ftp.downloadFile(etpConst['rss-name'],etpConst['etpdatabasedir'],True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Download of ")+bold(etpConst['rss-name'])+red(" completed."))
|
|
else:
|
|
print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
except:
|
|
print_warning(brown(" * ")+red("Cannot properly download RSS file: "+etpConst['rss-name']+" for: ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
|
|
|
|
try:
|
|
os.remove(etpConst['etpdatabasedir'] + "/" + dbfilename)
|
|
except OSError:
|
|
pass
|
|
# close connection
|
|
ftp.closeConnection()
|
|
|
|
# Reports in a list form the lock status of the mirrors
|
|
# @ [ uri , True/False, True/False ] --> True = locked, False = unlocked
|
|
# @ the second parameter is referred to upload locks, while the second to download ones
|
|
def getMirrorsLock():
|
|
|
|
# parse etpConst['activatoruploaduris']
|
|
dbstatus = []
|
|
for uri in etpConst['activatoruploaduris']:
|
|
data = [ uri, False , False ]
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
|
|
# Upload is locked
|
|
data[1] = True
|
|
if (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
|
|
# Upload is locked
|
|
data[2] = True
|
|
ftp.closeConnection()
|
|
dbstatus.append(data)
|
|
return dbstatus
|
|
|
|
# This function check the Entropy online database status
|
|
def getEtpRemoteDatabaseStatus():
|
|
|
|
uriDbInfo = []
|
|
for uri in etpConst['activatoruploaduris']:
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
|
|
if cmethod == None: raise exceptionTools.InvalidDataType("InvalidDataType: wrong database compression method passed.")
|
|
compressedfile = etpConst[cmethod[2]]
|
|
rc = ftp.isFileAvailable(compressedfile)
|
|
if (rc):
|
|
# then get the file revision, if exists
|
|
rc = ftp.isFileAvailable(etpConst['etpdatabaserevisionfile'])
|
|
if (rc):
|
|
# get the revision number
|
|
ftp.downloadFile(etpConst['etpdatabaserevisionfile'],etpConst['packagestmpdir'],True)
|
|
f = open( etpConst['packagestmpdir'] + "/" + etpConst['etpdatabaserevisionfile'],"r")
|
|
revision = int(f.readline().strip())
|
|
f.close()
|
|
Entropy.entropyTools.spawnCommand("rm -f "+etpConst['packagestmpdir']+etpConst['etpdatabaserevisionfile'])
|
|
else:
|
|
revision = 0
|
|
else:
|
|
# then set mtime to 0 and quit
|
|
revision = 0
|
|
info = [uri+"/"+etpConst['etpurirelativepath']+compressedfile,revision]
|
|
uriDbInfo.append(info)
|
|
ftp.closeConnection()
|
|
|
|
return uriDbInfo
|
|
|
|
def downloadPackageFromMirror(uri,pkgfile,branch):
|
|
|
|
tries = 0
|
|
maxtries = 5
|
|
for i in range(maxtries):
|
|
|
|
|
|
print_info(red(" * Connecting to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)), back = True)
|
|
# connect
|
|
ftp = FtpInterface(uri, Entropy)
|
|
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+branch)
|
|
# get the files
|
|
print_info(red(" * Downloading ")+brown(pkgfile)+red(" from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
rc = ftp.downloadFile(pkgfile,etpConst['packagesbindir']+"/"+branch)
|
|
if (rc is None):
|
|
# file does not exist
|
|
print_warning(red(" * File ")+brown(pkgfile)+red(" does not exist remotely on ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
|
|
ftp.closeConnection()
|
|
return None
|
|
# check md5
|
|
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
|
|
idpackage = dbconn.getIDPackageFromFileInBranch(pkgfile,branch)
|
|
storedmd5 = dbconn.retrieveDigest(idpackage)
|
|
dbconn.closeDB()
|
|
print_info(red(" * Checking MD5 of ")+brown(pkgfile)+red(": should be ")+bold(storedmd5), back = True)
|
|
md5check = Entropy.entropyTools.compareMd5(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile,storedmd5)
|
|
if (md5check):
|
|
print_info(red(" * Package ")+brown(pkgfile)+red("downloaded successfully."))
|
|
return True
|
|
else:
|
|
if (tries == maxtries):
|
|
print_warning(red(" * Package ")+brown(pkgfile)+red(" checksum does not match. Please consider to download or repackage again. Giving up."))
|
|
return False
|
|
else:
|
|
print_warning(red(" * Package ")+brown(pkgfile)+red(" checksum does not match. Trying to download it again..."))
|
|
tries += 1
|
|
if os.path.isfile(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile):
|
|
os.remove(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile)
|
|
|
|
def lockDatabases(lock = True, mirrorList = []):
|
|
|
|
outstat = False
|
|
if (mirrorList == []):
|
|
mirrorList = etpConst['activatoruploaduris']
|
|
for uri in mirrorList:
|
|
|
|
|
|
if (lock):
|
|
print_info(brown(" * ")+red("Locking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror..."),back = True)
|
|
else:
|
|
print_info(brown(" * ")+red("Unlocking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror..."),back = True)
|
|
ftp = FtpInterface(uri, Entropy)
|
|
# upload the lock file to database/%ARCH% directory
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
# check if the lock is already there
|
|
if (lock):
|
|
if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
|
|
print_info(green(" * ")+red("Mirror database at ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" already locked."))
|
|
ftp.closeConnection()
|
|
continue
|
|
else:
|
|
if (not ftp.isFileAvailable(etpConst['etpdatabaselockfile'])):
|
|
print_info(green(" * ")+red("Mirror database at ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" already unlocked."))
|
|
ftp.closeConnection()
|
|
continue
|
|
if (lock):
|
|
f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],"w")
|
|
f.write("database locked\n")
|
|
f.flush()
|
|
f.close()
|
|
rc = ftp.uploadFile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],ascii= True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Succesfully locked ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror."))
|
|
else:
|
|
outstat = True
|
|
print "\n"
|
|
print_warning(red(" * ")+red("A problem occured while locking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror. Please have a look."))
|
|
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
|
|
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
|
|
else:
|
|
rc = ftp.deleteFile(etpConst['etpdatabaselockfile'])
|
|
if (rc):
|
|
print_info(green(" * ")+red("Succesfully unlocked ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror."))
|
|
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
|
|
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
|
|
else:
|
|
outstat = True
|
|
print "\n"
|
|
print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" mirror. Please have a look."))
|
|
ftp.closeConnection()
|
|
return outstat
|
|
|
|
def downloadLockDatabases(lock = True, mirrorList = []):
|
|
|
|
outstat = False
|
|
if (mirrorList == []):
|
|
mirrorList = etpConst['activatoruploaduris']
|
|
for uri in mirrorList:
|
|
if (lock):
|
|
print_info(brown(" * ")+red("Locking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror..."),back = True)
|
|
else:
|
|
print_info(brown(" * ")+red("Unlocking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror..."),back = True)
|
|
ftp = FtpInterface(uri, Entropy)
|
|
# upload the lock file to database/%ARCH% directory
|
|
ftp.setCWD(etpConst['etpurirelativepath'])
|
|
# check if the lock is already there
|
|
if (lock):
|
|
if (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
|
|
print_info(green(" * ")+red("Download mirror at ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" already locked."))
|
|
ftp.closeConnection()
|
|
continue
|
|
else:
|
|
if (not ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
|
|
print_info(green(" * ")+red("Download mirror at ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" already unlocked."))
|
|
ftp.closeConnection()
|
|
continue
|
|
if (lock):
|
|
f = open(etpConst['packagestmpdir']+"/"+etpConst['etpdatabasedownloadlockfile'],"w")
|
|
f.write("database locked\n")
|
|
f.flush()
|
|
f.close()
|
|
rc = ftp.uploadFile(etpConst['packagestmpdir']+"/"+etpConst['etpdatabasedownloadlockfile'],ascii= True)
|
|
if (rc == True):
|
|
print_info(green(" * ")+red("Succesfully locked ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror."))
|
|
else:
|
|
outstat = True
|
|
print "\n"
|
|
print_warning(red(" * ")+red("A problem occured while locking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror. Please have a look."))
|
|
else:
|
|
rc = ftp.deleteFile(etpConst['etpdatabasedownloadlockfile'])
|
|
if (rc):
|
|
print_info(green(" * ")+red("Succesfully unlocked ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror."))
|
|
else:
|
|
outstat = True
|
|
print "\n"
|
|
print_warning(red(" * ")+red("A problem occured while unlocking ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(" download mirror. Please have a look."))
|
|
ftp.closeConnection()
|
|
return outstat
|
|
|
|
def getLocalDatabaseRevision():
|
|
if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile']):
|
|
f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaserevisionfile'])
|
|
rev = f.readline().strip()
|
|
f.close()
|
|
rev = int(rev)
|
|
return rev
|
|
else:
|
|
return 0
|