#!/usr/bin/python
'''
    # DESCRIPTION:
    # generic tools for enzyme application

    Copyright (C) 2007 Fabio Erculiani

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
'''

# Never do "import portage" here, please use entropyTools binding
# EXIT STATUSES: 400-499

from entropyConstants import *
from entropyTools import *

import sys
import os
import commands
import string

def sync(options):

    print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ..."))
    
    # firstly sync the packages
    packages(["sync" , "--ask"])
    # then sync the database
    database(["sync"])
    
    print_info(green(" * ")+red("Starting to collect packages that would be removed from the repository ..."), back = True)
    
    # now it's time to do some tidy
    # collect all the binaries in the database
    import databaseTools
    dbconn = databaseTools.etpDatabase(readOnly = True)
    dbBinaries = []
    pkglist = dbconn.listAllPackages()
    for pkg in pkglist:
	dl = dbconn.retrievePackageVar(pkg,"download")
	dl = dl.split("/")[len(dl.split("/"))-1]
	dbBinaries.append(dl)
    # filter dups?
    dbBinaries = list(set(dbBinaries))
    dbconn.closeDB()
    
    repoBinaries = os.listdir(etpConst['packagesbindir'])
    
    removeList = []
    # select packages
    for repoBin in repoBinaries:
	found = False
	for dbBin in dbBinaries:
	    if dbBin == repoBin:
		found = True
		break
	if (not found):
	    # then remove
	    removeList.append(repoBin)
    
    if (removeList == []):
	print_info(green(" * ")+red("No packages to remove from the mirrors."))
	print_info(green(" * ")+red("Syncronization across mirrors completed."))
	return
    
    print_info(green(" * ")+red("This is the list of the packages that would be removed from the mirrors: "))
    for file in removeList:
	print_info(green("\t* ")+yellow(file))
	
    # ask question
    rc = askquestion("     Would you like to continue ?")
    if rc == "No":
	sys.exit(0)

    # remove them!
    for uri in etpConst['activatoruploaduris']:
	print_info(green(" * ")+red("Connecting to: ")+bold(extractFTPHostFromUri(uri)))
	ftp = mirrorTools.handlerFTP(uri)
	ftp.setCWD(etpConst['binaryurirelativepath'])
	for file in removeList:
	    print_info(green(" * ")+red("Removing file: ")+bold(file), back = True)
	    # remove remotely
	    rc = ftp.deleteFile(file)
	    if (rc):
		print_info(green(" * ")+red("Package file: ")+bold(file)+red(" removed successfully."))
	    else:
		print_warning(yellow(" * ")+red("ATTENTION: remote file ")+bold(file)+red(" cannot be removed."))
	    # remove locally
	    if os.path.isfile(etpConst['packagesbindir']+"/"+file):
		os.remove(etpConst['packagesbindir']+"/"+file)
	ftp.closeFTPConnection()
	
    print_info(green(" * ")+red("Syncronization across mirrors completed."))


def packages(options):

    # Options available for all the packages submodules
    myopts = options[1:]
    activatorRequestAsk = False
    activatorRequestPretend = False
    for opt in myopts:
	if (opt == "--ask"):
	    activatorRequestAsk = True
	elif (opt == "--pretend"):
	    activatorRequestPretend = True

    if (options[0] == "sync"):
	print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ..."))
	for uri in etpConst['activatoruploaduris']:

	    print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror.")))
	    print_info(green(" * ")+yellow("Local Statistics"))
	    print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True)
	    uploadCounter = 0
	    toBeUploaded = [] # parse etpConst['packagessuploaddir']
	    for tbz2 in os.listdir(etpConst['packagessuploaddir']):
		if tbz2.endswith(".tbz2"):
		    toBeUploaded.append(tbz2)
		    uploadCounter += 1
	    print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
	    localPackagesRepository = [] # parse etpConst['packagesbindir']
	    print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
	    packageCounter = 0
	    for tbz2 in os.listdir(etpConst['packagesbindir']):
		if tbz2.endswith(".tbz2"):
		    localPackagesRepository.append(tbz2)
		    packageCounter += 1
	    print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
	    
	    print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
	    ftp = mirrorTools.handlerFTP(uri)
	    ftp.setCWD(etpConst['binaryurirelativepath'])
	    remotePackages = ftp.listFTPdir()
	    remotePackagesInfo = ftp.getRoughList()
	    ftp.closeFTPConnection()

	    print_info(green(" * ")+yellow("Remote statistics"))
	    remoteCounter = 0
	    for tbz2 in remotePackages:
		if tbz2.endswith(".tbz2"):
		    remoteCounter += 1
	    print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
	    
	    print_info(green(" * ")+yellow("Calculating..."))
	    uploadQueue = []
	    downloadQueue = []
	    removalQueue = []
	    
	    # Fill uploadQueue and if something weird is found, add the packages to downloadQueue
	    for localPackage in toBeUploaded:
		pkgfound = False
		for remotePackage in remotePackages:
		    if localPackage == remotePackage:
			pkgfound = True
			# it's already on the mirror, but... is its size correct??
			localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6])
			remoteSize = 0
			for file in remotePackagesInfo:
			    if file.split()[8] == remotePackage:
				remoteSize = int(file.split()[4])
			if (localSize != remoteSize) and (localSize != 0):
			    # size does not match, adding to the upload queue
			    uploadQueue.append(localPackage)
			break
		
		if (not pkgfound):
		    # this means that the local package does not exist
		    # so, we need to download it
		    uploadQueue.append(localPackage)
	    
	    # Fill downloadQueue and removalQueue
	    for remotePackage in remotePackages:
		pkgfound = False
		for localPackage in localPackagesRepository:
		    if localPackage == remotePackage:
			pkgfound = True
			# it's already on the mirror, but... is its size correct??
			localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6])
			remoteSize = 0
			for file in remotePackagesInfo:
			    if file.split()[8] == remotePackage:
				remoteSize = int(file.split()[4])
			if (localSize != remoteSize) and (localSize != 0):
			    # size does not match, remove first
			    removalQueue.append(localPackage)
			    # then add to the download queue
			    downloadQueue.append(remotePackage)
			break
		
		if (not pkgfound):
		    # this means that the local package does not exist
		    # so, we need to download it
		    downloadQueue.append(remotePackage)

	    # Fill removalQueue and downloadQueue
	    # if the online package does not exist anymore, we have to remove it locally
	    for localPackage in localPackagesRepository:
		pkgfound = False
		for remotePackage in remotePackages:
		    if localPackage == remotePackage:
			pkgfound = True
			break
		
		if (not pkgfound):
		    # this means that the local package does not exist
		    # so, we need to download it
		    removalQueue.append(localPackage)


	    # filter duplicates
	    removalQueue = list(set(removalQueue))
	    downloadQueue = list(set(downloadQueue))
	    uploadQueue = list(set(uploadQueue))

	    if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0):
		print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty.")))
		continue


	    totalRemovalSize = 0
	    totalDownloadSize = 0
	    totalUploadSize = 0

	    print_info(green(" * ")+yellow("Queue tasks:"))
	    detailedRemovalQueue = []
	    detailedDownloadQueue = []
	    detailedUploadQueue = []

	    for item in removalQueue:
		fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6]
		totalRemovalSize += int(fileSize)
		print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
		detailedRemovalQueue.append([item,fileSize])
	    import time
	    time.sleep(10)

	    for item in downloadQueue:
		# if the package is already in the upload directory, do not add the size
		if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item):
		    fileSize = "0"
		    for remotePackage in remotePackagesInfo:
		        if remotePackage.split()[8] == item:
			    fileSize = remotePackage.split()[4]
			    break
		    totalDownloadSize += int(fileSize)
		    print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
		    detailedDownloadQueue.append([item,fileSize])

	    for item in uploadQueue:
		fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
		totalUploadSize += int(fileSize)
		print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
		detailedUploadQueue.append([item,fileSize])

	    print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue))))
	    print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue))))
	    print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue))))
	    print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize))))
	    print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize))))
	    print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
	    
	    if (activatorRequestAsk):
		rc = askquestion("\n     Would you like to run the steps above ?")
		if rc == "No":
		    print "\n"
		    continue
	    elif (activatorRequestPretend):
		continue

	    # removal queue
	    if (detailedRemovalQueue != []):
		for item in detailedRemovalQueue:
		    print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True)
		    os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0])
		print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']))

	    # upload queue
	    if (detailedUploadQueue != []):
	        ftp = mirrorTools.handlerFTP(uri)
	        ftp.setCWD(etpConst['binaryurirelativepath'])
		uploadCounter = str(len(detailedUploadQueue))
		currentCounter = 0
		for item in detailedUploadQueue:
		    currentCounter += 1
		    counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")")
		    print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
		    ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
		print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
		ftp.closeFTPConnection()

	    # download queue
	    if (detailedDownloadQueue != []):
	        ftp = mirrorTools.handlerFTP(uri)
	        ftp.setCWD(etpConst['binaryurirelativepath'])
		downloadCounter = str(len(detailedDownloadQueue))
		currentCounter = 0
		for item in detailedDownloadQueue:
		    currentCounter += 1
		    counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")")
		    if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]):
			localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6])
			remoteSize = int(item[1])
			if localSize == remoteSize:
			    # skip that, we'll move at the end of the mirrors sync
			    continue
		    print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
		    ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
		print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri)))
		ftp.closeFTPConnection()

	# now we can store the files in upload/%ARCH% in packages/%ARCH%
	os.system("mv -f "+etpConst['packagessuploaddir']+"/* "+etpConst['packagesbindir']+"/ &> /dev/null")

def database(options):

    # lock tool
    if (options[0] == "lock"):
	print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
	rc = lockDatabases(lock = True)
	if (rc):
	    print_info(green(" * ")+green("A problem occured on at least one mirror..."))
	else:
	    print_info(green(" * ")+green("Databases lock complete"))

    # unlock tool
    elif (options[0] == "unlock"):
	print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
	rc = lockDatabases(lock = False)
	if (rc):
	    print_info(green(" * ")+green("A problem occured on at least one mirror..."))
	else:
	    print_info(green(" * ")+green("Databases unlock complete"))

    # download lock tool
    elif (options[0] == "download-lock"):
	print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
	rc = downloadLockDatabases(lock = True)
	if (rc):
	    print_info(green(" * ")+green("A problem occured on at least one mirror..."))
	else:
	    print_info(green(" * ")+green("Download mirrors lock complete"))

    # download unlock tool
    elif (options[0] == "download-unlock"):
	print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
	rc = downloadLockDatabases(lock = False)
	if (rc):
	    print_info(green(" * ")+green("A problem occured on at least one mirror..."))
	else:
	    print_info(green(" * ")+green("Download mirrors unlock complete"))

    # lock status tool
    elif (options[0] == "lock-status"):
	print_info(yellow(" * ")+green("Mirrors status table:"))
	dbstatus = getMirrorsLock()
	for db in dbstatus:
	    if (db[1]):
	        db[1] = red("Locked")
	    else:
	        db[1] = green("Unlocked")
	    if (db[2]):
	        db[2] = red("Locked")
	    else:
	        db[2] = green("Unlocked")
	    print_info(bold("\t"+extractFTPHostFromUri(db[0])+": ")+red("[")+yellow("DATABASE: ")+db[1]+red("] [")+yellow("DOWNLOAD: ")+db[2]+red("]"))

    # database sync tool
    elif (options[0] == "sync"):
	
	print_info(green(" * ")+red("Checking database status ..."), back = True)
	
	dbLockFile = False
	# does the taint file exist?
	if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']):
	    dbLockFile = True
	
	# are online mirrors locked?
	mirrorsLocked = False
	for uri in etpConst['activatoruploaduris']:
	    ftp = mirrorTools.handlerFTP(uri)
	    ftp.setCWD(etpConst['etpurirelativepath'])
	    if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) or (ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile'])):
		mirrorsLocked = True
		ftp.closeFTPConnection()
		break
	
	if (mirrorsLocked):
	    # if the mirrors are locked, we need to change if we have
	    # the taint file in place. Because in this case, the one
	    # that tainted the db, was me.
	    if (dbLockFile):
		print_info(green(" * ")+red("Updating mirrors with new information ..."))
		# it's safe to sync
	        syncRemoteDatabases()
		# remove the online lock file
		lockDatabases(False)
		# remove the taint file
		if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']):
		    os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
	    else:
		print
		print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ..."))
		sys.exit(422)

    else:
	print_error(red(" * ")+green("No valid tool specified."))
	sys.exit(400)