some daily fixes
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@273 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
@@ -1,17 +1,18 @@
|
||||
TODO list (for developers only):
|
||||
- build() and world(), on enzyme, add the support for whitelist+cron
|
||||
- build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form)
|
||||
- reagent: get orphans working and not taking forever
|
||||
- entropy: add md5 checksum to every tbz2 package
|
||||
- improve logTools and put it in an usable status
|
||||
|
||||
- even binary packages repo should have the upload lock?
|
||||
- reagent: add a tool that checks the checksum of the packages in the database
|
||||
- reagent: complete smartapps section
|
||||
- enzyme, build(), save packages built in a file instead of a variable (for power down issues)
|
||||
- Sabayon Linux USE flags: remove all server related use flags
|
||||
|
||||
Project Status:
|
||||
- entropy: will handle all the three tools below
|
||||
- enzyme: first release up and working (more or less)
|
||||
- reagent: first release up and working.
|
||||
- activator: 95%
|
||||
- enzyme: complete. Needs bug fixes ?
|
||||
- reagent: complete. Needs bug fixes.
|
||||
- activator: complete. Needs bug fixes.
|
||||
============
|
||||
- client part: not yet defined apart from equilibrium name.
|
||||
|
||||
|
||||
+3
-1
@@ -47,6 +47,7 @@ def print_help():
|
||||
entropyTools.print_info(" \t\t"+entropyTools.red("--force-bump")+"\t\t to force the revision bumping of the package entries")
|
||||
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t Entropy database tool manager."))
|
||||
entropyTools.print_info(" \t\t"+entropyTools.red("--initialize")+"\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.red("statistics")+"\t\t Show Entropy database statistics.")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("search")+"\t\t\t Search a package inside the Entropy packages database")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("dump-package-info")+"\t Dump the stored information of package atom(s) to a file")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("inject-package-info")+"\t Inject an Entropy dump inside the database (file path is mandatory)")
|
||||
@@ -55,7 +56,8 @@ def print_help():
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("create-empty-database")+"\t Create an empty Entropy database file in the specified <path>")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("stabilize")+"\t\t Mark as stable a package, a list of packages, world")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("unstabilize")+"\t\t Mark as unstable a package, a list of packages, world")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green("md5check")+"\t\t Check digest of a package, a list of packages, world")
|
||||
#entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package")
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("sanity-check"))+"\t\t Do a quick check on the database to assure data integrity")
|
||||
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("smartapps"))+entropyTools.yellow("\t Entropy application tool to create self-dependant applications [EXPERIMENTAL]"))
|
||||
entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("create"))+"\t\t\t Create a smartapp package using the package names provided")
|
||||
|
||||
+215
-190
@@ -117,209 +117,225 @@ def packages(options):
|
||||
elif (opt == "--pretend"):
|
||||
activatorRequestPretend = True
|
||||
|
||||
|
||||
# except:
|
||||
# print_warning(yellow(" * ")+red("Cannot Syncronize binary packages !!! Pay attention to what you do."), back = True)
|
||||
# import time
|
||||
# time.sleep(5)
|
||||
|
||||
|
||||
if (options[0] == "sync"):
|
||||
print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ..."))
|
||||
|
||||
syncSuccessful = False
|
||||
|
||||
for uri in etpConst['activatoruploaduris']:
|
||||
|
||||
print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror.")))
|
||||
print_info(green(" * ")+yellow("Local Statistics"))
|
||||
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True)
|
||||
uploadCounter = 0
|
||||
toBeUploaded = [] # parse etpConst['packagessuploaddir']
|
||||
for tbz2 in os.listdir(etpConst['packagessuploaddir']):
|
||||
if tbz2.endswith(".tbz2"):
|
||||
toBeUploaded.append(tbz2)
|
||||
uploadCounter += 1
|
||||
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
|
||||
localPackagesRepository = [] # parse etpConst['packagesbindir']
|
||||
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
|
||||
packageCounter = 0
|
||||
for tbz2 in os.listdir(etpConst['packagesbindir']):
|
||||
if tbz2.endswith(".tbz2"):
|
||||
localPackagesRepository.append(tbz2)
|
||||
packageCounter += 1
|
||||
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
|
||||
|
||||
print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
remotePackages = ftp.listFTPdir()
|
||||
remotePackagesInfo = ftp.getRoughList()
|
||||
ftp.closeFTPConnection()
|
||||
|
||||
print_info(green(" * ")+yellow("Remote statistics"))
|
||||
remoteCounter = 0
|
||||
for tbz2 in remotePackages:
|
||||
if tbz2.endswith(".tbz2"):
|
||||
remoteCounter += 1
|
||||
print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
|
||||
|
||||
print_info(green(" * ")+yellow("Calculating..."))
|
||||
uploadQueue = []
|
||||
downloadQueue = []
|
||||
removalQueue = []
|
||||
|
||||
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
|
||||
for localPackage in toBeUploaded:
|
||||
pkgfound = False
|
||||
for remotePackage in remotePackages:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
# it's already on the mirror, but... is its size correct??
|
||||
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6])
|
||||
remoteSize = 0
|
||||
for file in remotePackagesInfo:
|
||||
if file.split()[8] == remotePackage:
|
||||
remoteSize = int(file.split()[4])
|
||||
if (localSize != remoteSize) and (localSize != 0):
|
||||
# size does not match, adding to the upload queue
|
||||
uploadQueue.append(localPackage)
|
||||
break
|
||||
try:
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
uploadQueue.append(localPackage)
|
||||
print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror.")))
|
||||
print_info(green(" * ")+yellow("Local Statistics"))
|
||||
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True)
|
||||
uploadCounter = 0
|
||||
toBeUploaded = [] # parse etpConst['packagessuploaddir']
|
||||
for tbz2 in os.listdir(etpConst['packagessuploaddir']):
|
||||
if tbz2.endswith(".tbz2"):
|
||||
toBeUploaded.append(tbz2)
|
||||
uploadCounter += 1
|
||||
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
|
||||
localPackagesRepository = [] # parse etpConst['packagesbindir']
|
||||
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
|
||||
packageCounter = 0
|
||||
for tbz2 in os.listdir(etpConst['packagesbindir']):
|
||||
if tbz2.endswith(".tbz2"):
|
||||
localPackagesRepository.append(tbz2)
|
||||
packageCounter += 1
|
||||
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
|
||||
|
||||
# Fill downloadQueue and removalQueue
|
||||
for remotePackage in remotePackages:
|
||||
pkgfound = False
|
||||
for localPackage in localPackagesRepository:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
# it's already on the mirror, but... is its size correct??
|
||||
localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6])
|
||||
remoteSize = 0
|
||||
for file in remotePackagesInfo:
|
||||
if file.split()[8] == remotePackage:
|
||||
remoteSize = int(file.split()[4])
|
||||
if (localSize != remoteSize) and (localSize != 0):
|
||||
# size does not match, remove first
|
||||
removalQueue.append(localPackage)
|
||||
# then add to the download queue
|
||||
downloadQueue.append(remotePackage)
|
||||
break
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
downloadQueue.append(remotePackage)
|
||||
print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
remotePackages = ftp.listFTPdir()
|
||||
remotePackagesInfo = ftp.getRoughList()
|
||||
ftp.closeFTPConnection()
|
||||
|
||||
# Fill removalQueue and downloadQueue
|
||||
# if the online package does not exist anymore, we have to remove it locally
|
||||
for localPackage in localPackagesRepository:
|
||||
pkgfound = False
|
||||
for remotePackage in remotePackages:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
break
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
removalQueue.append(localPackage)
|
||||
|
||||
|
||||
# filter duplicates
|
||||
removalQueue = list(set(removalQueue))
|
||||
downloadQueue = list(set(downloadQueue))
|
||||
uploadQueue = list(set(uploadQueue))
|
||||
|
||||
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0):
|
||||
print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty.")))
|
||||
continue
|
||||
|
||||
|
||||
totalRemovalSize = 0
|
||||
totalDownloadSize = 0
|
||||
totalUploadSize = 0
|
||||
|
||||
print_info(green(" * ")+yellow("Queue tasks:"))
|
||||
detailedRemovalQueue = []
|
||||
detailedDownloadQueue = []
|
||||
detailedUploadQueue = []
|
||||
|
||||
for item in removalQueue:
|
||||
fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6]
|
||||
totalRemovalSize += int(fileSize)
|
||||
print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedRemovalQueue.append([item,fileSize])
|
||||
import time
|
||||
time.sleep(10)
|
||||
|
||||
for item in downloadQueue:
|
||||
# if the package is already in the upload directory, do not add the size
|
||||
if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item):
|
||||
fileSize = "0"
|
||||
for remotePackage in remotePackagesInfo:
|
||||
if remotePackage.split()[8] == item:
|
||||
fileSize = remotePackage.split()[4]
|
||||
print_info(green(" * ")+yellow("Remote statistics"))
|
||||
remoteCounter = 0
|
||||
for tbz2 in remotePackages:
|
||||
if tbz2.endswith(".tbz2"):
|
||||
remoteCounter += 1
|
||||
print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
|
||||
|
||||
print_info(green(" * ")+yellow("Calculating..."))
|
||||
uploadQueue = []
|
||||
downloadQueue = []
|
||||
removalQueue = []
|
||||
|
||||
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
|
||||
for localPackage in toBeUploaded:
|
||||
pkgfound = False
|
||||
for remotePackage in remotePackages:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
# it's already on the mirror, but... is its size correct??
|
||||
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6])
|
||||
remoteSize = 0
|
||||
for file in remotePackagesInfo:
|
||||
if file.split()[8] == remotePackage:
|
||||
remoteSize = int(file.split()[4])
|
||||
if (localSize != remoteSize) and (localSize != 0):
|
||||
# size does not match, adding to the upload queue
|
||||
uploadQueue.append(localPackage)
|
||||
break
|
||||
totalDownloadSize += int(fileSize)
|
||||
print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedDownloadQueue.append([item,fileSize])
|
||||
|
||||
for item in uploadQueue:
|
||||
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
|
||||
totalUploadSize += int(fileSize)
|
||||
print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedUploadQueue.append([item,fileSize])
|
||||
|
||||
print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue))))
|
||||
print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue))))
|
||||
print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue))))
|
||||
print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize))))
|
||||
print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize))))
|
||||
print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
uploadQueue.append(localPackage)
|
||||
|
||||
if (activatorRequestAsk):
|
||||
rc = askquestion("\n Would you like to run the steps above ?")
|
||||
if rc == "No":
|
||||
print "\n"
|
||||
# Fill downloadQueue and removalQueue
|
||||
for remotePackage in remotePackages:
|
||||
pkgfound = False
|
||||
for localPackage in localPackagesRepository:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
# it's already on the mirror, but... is its size correct??
|
||||
localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6])
|
||||
remoteSize = 0
|
||||
for file in remotePackagesInfo:
|
||||
if file.split()[8] == remotePackage:
|
||||
remoteSize = int(file.split()[4])
|
||||
if (localSize != remoteSize) and (localSize != 0):
|
||||
# size does not match, remove first
|
||||
removalQueue.append(localPackage)
|
||||
# then add to the download queue
|
||||
downloadQueue.append(remotePackage)
|
||||
break
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
downloadQueue.append(remotePackage)
|
||||
|
||||
# Fill removalQueue and downloadQueue
|
||||
# if the online package does not exist anymore, we have to remove it locally
|
||||
for localPackage in localPackagesRepository:
|
||||
pkgfound = False
|
||||
for remotePackage in remotePackages:
|
||||
if localPackage == remotePackage:
|
||||
pkgfound = True
|
||||
break
|
||||
|
||||
if (not pkgfound):
|
||||
# this means that the local package does not exist
|
||||
# so, we need to download it
|
||||
removalQueue.append(localPackage)
|
||||
|
||||
|
||||
# filter duplicates
|
||||
removalQueue = list(set(removalQueue))
|
||||
downloadQueue = list(set(downloadQueue))
|
||||
uploadQueue = list(set(uploadQueue))
|
||||
|
||||
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0):
|
||||
print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty.")))
|
||||
continue
|
||||
elif (activatorRequestPretend):
|
||||
continue
|
||||
|
||||
# removal queue
|
||||
if (detailedRemovalQueue != []):
|
||||
for item in detailedRemovalQueue:
|
||||
print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True)
|
||||
os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0])
|
||||
print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']))
|
||||
|
||||
# upload queue
|
||||
if (detailedUploadQueue != []):
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
uploadCounter = str(len(detailedUploadQueue))
|
||||
currentCounter = 0
|
||||
for item in detailedUploadQueue:
|
||||
currentCounter += 1
|
||||
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")")
|
||||
print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
|
||||
ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
|
||||
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
|
||||
ftp.closeFTPConnection()
|
||||
totalRemovalSize = 0
|
||||
totalDownloadSize = 0
|
||||
totalUploadSize = 0
|
||||
|
||||
print_info(green(" * ")+yellow("Queue tasks:"))
|
||||
detailedRemovalQueue = []
|
||||
detailedDownloadQueue = []
|
||||
detailedUploadQueue = []
|
||||
|
||||
for item in removalQueue:
|
||||
fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6]
|
||||
totalRemovalSize += int(fileSize)
|
||||
print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedRemovalQueue.append([item,fileSize])
|
||||
import time
|
||||
time.sleep(10)
|
||||
|
||||
for item in downloadQueue:
|
||||
# if the package is already in the upload directory, do not add the size
|
||||
if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item):
|
||||
fileSize = "0"
|
||||
for remotePackage in remotePackagesInfo:
|
||||
if remotePackage.split()[8] == item:
|
||||
fileSize = remotePackage.split()[4]
|
||||
break
|
||||
totalDownloadSize += int(fileSize)
|
||||
print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedDownloadQueue.append([item,fileSize])
|
||||
|
||||
for item in uploadQueue:
|
||||
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
|
||||
totalUploadSize += int(fileSize)
|
||||
print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
|
||||
detailedUploadQueue.append([item,fileSize])
|
||||
|
||||
print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue))))
|
||||
print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue))))
|
||||
print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue))))
|
||||
print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize))))
|
||||
print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize))))
|
||||
print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
|
||||
|
||||
if (activatorRequestAsk):
|
||||
rc = askquestion("\n Would you like to run the steps above ?")
|
||||
if rc == "No":
|
||||
print "\n"
|
||||
continue
|
||||
elif (activatorRequestPretend):
|
||||
continue
|
||||
|
||||
# removal queue
|
||||
if (detailedRemovalQueue != []):
|
||||
for item in detailedRemovalQueue:
|
||||
print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True)
|
||||
os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0])
|
||||
print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']))
|
||||
|
||||
# upload queue
|
||||
if (detailedUploadQueue != []):
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
uploadCounter = str(len(detailedUploadQueue))
|
||||
currentCounter = 0
|
||||
for item in detailedUploadQueue:
|
||||
currentCounter += 1
|
||||
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")")
|
||||
print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
|
||||
ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
|
||||
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
|
||||
ftp.closeFTPConnection()
|
||||
|
||||
# download queue
|
||||
if (detailedDownloadQueue != []):
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
downloadCounter = str(len(detailedDownloadQueue))
|
||||
currentCounter = 0
|
||||
for item in detailedDownloadQueue:
|
||||
currentCounter += 1
|
||||
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")")
|
||||
if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]):
|
||||
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6])
|
||||
remoteSize = int(item[1])
|
||||
if localSize == remoteSize:
|
||||
# skip that, we'll move at the end of the mirrors sync
|
||||
continue
|
||||
print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
|
||||
ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
|
||||
print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri)))
|
||||
ftp.closeFTPConnection()
|
||||
|
||||
# at least one sync ran successfully
|
||||
syncSuccessful = True
|
||||
|
||||
# download queue
|
||||
if (detailedDownloadQueue != []):
|
||||
ftp = mirrorTools.handlerFTP(uri)
|
||||
ftp.setCWD(etpConst['binaryurirelativepath'])
|
||||
downloadCounter = str(len(detailedDownloadQueue))
|
||||
currentCounter = 0
|
||||
for item in detailedDownloadQueue:
|
||||
currentCounter += 1
|
||||
counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")")
|
||||
if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]):
|
||||
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6])
|
||||
remoteSize = int(item[1])
|
||||
if localSize == remoteSize:
|
||||
# skip that, we'll move at the end of the mirrors sync
|
||||
continue
|
||||
print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
|
||||
ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
|
||||
print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri)))
|
||||
ftp.closeFTPConnection()
|
||||
|
||||
# now we can store the files in upload/%ARCH% in packages/%ARCH%
|
||||
os.system("mv -f "+etpConst['packagessuploaddir']+"/* "+etpConst['packagesbindir']+"/ &> /dev/null")
|
||||
@@ -414,6 +430,15 @@ def database(options):
|
||||
print
|
||||
print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ..."))
|
||||
sys.exit(422)
|
||||
|
||||
else:
|
||||
if (dbLockFile):
|
||||
print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']))
|
||||
sys.exit(423)
|
||||
else:
|
||||
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
|
||||
|
||||
syncRemoteDatabases()
|
||||
|
||||
else:
|
||||
print_error(red(" * ")+green("No valid tool specified."))
|
||||
|
||||
+85
-18
@@ -400,6 +400,7 @@ def database(options):
|
||||
entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("All the selected packages have been marked as requested. Have fun."))
|
||||
dbconn.closeDB()
|
||||
|
||||
# FIXME: not working function... remove?
|
||||
elif (options[0] == "orphans"):
|
||||
entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Collecting files in database..."))
|
||||
# FIXME: complete this!
|
||||
@@ -533,6 +534,66 @@ def database(options):
|
||||
entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("All the selected packages have been removed as requested. Have fun."))
|
||||
dbconn.closeDB()
|
||||
|
||||
# used by reagent
|
||||
elif (options[0] == "statistics"):
|
||||
entropyTools.print_info(entropyTools.green(" [LOCAL DB STATISTIC]\t\t")+entropyTools.red("Information"))
|
||||
# fetch total packages
|
||||
dbconn = etpDatabase(readOnly = True)
|
||||
totalpkgs = len(dbconn.listAllPackages())
|
||||
totalstablepkgs = len(dbconn.listStablePackages())
|
||||
totalunstablepkgs = len(dbconn.listUnstablePackages())
|
||||
entropyTools.print_info(entropyTools.green(" Total Installed Packages\t\t")+entropyTools.red(str(totalpkgs)))
|
||||
entropyTools.print_info(entropyTools.green(" Total Stable Packages\t\t")+entropyTools.red(str(totalstablepkgs)))
|
||||
entropyTools.print_info(entropyTools.green(" Total Unstable Packages\t\t")+entropyTools.red(str(totalunstablepkgs)))
|
||||
entropyTools.syncRemoteDatabases(justStats = True)
|
||||
dbconn.closeDB()
|
||||
|
||||
# used by reagent
|
||||
elif (options[0] == "md5check"):
|
||||
|
||||
entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Integrity verification of the selected packages:"))
|
||||
|
||||
mypackages = options[1:]
|
||||
dbconn = etpDatabase(readOnly = True)
|
||||
|
||||
if (len(mypackages) == 0):
|
||||
# check world
|
||||
# create packages list
|
||||
pkgs2check = dbconn.listAllPackages()
|
||||
elif (mypackages[0] == "world"):
|
||||
# check world
|
||||
# create packages list
|
||||
pkgs2check = dbconn.listAllPackages()
|
||||
else:
|
||||
# catch the names
|
||||
pkgs2check = []
|
||||
for pkg in mypackages:
|
||||
results = dbconn.searchPackages(pkg)
|
||||
for i in results:
|
||||
pkgs2check.append(i[0])
|
||||
|
||||
entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:"))
|
||||
|
||||
toBeDownloaded = []
|
||||
availList = []
|
||||
for i in pkgs2check:
|
||||
pkgfile = dbconn.retrievePackageVar(i,"download")
|
||||
pkgfile = pkgfile.split("/")[len(pkgfile.split("/"))-1]
|
||||
if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)):
|
||||
entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i))
|
||||
availList.append(pkgfile)
|
||||
elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)):
|
||||
entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i))
|
||||
else:
|
||||
entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i))
|
||||
toBeDownloaded.append(pkgfile)
|
||||
|
||||
# FIXME add download support
|
||||
# FIXME complete this
|
||||
|
||||
|
||||
dbconn.closeDB()
|
||||
|
||||
############
|
||||
# Functions and Classes
|
||||
#####################################################################################
|
||||
@@ -856,10 +917,10 @@ class etpDatabase:
|
||||
def retrievePackageVar(self,pkgkey,pkgvar):
|
||||
pkgkey = entropyTools.removePackageOperators(pkgkey)
|
||||
result = []
|
||||
self.cursor.execute('SELECT '+pkgvar+' FROM etpData WHERE atom LIKE "'+pkgkey+'"')
|
||||
self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'"')
|
||||
for row in self.cursor:
|
||||
result.append(row)
|
||||
return result[0][0]
|
||||
result.append(row[0])
|
||||
return result[0]
|
||||
|
||||
# You must provide the full atom to this function
|
||||
def isPackageAvailable(self,pkgkey):
|
||||
@@ -905,19 +966,6 @@ class etpDatabase:
|
||||
result.append(row[0])
|
||||
return result
|
||||
|
||||
def searchStablePackages(self,keyword):
|
||||
results = []
|
||||
self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "%'+keyword+'%"')
|
||||
for row in self.cursor:
|
||||
results.append(row)
|
||||
output = []
|
||||
for result in results:
|
||||
result = result[0]
|
||||
self.cursor.execute('SELECT '+result+' FROM etpData WHERE branch = "stable"')
|
||||
for row in self.cursor:
|
||||
output.append(row[0])
|
||||
return output
|
||||
|
||||
def listUnstablePackages(self):
|
||||
result = []
|
||||
self.cursor.execute('SELECT * FROM etpData WHERE branch = "unstable"')
|
||||
@@ -925,6 +973,24 @@ class etpDatabase:
|
||||
result.append(row[0])
|
||||
return result
|
||||
|
||||
def searchStablePackages(self,atom):
|
||||
category = atom.split("/")[0]
|
||||
name = atom.split("/")[1]
|
||||
result = []
|
||||
self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "stable"')
|
||||
for row in self.cursor:
|
||||
result.append(row[0])
|
||||
return result
|
||||
|
||||
def searchUnstablePackages(self,atom):
|
||||
category = atom.split("/")[0]
|
||||
name = atom.split("/")[1]
|
||||
result = []
|
||||
self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "stable"')
|
||||
for row in self.cursor:
|
||||
result.append(row[0])
|
||||
return result
|
||||
|
||||
# useful to quickly retrieve (and trash) all the data
|
||||
# and look for problems.
|
||||
def noopCycle(self):
|
||||
@@ -945,9 +1011,10 @@ class etpDatabase:
|
||||
removelist = []
|
||||
for result in results:
|
||||
# have a look if the slot matches
|
||||
myslot = self.retrievePackageVar(result[0],"slot")
|
||||
#print result
|
||||
myslot = self.retrievePackageVar(result,"slot")
|
||||
if (myslot == slot):
|
||||
removelist.append(result[0])
|
||||
removelist.append(result)
|
||||
for pkg in removelist:
|
||||
self.removePackage(pkg)
|
||||
|
||||
|
||||
@@ -81,6 +81,12 @@ def md5sum(filepath):
|
||||
block = readfile.read(1024)
|
||||
return m.hexdigest()
|
||||
|
||||
def compareMd5(filepath,md5sum):
|
||||
result = md5sum(filepath)
|
||||
if md5sum == result:
|
||||
return True
|
||||
return False
|
||||
|
||||
def md5string(string):
|
||||
import md5
|
||||
m = md5.new()
|
||||
@@ -265,9 +271,8 @@ def getEtpRemoteDatabaseStatus():
|
||||
|
||||
return uriDbInfo
|
||||
|
||||
def syncRemoteDatabases(noUpload = False):
|
||||
def syncRemoteDatabases(noUpload = False, justStats = False):
|
||||
|
||||
print_info(green(" * ")+red("Checking the status of the remote Entropy Database Repository"))
|
||||
remoteDbsStatus = getEtpRemoteDatabaseStatus()
|
||||
print_info(green(" * ")+red("Remote Entropy Database Repository Status:"))
|
||||
for dbstat in remoteDbsStatus:
|
||||
@@ -286,6 +291,9 @@ def syncRemoteDatabases(noUpload = False):
|
||||
|
||||
print_info(red("\t * Database local revision currently at: ")+blue(str(etpDbLocalRevision)))
|
||||
|
||||
if (justStats):
|
||||
return
|
||||
|
||||
downloadLatest = []
|
||||
uploadLatest = False
|
||||
uploadList = []
|
||||
|
||||
@@ -377,7 +377,15 @@ def build(atoms):
|
||||
rc = askquestion("\n Would you like to run the steps above ?")
|
||||
if rc == "No":
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
# sync binary packages repository
|
||||
import activatorTools
|
||||
if (enzymeRequestInteraction):
|
||||
activatorTools.packages(["sync" , "--ask"])
|
||||
else:
|
||||
activatorTools.packages(["sync"])
|
||||
|
||||
|
||||
# when the compilation ends, enzyme runs reagent
|
||||
packagesPaths = []
|
||||
|
||||
|
||||
@@ -147,6 +147,7 @@ class handlerFTP:
|
||||
except:
|
||||
return False
|
||||
|
||||
# FIXME: add progress
|
||||
def uploadFile(self,file,ascii = False):
|
||||
for i in range(10): # ten tries
|
||||
f = open(file)
|
||||
|
||||
Reference in New Issue
Block a user