From f76a55cbfb0b2135327ebc459558d71066c0eefb Mon Sep 17 00:00:00 2001 From: lxnay Date: Sun, 8 Apr 2007 13:19:56 +0000 Subject: [PATCH] some daily fixes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@273 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 13 +- handlers/reagent | 4 +- libraries/activatorTools.py | 405 +++++++++++++++++++----------------- libraries/databaseTools.py | 103 +++++++-- libraries/entropyTools.py | 12 +- libraries/enzymeTools.py | 10 +- libraries/mirrorTools.py | 1 + 7 files changed, 330 insertions(+), 218 deletions(-) diff --git a/TODO b/TODO index c18d846d1..7dab85fbb 100644 --- a/TODO +++ b/TODO @@ -1,17 +1,18 @@ TODO list (for developers only): - build() and world(), on enzyme, add the support for whitelist+cron - build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form) -- reagent: get orphans working and not taking forever -- entropy: add md5 checksum to every tbz2 package - improve logTools and put it in an usable status - +- even binary packages repo should have the upload lock? +- reagent: add a tool that checks the checksum of the packages in the database +- reagent: complete smartapps section +- enzyme, build(), save packages built in a file instead of a variable (for power down issues) - Sabayon Linux USE flags: remove all server related use flags Project Status: - entropy: will handle all the three tools below -- enzyme: first release up and working (more or less) -- reagent: first release up and working. -- activator: 95% +- enzyme: complete. Needs bug fixes ? +- reagent: complete. Needs bug fixes. +- activator: complete. Needs bug fixes. ============ - client part: not yet defined apart from equilibrium name. diff --git a/handlers/reagent b/handlers/reagent index 59f9c3c31..5deb057db 100644 --- a/handlers/reagent +++ b/handlers/reagent @@ -47,6 +47,7 @@ def print_help(): entropyTools.print_info(" \t\t"+entropyTools.red("--force-bump")+"\t\t to force the revision bumping of the package entries") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t Entropy database tool manager.")) entropyTools.print_info(" \t\t"+entropyTools.red("--initialize")+"\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]") + entropyTools.print_info(" \t\t"+entropyTools.red("statistics")+"\t\t Show Entropy database statistics.") entropyTools.print_info(" \t\t"+entropyTools.green("search")+"\t\t\t Search a package inside the Entropy packages database") entropyTools.print_info(" \t\t"+entropyTools.green("dump-package-info")+"\t Dump the stored information of package atom(s) to a file") entropyTools.print_info(" \t\t"+entropyTools.green("inject-package-info")+"\t Inject an Entropy dump inside the database (file path is mandatory)") @@ -55,7 +56,8 @@ def print_help(): entropyTools.print_info(" \t\t"+entropyTools.green("create-empty-database")+"\t Create an empty Entropy database file in the specified ") entropyTools.print_info(" \t\t"+entropyTools.green("stabilize")+"\t\t Mark as stable a package, a list of packages, world") entropyTools.print_info(" \t\t"+entropyTools.green("unstabilize")+"\t\t Mark as unstable a package, a list of packages, world") - entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package") + entropyTools.print_info(" \t\t"+entropyTools.green("md5check")+"\t\t Check digest of a package, a list of packages, world") + #entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package") entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("sanity-check"))+"\t\t Do a quick check on the database to assure data integrity") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("smartapps"))+entropyTools.yellow("\t Entropy application tool to create self-dependant applications [EXPERIMENTAL]")) entropyTools.print_info(" \t\t"+entropyTools.green(entropyTools.bold("create"))+"\t\t\t Create a smartapp package using the package names provided") diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 8288aa4f6..f5e4ab3d6 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -117,209 +117,225 @@ def packages(options): elif (opt == "--pretend"): activatorRequestPretend = True + +# except: +# print_warning(yellow(" * ")+red("Cannot Syncronize binary packages !!! Pay attention to what you do."), back = True) +# import time +# time.sleep(5) + + if (options[0] == "sync"): print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ...")) + + syncSuccessful = False + for uri in etpConst['activatoruploaduris']: - print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror."))) - print_info(green(" * ")+yellow("Local Statistics")) - print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True) - uploadCounter = 0 - toBeUploaded = [] # parse etpConst['packagessuploaddir'] - for tbz2 in os.listdir(etpConst['packagessuploaddir']): - if tbz2.endswith(".tbz2"): - toBeUploaded.append(tbz2) - uploadCounter += 1 - print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) - localPackagesRepository = [] # parse etpConst['packagesbindir'] - print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True) - packageCounter = 0 - for tbz2 in os.listdir(etpConst['packagesbindir']): - if tbz2.endswith(".tbz2"): - localPackagesRepository.append(tbz2) - packageCounter += 1 - print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) - - print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) - ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['binaryurirelativepath']) - remotePackages = ftp.listFTPdir() - remotePackagesInfo = ftp.getRoughList() - ftp.closeFTPConnection() - - print_info(green(" * ")+yellow("Remote statistics")) - remoteCounter = 0 - for tbz2 in remotePackages: - if tbz2.endswith(".tbz2"): - remoteCounter += 1 - print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored.")) - - print_info(green(" * ")+yellow("Calculating...")) - uploadQueue = [] - downloadQueue = [] - removalQueue = [] - - # Fill uploadQueue and if something weird is found, add the packages to downloadQueue - for localPackage in toBeUploaded: - pkgfound = False - for remotePackage in remotePackages: - if localPackage == remotePackage: - pkgfound = True - # it's already on the mirror, but... is its size correct?? - localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6]) - remoteSize = 0 - for file in remotePackagesInfo: - if file.split()[8] == remotePackage: - remoteSize = int(file.split()[4]) - if (localSize != remoteSize) and (localSize != 0): - # size does not match, adding to the upload queue - uploadQueue.append(localPackage) - break + try: - if (not pkgfound): - # this means that the local package does not exist - # so, we need to download it - uploadQueue.append(localPackage) + print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror."))) + print_info(green(" * ")+yellow("Local Statistics")) + print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True) + uploadCounter = 0 + toBeUploaded = [] # parse etpConst['packagessuploaddir'] + for tbz2 in os.listdir(etpConst['packagessuploaddir']): + if tbz2.endswith(".tbz2"): + toBeUploaded.append(tbz2) + uploadCounter += 1 + print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) + localPackagesRepository = [] # parse etpConst['packagesbindir'] + print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True) + packageCounter = 0 + for tbz2 in os.listdir(etpConst['packagesbindir']): + if tbz2.endswith(".tbz2"): + localPackagesRepository.append(tbz2) + packageCounter += 1 + print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) - # Fill downloadQueue and removalQueue - for remotePackage in remotePackages: - pkgfound = False - for localPackage in localPackagesRepository: - if localPackage == remotePackage: - pkgfound = True - # it's already on the mirror, but... is its size correct?? - localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6]) - remoteSize = 0 - for file in remotePackagesInfo: - if file.split()[8] == remotePackage: - remoteSize = int(file.split()[4]) - if (localSize != remoteSize) and (localSize != 0): - # size does not match, remove first - removalQueue.append(localPackage) - # then add to the download queue - downloadQueue.append(remotePackage) - break - - if (not pkgfound): - # this means that the local package does not exist - # so, we need to download it - downloadQueue.append(remotePackage) + print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) + ftp = mirrorTools.handlerFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + remotePackages = ftp.listFTPdir() + remotePackagesInfo = ftp.getRoughList() + ftp.closeFTPConnection() - # Fill removalQueue and downloadQueue - # if the online package does not exist anymore, we have to remove it locally - for localPackage in localPackagesRepository: - pkgfound = False - for remotePackage in remotePackages: - if localPackage == remotePackage: - pkgfound = True - break - - if (not pkgfound): - # this means that the local package does not exist - # so, we need to download it - removalQueue.append(localPackage) - - - # filter duplicates - removalQueue = list(set(removalQueue)) - downloadQueue = list(set(downloadQueue)) - uploadQueue = list(set(uploadQueue)) - - if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0): - print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty."))) - continue - - - totalRemovalSize = 0 - totalDownloadSize = 0 - totalUploadSize = 0 - - print_info(green(" * ")+yellow("Queue tasks:")) - detailedRemovalQueue = [] - detailedDownloadQueue = [] - detailedUploadQueue = [] - - for item in removalQueue: - fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6] - totalRemovalSize += int(fileSize) - print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) - detailedRemovalQueue.append([item,fileSize]) - import time - time.sleep(10) - - for item in downloadQueue: - # if the package is already in the upload directory, do not add the size - if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item): - fileSize = "0" - for remotePackage in remotePackagesInfo: - if remotePackage.split()[8] == item: - fileSize = remotePackage.split()[4] + print_info(green(" * ")+yellow("Remote statistics")) + remoteCounter = 0 + for tbz2 in remotePackages: + if tbz2.endswith(".tbz2"): + remoteCounter += 1 + print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored.")) + + print_info(green(" * ")+yellow("Calculating...")) + uploadQueue = [] + downloadQueue = [] + removalQueue = [] + + # Fill uploadQueue and if something weird is found, add the packages to downloadQueue + for localPackage in toBeUploaded: + pkgfound = False + for remotePackage in remotePackages: + if localPackage == remotePackage: + pkgfound = True + # it's already on the mirror, but... is its size correct?? + localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6]) + remoteSize = 0 + for file in remotePackagesInfo: + if file.split()[8] == remotePackage: + remoteSize = int(file.split()[4]) + if (localSize != remoteSize) and (localSize != 0): + # size does not match, adding to the upload queue + uploadQueue.append(localPackage) break - totalDownloadSize += int(fileSize) - print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) - detailedDownloadQueue.append([item,fileSize]) - - for item in uploadQueue: - fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] - totalUploadSize += int(fileSize) - print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) - detailedUploadQueue.append([item,fileSize]) - - print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue)))) - print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue)))) - print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue)))) - print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize)))) - print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize)))) - print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize)))) + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + uploadQueue.append(localPackage) - if (activatorRequestAsk): - rc = askquestion("\n Would you like to run the steps above ?") - if rc == "No": - print "\n" + # Fill downloadQueue and removalQueue + for remotePackage in remotePackages: + pkgfound = False + for localPackage in localPackagesRepository: + if localPackage == remotePackage: + pkgfound = True + # it's already on the mirror, but... is its size correct?? + localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6]) + remoteSize = 0 + for file in remotePackagesInfo: + if file.split()[8] == remotePackage: + remoteSize = int(file.split()[4]) + if (localSize != remoteSize) and (localSize != 0): + # size does not match, remove first + removalQueue.append(localPackage) + # then add to the download queue + downloadQueue.append(remotePackage) + break + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + downloadQueue.append(remotePackage) + + # Fill removalQueue and downloadQueue + # if the online package does not exist anymore, we have to remove it locally + for localPackage in localPackagesRepository: + pkgfound = False + for remotePackage in remotePackages: + if localPackage == remotePackage: + pkgfound = True + break + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + removalQueue.append(localPackage) + + + # filter duplicates + removalQueue = list(set(removalQueue)) + downloadQueue = list(set(downloadQueue)) + uploadQueue = list(set(uploadQueue)) + + if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0): + print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queue empty."))) continue - elif (activatorRequestPretend): - continue - # removal queue - if (detailedRemovalQueue != []): - for item in detailedRemovalQueue: - print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True) - os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]) - print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir'])) - # upload queue - if (detailedUploadQueue != []): - ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['binaryurirelativepath']) - uploadCounter = str(len(detailedUploadQueue)) - currentCounter = 0 - for item in detailedUploadQueue: - currentCounter += 1 - counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")") - print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) - ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0]) - print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) - ftp.closeFTPConnection() + totalRemovalSize = 0 + totalDownloadSize = 0 + totalUploadSize = 0 + + print_info(green(" * ")+yellow("Queue tasks:")) + detailedRemovalQueue = [] + detailedDownloadQueue = [] + detailedUploadQueue = [] + + for item in removalQueue: + fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6] + totalRemovalSize += int(fileSize) + print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedRemovalQueue.append([item,fileSize]) + import time + time.sleep(10) + + for item in downloadQueue: + # if the package is already in the upload directory, do not add the size + if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item): + fileSize = "0" + for remotePackage in remotePackagesInfo: + if remotePackage.split()[8] == item: + fileSize = remotePackage.split()[4] + break + totalDownloadSize += int(fileSize) + print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedDownloadQueue.append([item,fileSize]) + + for item in uploadQueue: + fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] + totalUploadSize += int(fileSize) + print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedUploadQueue.append([item,fileSize]) + + print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue)))) + print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue)))) + print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue)))) + print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize)))) + print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize)))) + print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize)))) + + if (activatorRequestAsk): + rc = askquestion("\n Would you like to run the steps above ?") + if rc == "No": + print "\n" + continue + elif (activatorRequestPretend): + continue + + # removal queue + if (detailedRemovalQueue != []): + for item in detailedRemovalQueue: + print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True) + os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]) + print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir'])) + + # upload queue + if (detailedUploadQueue != []): + ftp = mirrorTools.handlerFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + uploadCounter = str(len(detailedUploadQueue)) + currentCounter = 0 + for item in detailedUploadQueue: + currentCounter += 1 + counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(uploadCounter)+bold(")") + print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) + ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0]) + print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) + ftp.closeFTPConnection() + + # download queue + if (detailedDownloadQueue != []): + ftp = mirrorTools.handlerFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + downloadCounter = str(len(detailedDownloadQueue)) + currentCounter = 0 + for item in detailedDownloadQueue: + currentCounter += 1 + counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")") + if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]): + localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6]) + remoteSize = int(item[1]) + if localSize == remoteSize: + # skip that, we'll move at the end of the mirrors sync + continue + print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) + ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") + print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri))) + ftp.closeFTPConnection() + + # at least one sync ran successfully + syncSuccessful = True - # download queue - if (detailedDownloadQueue != []): - ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['binaryurirelativepath']) - downloadCounter = str(len(detailedDownloadQueue)) - currentCounter = 0 - for item in detailedDownloadQueue: - currentCounter += 1 - counterInfo = bold(" (")+blue(str(currentCounter))+"/"+red(downloadCounter)+bold(")") - if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]): - localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6]) - remoteSize = int(item[1]) - if localSize == remoteSize: - # skip that, we'll move at the end of the mirrors sync - continue - print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) - ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") - print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri))) - ftp.closeFTPConnection() # now we can store the files in upload/%ARCH% in packages/%ARCH% os.system("mv -f "+etpConst['packagessuploaddir']+"/* "+etpConst['packagesbindir']+"/ &> /dev/null") @@ -414,6 +430,15 @@ def database(options): print print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ...")) sys.exit(422) + + else: + if (dbLockFile): + print_info(green(" * ")+red("Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])) + sys.exit(423) + else: + print_info(green(" * ")+red("Mirrors are not locked. Fetching data...")) + + syncRemoteDatabases() else: print_error(red(" * ")+green("No valid tool specified.")) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 490a75b95..cbb9bbfec 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -400,6 +400,7 @@ def database(options): entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("All the selected packages have been marked as requested. Have fun.")) dbconn.closeDB() + # FIXME: not working function... remove? elif (options[0] == "orphans"): entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Collecting files in database...")) # FIXME: complete this! @@ -533,6 +534,66 @@ def database(options): entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("All the selected packages have been removed as requested. Have fun.")) dbconn.closeDB() + # used by reagent + elif (options[0] == "statistics"): + entropyTools.print_info(entropyTools.green(" [LOCAL DB STATISTIC]\t\t")+entropyTools.red("Information")) + # fetch total packages + dbconn = etpDatabase(readOnly = True) + totalpkgs = len(dbconn.listAllPackages()) + totalstablepkgs = len(dbconn.listStablePackages()) + totalunstablepkgs = len(dbconn.listUnstablePackages()) + entropyTools.print_info(entropyTools.green(" Total Installed Packages\t\t")+entropyTools.red(str(totalpkgs))) + entropyTools.print_info(entropyTools.green(" Total Stable Packages\t\t")+entropyTools.red(str(totalstablepkgs))) + entropyTools.print_info(entropyTools.green(" Total Unstable Packages\t\t")+entropyTools.red(str(totalunstablepkgs))) + entropyTools.syncRemoteDatabases(justStats = True) + dbconn.closeDB() + + # used by reagent + elif (options[0] == "md5check"): + + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Integrity verification of the selected packages:")) + + mypackages = options[1:] + dbconn = etpDatabase(readOnly = True) + + if (len(mypackages) == 0): + # check world + # create packages list + pkgs2check = dbconn.listAllPackages() + elif (mypackages[0] == "world"): + # check world + # create packages list + pkgs2check = dbconn.listAllPackages() + else: + # catch the names + pkgs2check = [] + for pkg in mypackages: + results = dbconn.searchPackages(pkg) + for i in results: + pkgs2check.append(i[0]) + + entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:")) + + toBeDownloaded = [] + availList = [] + for i in pkgs2check: + pkgfile = dbconn.retrievePackageVar(i,"download") + pkgfile = pkgfile.split("/")[len(pkgfile.split("/"))-1] + if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)): + entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i)) + availList.append(pkgfile) + elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)): + entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i)) + else: + entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i)) + toBeDownloaded.append(pkgfile) + + # FIXME add download support + # FIXME complete this + + + dbconn.closeDB() + ############ # Functions and Classes ##################################################################################### @@ -856,10 +917,10 @@ class etpDatabase: def retrievePackageVar(self,pkgkey,pkgvar): pkgkey = entropyTools.removePackageOperators(pkgkey) result = [] - self.cursor.execute('SELECT '+pkgvar+' FROM etpData WHERE atom LIKE "'+pkgkey+'"') + self.cursor.execute('SELECT "'+pkgvar+'" FROM etpData WHERE atom = "'+pkgkey+'"') for row in self.cursor: - result.append(row) - return result[0][0] + result.append(row[0]) + return result[0] # You must provide the full atom to this function def isPackageAvailable(self,pkgkey): @@ -905,19 +966,6 @@ class etpDatabase: result.append(row[0]) return result - def searchStablePackages(self,keyword): - results = [] - self.cursor.execute('SELECT * FROM etpData WHERE atom LIKE "%'+keyword+'%"') - for row in self.cursor: - results.append(row) - output = [] - for result in results: - result = result[0] - self.cursor.execute('SELECT '+result+' FROM etpData WHERE branch = "stable"') - for row in self.cursor: - output.append(row[0]) - return output - def listUnstablePackages(self): result = [] self.cursor.execute('SELECT * FROM etpData WHERE branch = "unstable"') @@ -925,6 +973,24 @@ class etpDatabase: result.append(row[0]) return result + def searchStablePackages(self,atom): + category = atom.split("/")[0] + name = atom.split("/")[1] + result = [] + self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "stable"') + for row in self.cursor: + result.append(row[0]) + return result + + def searchUnstablePackages(self,atom): + category = atom.split("/")[0] + name = atom.split("/")[1] + result = [] + self.cursor.execute('SELECT atom FROM etpData WHERE category = "'+category+'" AND name = "'+name+'" AND branch = "stable"') + for row in self.cursor: + result.append(row[0]) + return result + # useful to quickly retrieve (and trash) all the data # and look for problems. def noopCycle(self): @@ -945,9 +1011,10 @@ class etpDatabase: removelist = [] for result in results: # have a look if the slot matches - myslot = self.retrievePackageVar(result[0],"slot") + #print result + myslot = self.retrievePackageVar(result,"slot") if (myslot == slot): - removelist.append(result[0]) + removelist.append(result) for pkg in removelist: self.removePackage(pkg) diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 6135132e4..3d97ef43c 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -81,6 +81,12 @@ def md5sum(filepath): block = readfile.read(1024) return m.hexdigest() +def compareMd5(filepath,md5sum): + result = md5sum(filepath) + if md5sum == result: + return True + return False + def md5string(string): import md5 m = md5.new() @@ -265,9 +271,8 @@ def getEtpRemoteDatabaseStatus(): return uriDbInfo -def syncRemoteDatabases(noUpload = False): +def syncRemoteDatabases(noUpload = False, justStats = False): - print_info(green(" * ")+red("Checking the status of the remote Entropy Database Repository")) remoteDbsStatus = getEtpRemoteDatabaseStatus() print_info(green(" * ")+red("Remote Entropy Database Repository Status:")) for dbstat in remoteDbsStatus: @@ -286,6 +291,9 @@ def syncRemoteDatabases(noUpload = False): print_info(red("\t * Database local revision currently at: ")+blue(str(etpDbLocalRevision))) + if (justStats): + return + downloadLatest = [] uploadLatest = False uploadList = [] diff --git a/libraries/enzymeTools.py b/libraries/enzymeTools.py index 55f717a85..7aa224a1d 100644 --- a/libraries/enzymeTools.py +++ b/libraries/enzymeTools.py @@ -377,7 +377,15 @@ def build(atoms): rc = askquestion("\n Would you like to run the steps above ?") if rc == "No": sys.exit(0) - + + # sync binary packages repository + import activatorTools + if (enzymeRequestInteraction): + activatorTools.packages(["sync" , "--ask"]) + else: + activatorTools.packages(["sync"]) + + # when the compilation ends, enzyme runs reagent packagesPaths = [] diff --git a/libraries/mirrorTools.py b/libraries/mirrorTools.py index d422034fe..f1ee4d801 100644 --- a/libraries/mirrorTools.py +++ b/libraries/mirrorTools.py @@ -147,6 +147,7 @@ class handlerFTP: except: return False + # FIXME: add progress def uploadFile(self,file,ascii = False): for i in range(10): # ten tries f = open(file)