diff --git a/TODO b/TODO index 4b5f59ec7..e8c6057ce 100644 --- a/TODO +++ b/TODO @@ -1,9 +1,13 @@ TODO list (for developers only): - build() and world(), on enzyme, add the support for whitelist+cron - build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form) +- system-wide: add free space check +- activator: for the tidy module I need that reagent includes the SLOT variable. - activator tasks: - add/remove packages from the database - - cleanup policy to prune old binaries + - tidy tool: cleanup policy to prune old binaries + - sync tool removal + - database update and management between me and reagent - Sabayon Linux USE flags: remove all server related use flags @@ -11,11 +15,11 @@ Project Status: - entropy: not yet started, nothing to say then - enzyme: first release up and working (more or less) - reagent: first release up and working. -- activator: 50% +- activator: 60% ============ - client part: not yet defined apart from equilibrium name. Features plan: -- enzyme: distcc support +- enzyme: distcc support on cross platforms and on automake - activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one. \ No newline at end of file diff --git a/handlers/activator b/handlers/activator index 1f4386e5f..f2d41f7ab 100644 --- a/handlers/activator +++ b/handlers/activator @@ -47,6 +47,8 @@ def print_help(): entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)") entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("packages"))+entropyTools.yellow("\t to manage binary packages")) + entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t ask before making any changes") + entropyTools.print_info(" \t\t"+entropyTools.red("--pretend")+"\t\t just show what would be done") entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the binary packages across primary mirrors")) entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t to manage database status and settings")) entropyTools.print_info(" \t\t"+entropyTools.green("lock")+entropyTools.red("\t\t\t to lock the database(s) status ["+entropyTools.yellow("DANGEROUS")+entropyTools.red("]"))) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index d2f6c4db2..f05d4cfa5 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -41,10 +41,6 @@ def sync(options): #print ftp.getFileSize("index.htm") list = ftp.getRoughList() - for item in list: - if item.find("index.htm") != -1: - # extact the size - print item #print ftp.spawnFTPCommand("mdtm index.htm") ftp.closeFTPConnection() @@ -79,13 +75,21 @@ def sync(options): def packages(options): - # FIXME: add support for --ask - # FIXME: complete this part + # Options available for all the packages submodules + myopts = options[1:] + activatorRequestAsk = False + activatorRequestPretend = False + for opt in myopts: + if (opt == "--ask"): + activatorRequestAsk = True + elif (opt == "--pretend"): + activatorRequestPretend = True + if (options[0] == "sync"): print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ...")) for uri in etpConst['activatoruploaduris']: - print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+" mirror.")) + print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror."))) print_info(green(" * ")+yellow("Local Statistics")) print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True) uploadCounter = 0 @@ -104,13 +108,14 @@ def packages(options): packageCounter += 1 print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) - print_info(green(" * ")+yellow("Remote statistics")) + print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) ftp = activatorFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) remotePackages = ftp.listFTPdir() remotePackagesInfo = ftp.getRoughList() ftp.closeFTPConnection() + print_info(green(" * ")+yellow("Remote statistics")) remoteCounter = 0 for tbz2 in remotePackages: if tbz2.endswith(".tbz2"): @@ -118,16 +123,128 @@ def packages(options): print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored.")) print_info(green(" * ")+yellow("Calculating...")) - # now it's time to compare what I have to upload - for remotePackage in remotePackages: - pkgfound = True - for localPackage in toBeUploaded: + uploadQueue = [] + downloadQueue = [] + + # Fill uploadQueue and if something weird is found, add the packages to downloadQueue + # --> UPLOAD + for localPackage in toBeUploaded: + pkgfound = False + for remotePackage in remotePackages: if localPackage == remotePackage: + pkgfound = True # it's already on the mirror, but... is its size correct?? - remoteSize = ftp.getFileSize(remotePackage) + localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6]) + remoteSize = 0 + for file in remotePackagesInfo: + if file.split()[8] == remotePackage: + remoteSize = int(file.split()[4]) + if (localSize != remoteSize) and (localSize != 0): + # size does not match, adding to the upload queue + uploadQueue.append(localPackage) + break + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + uploadQueue.append(localPackage) + + # Fill downloadQueue and if something weird is found, add the packages to uploadQueue + for remotePackage in remotePackages: + pkgfound = False + for localPackage in toBeDownloaded: + if localPackage == remotePackage: + pkgfound = True + # it's already on the mirror, but... is its size correct?? + localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6]) + remoteSize = 0 + for file in remotePackagesInfo: + if file.split()[8] == remotePackage: + remoteSize = int(file.split()[4]) + if (localSize != remoteSize) and (localSize != 0): + # size does not match, adding to the download queue + downloadQueue.append(remotePackage) + break + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + downloadQueue.append(remotePackage) + + # filter duplicates + uploadQueue = list(set(uploadQueue)) + downloadQueue = list(set(downloadQueue)) + moveQueue = [] + + if (len(uploadQueue) == 0) and (len(downloadQueue) == 0): + print_info(green(" * ")+red("Nothing to syncronize. Queues empty.")) + sys.exit(0) + totalUploadSize = 0 + totalDownloadSize = 0 + print_info(green(" * ")+yellow("Queue tasks:")) + detailedUploadQueue = [] + detailedDownloadQueue = [] + for item in uploadQueue: + fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] + totalUploadSize += int(fileSize) + print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedUploadQueue.append([item,fileSize]) + for item in downloadQueue: + fileSize = "0" + for remotePackage in remotePackagesInfo: + if remotePackage.split()[8] == item: + fileSize = remotePackage.split()[4] + break + totalDownloadSize += int(fileSize) + print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedDownloadQueue.append([item,fileSize]) + print_info(red(" * ")+blue("Packages that would be ")+red("uploaded:\t\t")+bold(str(len(uploadQueue)))) + print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue)))) + print_info(red(" * ")+blue("Total upload ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize)))) + print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize)))) + if (activatorRequestAsk): + rc = askquestion("\n Would you like to run the steps above ?") + if rc == "No": + print "\n" + continue + elif (activatorRequestPretend): + continue + + # upload queue + if (detailedUploadQueue != []): + ftp = activatorFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + for item in detailedUploadQueue: + print_info(red(" * Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) + ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0]) + # now move the file into etpConst['packagesbindir'] + os.system("mv "+etpConst['packagessuploaddir']+"/"+item[0]+" "+etpConst['packagesbindir']+"/") + print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) + ftp.closeFTPConnection() + + # for the download queue, also check in the upload directory + if (detailedDownloadQueue != []): + ftp = activatorFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + for item in detailedDownloadQueue: + if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]): + localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6]) + remoteSize = int(item[1]) + if localSize == remoteSize: + print_info(red(" * Moving file ")+bold(item[0])+red(" to ")+bold(etpConst['packagesbindir'])+red(" ..."),back = True) + os.system("mv "+etpConst['packagessuploaddir']+"/"+item[0]+" "+etpConst['packagesbindir']+"/") + continue + + print_info(red(" * Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) + ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") + print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) + ftp.closeFTPConnection() + + # Now I should do some tidy + print "Now it should be time for some tidy...?" def database(options): diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index b1c57f75d..d424371fe 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -880,6 +880,14 @@ class activatorFTP: # not supported by dreamhost.com def getFileSize(self,file): return self.ftpconn.size(file) + + def getFileSizeCompat(self,file): + list = getRoughList() + for item in list: + if item.find(file) != -1: + # extact the size + return item.split()[4] + return "" def bufferizer(self,buf): self.FTPbuffer.append(buf) @@ -1202,6 +1210,16 @@ def digestFile(filepath): digest.update(line) return digest.hexdigest() +def bytesIntoHuman(bytes): + bytes = str(bytes) + kbytes = str(int(bytes)/1024) + if len(kbytes) > 3: + kbytes = str(int(kbytes)/1024) + kbytes += "MB" + else: + kbytes += "kB" + return kbytes + # hide password from full ftp URI def hideFTPpassword(uri): ftppassword = uri.split("@")[:len(uri.split("@"))-1]