From 1c2aebb47ec59fda5d9bcd182f8988f37a858b74 Mon Sep 17 00:00:00 2001 From: lxnay Date: Thu, 12 Apr 2007 22:26:19 +0000 Subject: [PATCH] added support for .md5 files, some various bug fixes git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@282 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + handlers/activator | 1 + libraries/activatorTools.py | 76 +++++++++++++++++++++++++++-------- libraries/databaseTools.py | 27 +++++++++---- libraries/entropyConstants.py | 1 + libraries/entropyTools.py | 15 ++++++- libraries/enzymeTools.py | 5 ++- libraries/mirrorTools.py | 11 +++-- libraries/portageTools.py | 1 + libraries/reagentTools.py | 3 ++ 10 files changed, 111 insertions(+), 30 deletions(-) diff --git a/TODO b/TODO index 052f44078..0276ed33a 100644 --- a/TODO +++ b/TODO @@ -7,6 +7,7 @@ TODO list (for developers only): - mirrorTools: add PASV option (supported/not supported) - enzyme, reagent: test kernel dependent packages - enzyme, add a module that integrates revdep-rebuild +- add FXP support to activator ? - entropy, needed part, add multithreading support !! Remember that kernel dependent packages have: sys-kernel/linux-core-2.6.20-sabayon-r3 (for example) diff --git a/handlers/activator b/handlers/activator index 94c7c0473..76d5bf10d 100644 --- a/handlers/activator +++ b/handlers/activator @@ -47,6 +47,7 @@ def print_help(): entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("packages"))+entropyTools.yellow("\t to manage binary packages")) entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t ask before making any changes") entropyTools.print_info(" \t\t"+entropyTools.red("--pretend")+"\t\t just show what would be done") + entropyTools.print_info(" \t\t"+entropyTools.red("--do-packages-check")+"\t\t after the syncronization, also check packages checksum") entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the binary packages across primary mirrors")) entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t to manage database status and settings [")+entropyTools.bold("database content won't be touched")+entropyTools.yellow("]")) entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the database across primary mirrors")) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 23a89e808..4d9f919a1 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -30,6 +30,7 @@ import sys import os import commands import string +import time def sync(options, justTidy = False): @@ -37,11 +38,21 @@ def sync(options, justTidy = False): if (not justTidy): # firstly sync the packages - rc = packages(["sync" , "--ask"]) + rc = packages([ "sync" , "--ask" ]) # then sync the database, if the packages sync completed successfully if (rc == False): sys.exit(401) - database(["sync"]) + else: + # if packages are ok, we can sync the database + database(["sync"]) + # now check packages checksum + import databaseTools + databaseTools.database(['md5check']) + time.sleep(2) + # ask question + rc = askquestion(" Should I continue with the tidy procedure ?") + if rc == "No": + sys.exit(0) print_info(green(" * ")+red("Starting to collect packages that would be removed from the repository ..."), back = True) @@ -70,8 +81,9 @@ def sync(options, justTidy = False): found = True break if (not found): - # then remove - removeList.append(repoBin) + if (not repoBin.endswith(etpConst['packageshashfileext'])): # filter hash files + # then remove + removeList.append(repoBin) if (removeList == []): print_info(green(" * ")+red("No packages to remove from the mirrors.")) @@ -114,11 +126,14 @@ def packages(options): myopts = options[1:] activatorRequestAsk = False activatorRequestPretend = False + activatorRequestPackagesCheck = False for opt in myopts: if (opt == "--ask"): activatorRequestAsk = True elif (opt == "--pretend"): activatorRequestPretend = True + elif (opt == "--do-packages-check"): + activatorRequestPackagesCheck = True if (options[0] == "sync"): print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ...")) @@ -132,6 +147,7 @@ def packages(options): currentUri += 1 + # readd try try: print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror."))) @@ -140,17 +156,19 @@ def packages(options): uploadCounter = 0 toBeUploaded = [] # parse etpConst['packagessuploaddir'] for tbz2 in os.listdir(etpConst['packagessuploaddir']): - if tbz2.endswith(".tbz2"): + if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']): toBeUploaded.append(tbz2) - uploadCounter += 1 + if tbz2.endswith(".tbz2"): + uploadCounter += 1 print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) localPackagesRepository = [] # parse etpConst['packagesbindir'] print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True) packageCounter = 0 for tbz2 in os.listdir(etpConst['packagesbindir']): - if tbz2.endswith(".tbz2"): + if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']): localPackagesRepository.append(tbz2) - packageCounter += 1 + if tbz2.endswith(".tbz2"): + packageCounter += 1 print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True) @@ -306,8 +324,9 @@ def packages(options): print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) detailedDownloadQueue.append([item,fileSize]) else: - fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] - print_info(bold("\t[") + green("LOCAL COPY") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + if (not item.endswith(etpConst['packageshashfileext'])): + fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] + print_info(bold("\t[") + green("LOCAL COPY") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) # file exists locally and remotely (where is fine == fully uploaded) simpleCopyQueue.append(etpConst['packagessuploaddir']+"/"+item) @@ -347,6 +366,7 @@ def packages(options): for item in detailedRemovalQueue: print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir'])+red(" ...")) os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]) + os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]+etpConst['packageshashfileext']) print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir'])) # simple copy queue @@ -354,6 +374,7 @@ def packages(options): for item in simpleCopyQueue: print_info(red(" * Copying file from ") + bold(item) + red(" to ")+bold(etpConst['packagesbindir'])) os.system("cp -p "+item+" "+etpConst['packagesbindir']+"/ > /dev/null") + # md5 copy not needed, already in simpleCopyQueue # upload queue if (detailedUploadQueue != []): @@ -367,10 +388,18 @@ def packages(options): print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ...")) # is the package in the upload queue? if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]): - rc = ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0]) - else: # or in the packages queue ! - rc = ftp.uploadFile(etpConst['packagesbindir']+"/"+item[0]) - if (rc): + uploadItem = etpConst['packagessuploaddir']+"/"+item[0] + else: + uploadItem = etpConst['packagesbindir']+"/"+item[0] + rc = ftp.uploadFile(uploadItem) + if not os.path.isfile(uploadItem+etpConst['packageshashfileext']): + hashfile = createHashFile(uploadItem) + else: + hashfile = uploadItem+etpConst['packageshashfileext'] + # upload md5 hash + rcmd5 = ftp.uploadFile(hashfile,ascii = True) + + if (rc) and (rcmd5): successfulUploadCounter += 1 print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) ftp.closeFTPConnection() @@ -391,9 +420,16 @@ def packages(options): # skip that, we'll move at the end of the mirrors sync continue print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ...")) - rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") - # FIXME: add if condition --> if (rc): - successfulDownloadCounter += 1 + + # FIXME: test if the .md5 got downloaded + if item[0].endswith(etpConst['packageshashfileext']): + rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True) + else: + rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") + + if (rc): + successfulDownloadCounter += 1 + print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri))) ftp.closeFTPConnection() @@ -432,6 +468,12 @@ def packages(options): else: sys.exit(470) + # Now we should start to check all the packages in the packages directory + if (activatorRequestPackagesCheck): + import databaseTools + databaseTools.database(['md5check']) + + def database(options): # lock tool diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index d27e4eaa2..333425530 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -562,14 +562,17 @@ def database(options): pkgNotMatch = 0 pkgDownloadedSuccessfully = 0 pkgDownloadedError = 0 + worldSelected = False if (len(mypackages) == 0): # check world # create packages list + worldSelected = True pkgs2check = dbconn.listAllPackages() elif (mypackages[0] == "world"): # check world # create packages list + worldSelected = True pkgs2check = dbconn.listAllPackages() else: # catch the names @@ -579,7 +582,14 @@ def database(options): for i in results: pkgs2check.append(i[0]) - entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:")) + # order alphabetically + if (pkgs2check != []): + pkgs2check = entropyTools.alphaSorter(pkgs2check) + + if (not worldSelected): + entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:")) + else: + entropyTools.print_info(entropyTools.red(" All the packages in the Entropy Packages repository will be checked.")) toBeDownloaded = [] availList = [] @@ -587,12 +597,12 @@ def database(options): pkgfile = dbconn.retrievePackageVar(i,"download") pkgfile = pkgfile.split("/")[len(pkgfile.split("/"))-1] if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)): - entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i)+" -> "+entropyTools.bold(pkgfile)) + if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i)+" -> "+entropyTools.bold(pkgfile)) availList.append(pkgfile) elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)): - entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i)+" -> "+entropyTools.bold(pkgfile)) + if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i)+" -> "+entropyTools.bold(pkgfile)) else: - entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i)+" -> "+entropyTools.bold(pkgfile)) + if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i)+" -> "+entropyTools.bold(pkgfile)) toBeDownloaded.append(pkgfile) rc = entropyTools.askquestion(" Would you like to continue ?") @@ -632,13 +642,13 @@ def database(options): brokenPkgsList = [] for pkg in availList: - entropyTools.print_info(entropyTools.red(" Checking MD5 of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True) + entropyTools.print_info(entropyTools.red(" Checking Checksum of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True) storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkg,"digest") result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkg,storedmd5) if (result): # match ! pkgMatch += 1 - entropyTools.print_info(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.green(" is healthy. Checksum: ")+entropyTools.yellow(storedmd5)) + #entropyTools.print_info(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.green(" is healthy. Checksum: ")+entropyTools.yellow(storedmd5), back = True) else: pkgNotMatch += 1 entropyTools.print_error(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.red(" is _NOT_ healthy !!!! Stored checksum: ")+entropyTools.yellow(storedmd5)) @@ -730,7 +740,7 @@ class etpDatabase: entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Entropy database is already locked by you :-)")) else: # check if the database is locked REMOTELY - entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Locking and Sync Entropy database ..."), back = True) + entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Locking and Syncing Entropy database ..."), back = True) for uri in etpConst['activatoruploaduris']: ftp = mirrorTools.handlerFTP(uri) ftp.setCWD(etpConst['etpurirelativepath']) @@ -781,6 +791,7 @@ class etpDatabase: # if the class is opened readOnly, close and forget if (self.readOnly): + #self.connection.rollback() self.cursor.close() self.connection.close() return @@ -805,7 +816,7 @@ class etpDatabase: self.connection.commit() self.taintDatabase() else: - self.connection.rollback() # is it ok? + self.discardChanges() # is it ok? def taintDatabase(self): # taint the database status diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index f7ecaa777..63d1afb25 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -158,6 +158,7 @@ etpConst = { 'etpdatabasetaintfile': ETP_DBFILE+".tainted", # when this file exists, the database is not synced anymore with the online one 'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db" 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) + 'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit 'databaselogfile': ETP_LOG_DIR+"/database.log", # database operations log file 'distcc-status': False, # used by Enzyme, if True distcc is enabled diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 4bb7b0ef5..f19667888 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -81,6 +81,19 @@ def md5sum(filepath): block = readfile.read(1024) return m.hexdigest() +# This function creates the .hash file related to the given package file +# @returns the complete hash file path +# FIXME: add more hashes, SHA1 for example +def createHashFile(tbz2filepath): + md5hash = md5sum(tbz2filepath) + hashfile = tbz2filepath+etpConst['packageshashfileext'] + f = open(hashfile,"w") + tbz2name = os.path.basename(tbz2filepath) + f.write(md5hash+" "+tbz2name+"\n") + f.flush() + f.close() + return hashfile + def compareMd5(filepath,checksum): checksum = str(checksum) result = md5sum(filepath) @@ -1005,7 +1018,7 @@ def print_error(msg, back = False): print green(">>")+" "+msg def print_info(msg, back = False): - writechar("\r \r") + writechar("\r \r") if (back): writechar("\r"+green(">>")+" "+msg) return diff --git a/libraries/enzymeTools.py b/libraries/enzymeTools.py index e01641617..90a51bdc3 100644 --- a/libraries/enzymeTools.py +++ b/libraries/enzymeTools.py @@ -418,13 +418,16 @@ def build(atoms): #PackagesDependencies = list(set(PackagesDependencies)) compileError = False + PackagesDependenciesCounter = 0 + PackagesDependenciesLength = len(PackagesDependencies) if PackagesDependencies != []: #print print_info(yellow(" *")+" Building packages...") for dep in PackagesDependencies: outfile = etpConst['packagestmpdir']+"/.emerge-"+dep.split("/")[len(dep.split("/"))-1]+"-"+str(getRandomNumber()) - print_info(green(" *")+" Compiling: "+red(dep)+" ... ") + PackagesDependenciesCounter += 1 + print_info(green(" * ")+bold("(")+blue(str(PackagesDependenciesCounter))+green("/")+red(str(PackagesDependenciesLength))+bold(")")+" Compiling: "+red(dep)+" ... ") mountProc() if (not enzymeRequestVerbose): diff --git a/libraries/mirrorTools.py b/libraries/mirrorTools.py index df31a5ad1..6de103fb6 100644 --- a/libraries/mirrorTools.py +++ b/libraries/mirrorTools.py @@ -185,6 +185,9 @@ class handlerFTP: self.myFileSize = round(float(os.stat(file)[6])/1024,1) self.mykByteCount = 0 + if self.isFileAvailable(filename+".tmp"): + self.deleteFile(filename+".tmp") + if (ascii): rc = self.ftpconn.storlines("STOR "+filename+".tmp",f) else: @@ -196,11 +199,13 @@ class handlerFTP: return True else: return False - except socket.error: # connection reset by peer + except: # connection reset by peer entropyTools.print_info(entropyTools.red("Upload issue, retrying...")) self.reconnectHost() # reconnect - self.deleteFile(filename) - self.deleteFile(filename+".tmp") + if self.isFileAvailable(filename): + self.deleteFile(filename) + if self.isFileAvailable(filename+".tmp"): + self.deleteFile(filename+".tmp") f.close() continue diff --git a/libraries/portageTools.py b/libraries/portageTools.py index baf978360..4b4a4c76f 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -425,6 +425,7 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles): print "DEBUG:" print pkgBinaryFiles + print newPkgBinaryFiles # check for version bumps for pkg in pkgBinaryFiles: diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 55a48e257..4889d591e 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -107,7 +107,10 @@ def enzyme(options): rc, newFileName = generator(tbz2path, enzymeRequestBump, dbconn) if (rc): etpCreated += 1 + # create .hash file + hashFilePath = createHashFile(tbz2path) os.system("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f") + os.system("mv "+hashFilePath+" "+etpConst['packagessuploaddir']+"/ -f") else: etpNotCreated += 1 os.system("rm -rf "+tbz2path)