From 32b71e6edcaad31ffbca5be91421fec9d6447d36 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Fri, 21 Sep 2007 22:39:49 +0000 Subject: [PATCH] more stabilization for the new branch code git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@504 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- libraries/activatorTools.py | 74 ++++++++-------- libraries/databaseTools.py | 53 ++++++------ libraries/reagentTools.py | 165 +++++++++++++++++++----------------- server/reagent | 3 +- 4 files changed, 154 insertions(+), 141 deletions(-) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 0bcc11c36..9ccbbf866 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -624,28 +624,32 @@ def packages(options): rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/"+mybranch+"/") # verify upload using remoteTools - print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True) - ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0], mybranch) - if (ck == None): - print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported")) - ckOk = True - else: - if (ck == False): - # file does not exist??? - print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading...")) - else: - if len(ck) == 32: - # valid checksum, checking - filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0] - ckres = compareMd5(filepath,ck) - if (ckres): - print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been uploaded correctly.")) - ckOk = True - else: - print_warning(counterInfo+red(" -> Package ")+bold(item[0])+yellow(" has NOT been uploaded correctly. Reuploading...")) + if not item[0].endswith(etpConst['packageshashfileext']): + print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True) + ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0], mybranch) + if (ck == None): + print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported")) + ckOk = True + else: + if (ck == False): + # file does not exist??? + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Skipping ...")) + ckOk = True else: - # hum, what the hell is this checksum!?!?!?! - print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+" Reuploading...")) + if len(ck) == 32: + # valid checksum, checking + filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0] + ckres = compareMd5(filepath,ck) + if (ckres): + print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been downloaded correctly.")) + ckOk = True + else: + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+yellow(" has NOT been downloaded correctly. Redownloading...")) + else: + # hum, what the hell is this checksum!?!?!?! + print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+" Redownloading...")) + else: # skip checking for .md5 files + ckOk = True if (rc): successfulDownloadCounter += 1 @@ -1171,7 +1175,7 @@ def getEtpRemoteDatabaseStatus(): return uriDbInfo -def downloadPackageFromMirror(uri,pkgfile): +def downloadPackageFromMirror(uri,pkgfile,branch): entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+str(pkgfile)) @@ -1181,41 +1185,41 @@ def downloadPackageFromMirror(uri,pkgfile): entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") downloading -> "+pkgfile) - pkgfilename = pkgfile.split("/")[len(pkgfile.split("/"))-1] print_info(red(" * Connecting to ")+bold(extractFTPHostFromUri(uri)), back = True) # connect ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['binaryurirelativepath']) + ftp.setCWD(etpConst['binaryurirelativepath']+"/"+branch) # get the files - print_info(red(" * Downloading ")+yellow(pkgfilename)+red(" from ")+bold(extractFTPHostFromUri(uri))) - rc = ftp.downloadFile(pkgfilename,etpConst['packagesbindir']) + print_info(red(" * Downloading ")+yellow(pkgfile)+red(" from ")+bold(extractFTPHostFromUri(uri))) + rc = ftp.downloadFile(pkgfile,etpConst['packagesbindir']+"/"+branch) if (rc is None): entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") Error. File not found. -> "+pkgfile) # file does not exist - print_warning(red(" * File ")+yellow(pkgfilename)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri))) + print_warning(red(" * File ")+yellow(pkgfile)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri))) ftp.closeConnection() return None entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") checking md5 for -> "+pkgfile) # check md5 dbconn = databaseTools.etpDatabase(readOnly = True) - storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkgfilename,"digest") + idpackage = dbconn.getIDPackageFromFileInBranch(pkgfile,branch) + storedmd5 = dbconn.retrieveDigest(idpackage) dbconn.closeDB() - print_info(red(" * Checking MD5 of ")+yellow(pkgfilename)+red(": should be ")+bold(storedmd5), back = True) - md5check = compareMd5(etpConst['packagesbindir']+"/"+pkgfilename,storedmd5) + print_info(red(" * Checking MD5 of ")+yellow(pkgfile)+red(": should be ")+bold(storedmd5), back = True) + md5check = compareMd5(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile,storedmd5) if (md5check): - print_info(red(" * Package ")+yellow(pkgfilename)+red("downloaded successfully.")) + print_info(red(" * Package ")+yellow(pkgfile)+red("downloaded successfully.")) return True else: if (tries == maxtries): entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.") - print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Please consider to download or repackage again. Giving up.")) + print_warning(red(" * Package ")+yellow(pkgfile)+red(" checksum does not match. Please consider to download or repackage again. Giving up.")) return False else: entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...") - print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Trying to download it again...")) + print_warning(red(" * Package ")+yellow(pkgfile)+red(" checksum does not match. Trying to download it again...")) tries += 1 - if os.path.isfile(etpConst['packagesbindir']+"/"+pkgfilename): - os.remove(etpConst['packagesbindir']+"/"+pkgfilename) + if os.path.isfile(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile): + os.remove(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile) def lockDatabases(lock = True, mirrorList = []): diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 204deed4a..fba32bce0 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -498,16 +498,6 @@ def database(options): for i in results: pkgs2check.append(i) - # filter idpackage only - _pkgs2check = [] - for x in pkgs2check: - _pkgs2check.append(x[1]) - pkgs2check = _pkgs2check - - # filter dups - if (pkgs2check): - pkgs2check = entropyTools.filterDuplicatedEntries(pkgs2check) - if (not worldSelected): print_info(red(" This is the list of the packages that would be checked:")) else: @@ -515,18 +505,21 @@ def database(options): toBeDownloaded = [] availList = [] - for id in pkgs2check: - pkgfile = dbconn.retrieveDownloadURL(id) + for pkginfo in pkgs2check: + + pkgatom = pkginfo[0] + idpackage = pkginfo[1] + pkgbranch = pkginfo[2] + pkgfile = dbconn.retrieveDownloadURL(idpackage) pkgfile = os.path.basename(pkgfile) - pkgatom = dbconn.retrieveAtom(id) - if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)): + if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfile)): if (not worldSelected): print_info(green(" - [PKG AVAILABLE] ")+red(pkgatom)+" -> "+bold(pkgfile)) - availList.append(id) - elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)): + availList.append(idpackage) + elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgbranch+"/"+pkgfile)): if (not worldSelected): print_info(green(" - [RUN ACTIVATOR] ")+darkred(pkgatom)+" -> "+bold(pkgfile)) else: if (not worldSelected): print_info(green(" - [MUST DOWNLOAD] ")+yellow(pkgatom)+" -> "+bold(pkgfile)) - toBeDownloaded.append([id,pkgfile]) + toBeDownloaded.append([idpackage,pkgfile,pkgbranch]) if (not databaseRequestNoAsk): rc = entropyTools.askquestion(" Would you like to continue ?") @@ -544,11 +537,11 @@ def database(options): notDownloadedPackages = [] for pkg in toBeDownloaded: - rc = activatorTools.downloadPackageFromMirror(uri,pkg[1]) + rc = activatorTools.downloadPackageFromMirror(uri,pkg[1],pkg[2]) if (rc is None): - notDownloadedPackages.append(pkg[1]) + notDownloadedPackages.append([pkg[1],pkg[2]]) if (rc == False): - notDownloadedPackages.append(pkg[1]) + notDownloadedPackages.append([pkg[1],pkg[2]]) if (rc == True): pkgDownloadedSuccessfully += 1 availList.append(pkg[0]) @@ -561,31 +554,35 @@ def database(options): print_warning(red(" These are the packages that cannot be found online:")) for i in notDownloadedPackages: pkgDownloadedError += 1 - print_warning(red(" * ")+yellow(i)) + print_warning(red(" * ")+yellow(i[0])+" in "+blue(i[1])) print_warning(red(" They won't be checked.")) brokenPkgsList = [] + totalcounter = str(len(availList)) + currentcounter = 0 for pkg in availList: + currentcounter += 1 pkgfile = dbconn.retrieveDownloadURL(pkg) + pkgbranch = dbconn.retrieveBranch(pkg) pkgfile = os.path.basename(pkgfile) - print_info(red(" Checking hash of ")+yellow(pkgfile)+red(" ..."), back = True) + print_info(" ("+red(str(currentcounter))+"/"+blue(totalcounter)+") "+red("Checking hash of ")+yellow(pkgfile)+red(" in branch: ")+blue(pkgbranch)+red(" ..."), back = True) storedmd5 = dbconn.retrieveDigest(pkg) - result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkgfile,storedmd5) + result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfile,storedmd5) if (result): # match ! pkgMatch += 1 #print_info(red(" Package ")+yellow(pkg)+green(" is healthy. Checksum: ")+yellow(storedmd5), back = True) else: pkgNotMatch += 1 - print_error(red(" Package ")+yellow(pkgfile)+red(" is _NOT_ healthy !!!! Stored checksum: ")+yellow(storedmd5)) - brokenPkgsList.append(pkgfile) + print_error(red(" Package ")+yellow(pkgfile)+red(" in branch: ")+blue(pkgbranch)+red(" is _NOT_ healthy !!!! Stored checksum: ")+yellow(storedmd5)) + brokenPkgsList.append([pkgfile,pkgbranch]) dbconn.closeDB() if (brokenPkgsList != []): print_info(blue(" * This is the list of the BROKEN packages: ")) for bp in brokenPkgsList: - print_info(red(" * Package file: ")+bold(bp)) + print_info(red(" * Package file: ")+bold(bp[0])+red(" in branch: ")+blue(bp[1])) # print stats print_info(blue(" * Statistics: ")) @@ -2611,9 +2608,9 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackages: called for "+keyword) result = [] if (sensitive): - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE atom LIKE "%'+keyword+'%"') + self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE "%'+keyword+'%"') else: - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"') + self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"') for row in self.cursor: result.append(row) return result diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 569d4ebd1..8d337c024 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -97,91 +97,102 @@ def update(options): # differential checking # collect differences between the packages in the database and the ones on the system + reagentRequestSeekStore = False + _options = [] + for opt in options: + if opt.startswith("--seekstore"): + reagentRequestSeekStore = True + else: + _options.append(opt) + options = _options - print_info(yellow(" * ")+red("Scanning the database for differences...")) - dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True) - from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot - installedPackages = getInstalledPackagesCounters() - installedCounters = {} - databasePackages = dbconn.listAllPackages() - toBeAdded = [] - toBeRemoved = [] - - # fill lists - - # packages to be added - for x in installedPackages[0]: - installedCounters[x[1]] = 1 - counter = dbconn.isCounterAvailable(x[1]) - if (not counter): - toBeAdded.append(x) - # packages to be removed from the database - databaseCounters = dbconn.listAllCounters() - for x in databaseCounters: - match = installedCounters.get(x[0], None) - #print match - if (not match): - # check if the package is in toBeAdded - if (toBeAdded): - atomkey = dep_getkey(dbconn.retrieveAtom(x[1])) - atomslot = dbconn.retrieveSlot(x[1]) - add = True - for pkgdata in toBeAdded: - addslot = getPackageSlot(pkgdata[0]) - addkey = dep_getkey(pkgdata[0]) - # workaround for ebuilds not having slot - if addslot == None: - addslot = '' - if (atomkey == addkey) and (atomslot == addslot): - # do not add to toBeRemoved - add = False - break - if add: - toBeRemoved.append(x[1]) - else: - toBeRemoved.append(x[1]) + if (not reagentRequestSeekStore): + + print_info(yellow(" * ")+red("Scanning the database for differences...")) + dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True) + from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot + installedPackages = getInstalledPackagesCounters() + installedCounters = {} + databasePackages = dbconn.listAllPackages() + toBeAdded = [] + toBeRemoved = [] - if (not toBeRemoved) and (not toBeAdded): - print_info(yellow(" * ")+red("Nothing to do, check later.")) - # then exit gracefully - sys.exit(0) + # fill lists - if (toBeRemoved): - print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:")) - for x in toBeRemoved: - atom = dbconn.retrieveAtom(x) - print_info(yellow(" # ")+red(atom)) - rc = askquestion(">> Would you like to remove them now ?") - if rc == "Yes": - rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) - for x in toBeRemoved: - atom = rwdbconn.retrieveAtom(x) - print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True) - rwdbconn.removePackage(x) - rwdbconn.closeDB() - print_info(yellow(" @@ ")+blue("Database removal complete.")) + # packages to be added + for x in installedPackages[0]: + installedCounters[x[1]] = 1 + counter = dbconn.isCounterAvailable(x[1]) + if (not counter): + toBeAdded.append(x) + + # packages to be removed from the database + databaseCounters = dbconn.listAllCounters() + for x in databaseCounters: + match = installedCounters.get(x[0], None) + #print match + if (not match): + # check if the package is in toBeAdded + if (toBeAdded): + atomkey = dep_getkey(dbconn.retrieveAtom(x[1])) + atomslot = dbconn.retrieveSlot(x[1]) + add = True + for pkgdata in toBeAdded: + addslot = getPackageSlot(pkgdata[0]) + addkey = dep_getkey(pkgdata[0]) + # workaround for ebuilds not having slot + if addslot == None: + addslot = '' + if (atomkey == addkey) and (atomslot == addslot): + # do not add to toBeRemoved + add = False + break + if add: + toBeRemoved.append(x[1]) + else: + toBeRemoved.append(x[1]) - if (toBeAdded): - print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:")) - for x in toBeAdded: - print_info(yellow(" # ")+red(x[0])) - rc = askquestion(">> Would you like to packetize them now ?") - if rc == "No": + if (not toBeRemoved) and (not toBeAdded): + print_info(yellow(" * ")+red("Nothing to do, check later.")) + # then exit gracefully sys.exit(0) + + if (toBeRemoved): + print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:")) + for x in toBeRemoved: + atom = dbconn.retrieveAtom(x) + print_info(yellow(" # ")+red(atom)) + rc = askquestion(">> Would you like to remove them now ?") + if rc == "Yes": + rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) + for x in toBeRemoved: + atom = rwdbconn.retrieveAtom(x) + print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True) + rwdbconn.removePackage(x) + rwdbconn.closeDB() + print_info(yellow(" @@ ")+blue("Database removal complete.")) + + if (toBeAdded): + print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:")) + for x in toBeAdded: + print_info(yellow(" # ")+red(x[0])) + rc = askquestion(">> Would you like to packetize them now ?") + if rc == "No": + sys.exit(0) - # package them - print_info(yellow(" @@ ")+blue("Compressing packages...")) - for x in toBeAdded: - print_info(yellow(" # ")+red(x[0]+"...")) - rc = quickpkg(x[0],etpConst['packagesstoredir']) - if (rc is None): - reagentLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"update: "+str(dep)+" -> quickpkg error. Cannot continue.") - print_error(red(" *")+" quickpkg error for "+red(dep)) - print_error(red(" ***")+" Fatal error, cannot continue") - sys.exit(251) + # package them + print_info(yellow(" @@ ")+blue("Compressing packages...")) + for x in toBeAdded: + print_info(yellow(" # ")+red(x[0]+"...")) + rc = quickpkg(x[0],etpConst['packagesstoredir']) + if (rc is None): + reagentLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"update: "+str(dep)+" -> quickpkg error. Cannot continue.") + print_error(red(" *")+" quickpkg error for "+red(dep)) + print_error(red(" ***")+" Fatal error, cannot continue") + sys.exit(251) - dbconn.closeDB() + dbconn.closeDB() enzymeRequestBranch = etpConst['branch'] #_atoms = [] diff --git a/server/reagent b/server/reagent index 5d2d2e847..095b3d979 100644 --- a/server/reagent +++ b/server/reagent @@ -41,8 +41,9 @@ def print_help(): print_info(" --version\t\tprint version") print_info(" --nocolor\t\tdisable colorized output") print_info(blue("Tools available: ")) - print_info(" \t"+green("update")+yellow("\t\t Update Entropy Database analyzing the system for new installed packages.")) + print_info(" \t"+green("update")+yellow("\t\t Update Entropy Database analyzing the system for new installed packages")) print_info(" \t\t"+red("--branch=")+"\t\t Choose which branch to assign to the packages") + print_info(" \t\t"+red("--seekstore")+"\t\t\t Skip differential COUNTERS scanning and analyze STORE directory") print_info(" \t"+green("database")+yellow("\t Entropy database tool manager")) print_info(" \t\t"+red("--initialize")+"\t\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]") print_info(" \t\t"+red("statistics")+"\t\t\t Show Entropy database statistics.")