sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

more stabilization for the new branch code

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@504 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2007-09-21 22:39:49 +00:00
parent 4769fbd5ac
commit 32b71e6edc
4 changed files with 154 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
libraries/activatorTools.py
View File

@@ -624,28 +624,32 @@ def packages(options):
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/"+mybranch+"/")
# verify upload using remoteTools
print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0], mybranch)
if (ck == None):
print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported"))
ckOk = True
else:
if (ck == False):
# file does not exist???
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Reuploading..."))
else:
if len(ck) == 32:
# valid checksum, checking
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0]
ckres = compareMd5(filepath,ck)
if (ckres):
print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been uploaded correctly."))
ckOk = True
else:
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+yellow(" has NOT been uploaded correctly. Reuploading..."))
if not item[0].endswith(etpConst['packageshashfileext']):
print_info(counterInfo+red(" -> Verifying ")+green(item[0])+bold(" checksum")+red(" (if supported)"), back = True)
ck = remoteTools.getRemotePackageChecksum(extractFTPHostFromUri(uri),item[0], mybranch)
if (ck == None):
print_warning(counterInfo+red(" -> Digest verification of ")+green(item[0])+bold(" not supported"))
ckOk = True
else:
if (ck == False):
# file does not exist???
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not exist remotely. Skipping ..."))
ckOk = True
else:
# hum, what the hell is this checksum!?!?!?!
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+" Reuploading..."))
if len(ck) == 32:
# valid checksum, checking
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+item[0]
ckres = compareMd5(filepath,ck)
if (ckres):
print_info(counterInfo+red(" -> Package ")+bold(item[0])+red(" has been downloaded correctly."))
ckOk = True
else:
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+yellow(" has NOT been downloaded correctly. Redownloading..."))
else:
# hum, what the hell is this checksum!?!?!?!
print_warning(counterInfo+red(" -> Package ")+bold(item[0])+red(" does not have a proper checksum: "+str(ck)+" Redownloading..."))
else: # skip checking for .md5 files
ckOk = True
if (rc):
successfulDownloadCounter += 1
@@ -1171,7 +1175,7 @@ def getEtpRemoteDatabaseStatus():
return uriDbInfo
def downloadPackageFromMirror(uri,pkgfile):
def downloadPackageFromMirror(uri,pkgfile,branch):
entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: called for "+extractFTPHostFromUri(uri)+" and file -> "+str(pkgfile))
@@ -1181,41 +1185,41 @@ def downloadPackageFromMirror(uri,pkgfile):
entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") downloading -> "+pkgfile)
pkgfilename = pkgfile.split("/")[len(pkgfile.split("/"))-1]
print_info(red(" * Connecting to ")+bold(extractFTPHostFromUri(uri)), back = True)
# connect
ftp = mirrorTools.handlerFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+branch)
# get the files
print_info(red(" * Downloading ")+yellow(pkgfilename)+red(" from ")+bold(extractFTPHostFromUri(uri)))
rc = ftp.downloadFile(pkgfilename,etpConst['packagesbindir'])
print_info(red(" * Downloading ")+yellow(pkgfile)+red(" from ")+bold(extractFTPHostFromUri(uri)))
rc = ftp.downloadFile(pkgfile,etpConst['packagesbindir']+"/"+branch)
if (rc is None):
entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") Error. File not found. -> "+pkgfile)
# file does not exist
print_warning(red(" * File ")+yellow(pkgfilename)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri)))
print_warning(red(" * File ")+yellow(pkgfile)+red(" does not exist remotely on ")+bold(extractFTPHostFromUri(uri)))
ftp.closeConnection()
return None
entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: ("+str(tries)+"/"+str(maxtries)+") checking md5 for -> "+pkgfile)
# check md5
dbconn = databaseTools.etpDatabase(readOnly = True)
storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkgfilename,"digest")
idpackage = dbconn.getIDPackageFromFileInBranch(pkgfile,branch)
storedmd5 = dbconn.retrieveDigest(idpackage)
dbconn.closeDB()
print_info(red(" * Checking MD5 of ")+yellow(pkgfilename)+red(": should be ")+bold(storedmd5), back = True)
md5check = compareMd5(etpConst['packagesbindir']+"/"+pkgfilename,storedmd5)
print_info(red(" * Checking MD5 of ")+yellow(pkgfile)+red(": should be ")+bold(storedmd5), back = True)
md5check = compareMd5(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile,storedmd5)
if (md5check):
print_info(red(" * Package ")+yellow(pkgfilename)+red("downloaded successfully."))
print_info(red(" * Package ")+yellow(pkgfile)+red("downloaded successfully."))
return True
else:
if (tries == maxtries):
entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: Max tries limit reached. Checksum does not match. Please consider to download or repackage again. Giving up.")
print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Please consider to download or repackage again. Giving up."))
print_warning(red(" * Package ")+yellow(pkgfile)+red(" checksum does not match. Please consider to download or repackage again. Giving up."))
return False
else:
entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"downloadPackageFromMirror: Checksum does not match. Trying to download it again...")
print_warning(red(" * Package ")+yellow(pkgfilename)+red(" checksum does not match. Trying to download it again..."))
print_warning(red(" * Package ")+yellow(pkgfile)+red(" checksum does not match. Trying to download it again..."))
tries += 1
if os.path.isfile(etpConst['packagesbindir']+"/"+pkgfilename):
os.remove(etpConst['packagesbindir']+"/"+pkgfilename)
if os.path.isfile(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile):
os.remove(etpConst['packagesbindir']+"/"+branch+"/"+pkgfile)
def lockDatabases(lock = True, mirrorList = []):

53
libraries/databaseTools.py
View File

@@ -498,16 +498,6 @@ def database(options):
for i in results:
pkgs2check.append(i)
# filter idpackage only
_pkgs2check = []
for x in pkgs2check:
_pkgs2check.append(x[1])
pkgs2check = _pkgs2check
# filter dups
if (pkgs2check):
pkgs2check = entropyTools.filterDuplicatedEntries(pkgs2check)
if (not worldSelected):
print_info(red(" This is the list of the packages that would be checked:"))
else:
@@ -515,18 +505,21 @@ def database(options):
toBeDownloaded = []
availList = []
for id in pkgs2check:
pkgfile = dbconn.retrieveDownloadURL(id)
for pkginfo in pkgs2check:
pkgatom = pkginfo[0]
idpackage = pkginfo[1]
pkgbranch = pkginfo[2]
pkgfile = dbconn.retrieveDownloadURL(idpackage)
pkgfile = os.path.basename(pkgfile)
pkgatom = dbconn.retrieveAtom(id)
if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)):
if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfile)):
if (not worldSelected): print_info(green(" - [PKG AVAILABLE] ")+red(pkgatom)+" -> "+bold(pkgfile))
availList.append(id)
elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)):
availList.append(idpackage)
elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgbranch+"/"+pkgfile)):
if (not worldSelected): print_info(green(" - [RUN ACTIVATOR] ")+darkred(pkgatom)+" -> "+bold(pkgfile))
else:
if (not worldSelected): print_info(green(" - [MUST DOWNLOAD] ")+yellow(pkgatom)+" -> "+bold(pkgfile))
toBeDownloaded.append([id,pkgfile])
toBeDownloaded.append([idpackage,pkgfile,pkgbranch])
if (not databaseRequestNoAsk):
rc = entropyTools.askquestion(" Would you like to continue ?")
@@ -544,11 +537,11 @@ def database(options):
notDownloadedPackages = []
for pkg in toBeDownloaded:
rc = activatorTools.downloadPackageFromMirror(uri,pkg[1])
rc = activatorTools.downloadPackageFromMirror(uri,pkg[1],pkg[2])
if (rc is None):
notDownloadedPackages.append(pkg[1])
notDownloadedPackages.append([pkg[1],pkg[2]])
if (rc == False):
notDownloadedPackages.append(pkg[1])
notDownloadedPackages.append([pkg[1],pkg[2]])
if (rc == True):
pkgDownloadedSuccessfully += 1
availList.append(pkg[0])
@@ -561,31 +554,35 @@ def database(options):
print_warning(red(" These are the packages that cannot be found online:"))
for i in notDownloadedPackages:
pkgDownloadedError += 1
print_warning(red(" * ")+yellow(i))
print_warning(red(" * ")+yellow(i[0])+" in "+blue(i[1]))
print_warning(red(" They won't be checked."))
brokenPkgsList = []
totalcounter = str(len(availList))
currentcounter = 0
for pkg in availList:
currentcounter += 1
pkgfile = dbconn.retrieveDownloadURL(pkg)
pkgbranch = dbconn.retrieveBranch(pkg)
pkgfile = os.path.basename(pkgfile)
print_info(red(" Checking hash of ")+yellow(pkgfile)+red(" ..."), back = True)
print_info(" ("+red(str(currentcounter))+"/"+blue(totalcounter)+") "+red("Checking hash of ")+yellow(pkgfile)+red(" in branch: ")+blue(pkgbranch)+red(" ..."), back = True)
storedmd5 = dbconn.retrieveDigest(pkg)
result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkgfile,storedmd5)
result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkgbranch+"/"+pkgfile,storedmd5)
if (result):
# match !
pkgMatch += 1
#print_info(red(" Package ")+yellow(pkg)+green(" is healthy. Checksum: ")+yellow(storedmd5), back = True)
else:
pkgNotMatch += 1
print_error(red(" Package ")+yellow(pkgfile)+red(" is _NOT_ healthy !!!! Stored checksum: ")+yellow(storedmd5))
brokenPkgsList.append(pkgfile)
print_error(red(" Package ")+yellow(pkgfile)+red(" in branch: ")+blue(pkgbranch)+red(" is _NOT_ healthy !!!! Stored checksum: ")+yellow(storedmd5))
brokenPkgsList.append([pkgfile,pkgbranch])
dbconn.closeDB()
if (brokenPkgsList != []):
print_info(blue(" * This is the list of the BROKEN packages: "))
for bp in brokenPkgsList:
print_info(red(" * Package file: ")+bold(bp))
print_info(red(" * Package file: ")+bold(bp[0])+red(" in branch: ")+blue(bp[1]))
# print stats
print_info(blue(" * Statistics: "))
@@ -2611,9 +2608,9 @@ class etpDatabase:
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackages: called for "+keyword)
result = []
if (sensitive):
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE atom LIKE "%'+keyword+'%"')
self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE "%'+keyword+'%"')
else:
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"')
self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"')
for row in self.cursor:
result.append(row)
return result

165
libraries/reagentTools.py
View File

@@ -97,91 +97,102 @@ def update(options):
# differential checking
# collect differences between the packages in the database and the ones on the system
reagentRequestSeekStore = False
_options = []
for opt in options:
if opt.startswith("--seekstore"):
reagentRequestSeekStore = True
else:
_options.append(opt)
options = _options
print_info(yellow(" * ")+red("Scanning the database for differences..."))
dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True)
from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot
installedPackages = getInstalledPackagesCounters()
installedCounters = {}
databasePackages = dbconn.listAllPackages()
toBeAdded = []
toBeRemoved = []
# fill lists
# packages to be added
for x in installedPackages[0]:
installedCounters[x[1]] = 1
counter = dbconn.isCounterAvailable(x[1])
if (not counter):
toBeAdded.append(x)
# packages to be removed from the database
databaseCounters = dbconn.listAllCounters()
for x in databaseCounters:
match = installedCounters.get(x[0], None)
#print match
if (not match):
# check if the package is in toBeAdded
if (toBeAdded):
atomkey = dep_getkey(dbconn.retrieveAtom(x[1]))
atomslot = dbconn.retrieveSlot(x[1])
add = True
for pkgdata in toBeAdded:
addslot = getPackageSlot(pkgdata[0])
addkey = dep_getkey(pkgdata[0])
# workaround for ebuilds not having slot
if addslot == None:
addslot = ''
if (atomkey == addkey) and (atomslot == addslot):
# do not add to toBeRemoved
add = False
break
if add:
toBeRemoved.append(x[1])
else:
toBeRemoved.append(x[1])
if (not reagentRequestSeekStore):
print_info(yellow(" * ")+red("Scanning the database for differences..."))
dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True)
from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot
installedPackages = getInstalledPackagesCounters()
installedCounters = {}
databasePackages = dbconn.listAllPackages()
toBeAdded = []
toBeRemoved = []
if (not toBeRemoved) and (not toBeAdded):
print_info(yellow(" * ")+red("Nothing to do, check later."))
# then exit gracefully
sys.exit(0)
# fill lists
if (toBeRemoved):
print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:"))
for x in toBeRemoved:
atom = dbconn.retrieveAtom(x)
print_info(yellow(" # ")+red(atom))
rc = askquestion(">> Would you like to remove them now ?")
if rc == "Yes":
rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True)
for x in toBeRemoved:
atom = rwdbconn.retrieveAtom(x)
print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True)
rwdbconn.removePackage(x)
rwdbconn.closeDB()
print_info(yellow(" @@ ")+blue("Database removal complete."))
# packages to be added
for x in installedPackages[0]:
installedCounters[x[1]] = 1
counter = dbconn.isCounterAvailable(x[1])
if (not counter):
toBeAdded.append(x)
# packages to be removed from the database
databaseCounters = dbconn.listAllCounters()
for x in databaseCounters:
match = installedCounters.get(x[0], None)
#print match
if (not match):
# check if the package is in toBeAdded
if (toBeAdded):
atomkey = dep_getkey(dbconn.retrieveAtom(x[1]))
atomslot = dbconn.retrieveSlot(x[1])
add = True
for pkgdata in toBeAdded:
addslot = getPackageSlot(pkgdata[0])
addkey = dep_getkey(pkgdata[0])
# workaround for ebuilds not having slot
if addslot == None:
addslot = ''
if (atomkey == addkey) and (atomslot == addslot):
# do not add to toBeRemoved
add = False
break
if add:
toBeRemoved.append(x[1])
else:
toBeRemoved.append(x[1])
if (toBeAdded):
print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:"))
for x in toBeAdded:
print_info(yellow(" # ")+red(x[0]))
rc = askquestion(">> Would you like to packetize them now ?")
if rc == "No":
if (not toBeRemoved) and (not toBeAdded):
print_info(yellow(" * ")+red("Nothing to do, check later."))
# then exit gracefully
sys.exit(0)
if (toBeRemoved):
print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:"))
for x in toBeRemoved:
atom = dbconn.retrieveAtom(x)
print_info(yellow(" # ")+red(atom))
rc = askquestion(">> Would you like to remove them now ?")
if rc == "Yes":
rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True)
for x in toBeRemoved:
atom = rwdbconn.retrieveAtom(x)
print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True)
rwdbconn.removePackage(x)
rwdbconn.closeDB()
print_info(yellow(" @@ ")+blue("Database removal complete."))
if (toBeAdded):
print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:"))
for x in toBeAdded:
print_info(yellow(" # ")+red(x[0]))
rc = askquestion(">> Would you like to packetize them now ?")
if rc == "No":
sys.exit(0)
# package them
print_info(yellow(" @@ ")+blue("Compressing packages..."))
for x in toBeAdded:
print_info(yellow(" # ")+red(x[0]+"..."))
rc = quickpkg(x[0],etpConst['packagesstoredir'])
if (rc is None):
reagentLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"update: "+str(dep)+" -> quickpkg error. Cannot continue.")
print_error(red(" *")+" quickpkg error for "+red(dep))
print_error(red(" ***")+" Fatal error, cannot continue")
sys.exit(251)
# package them
print_info(yellow(" @@ ")+blue("Compressing packages..."))
for x in toBeAdded:
print_info(yellow(" # ")+red(x[0]+"..."))
rc = quickpkg(x[0],etpConst['packagesstoredir'])
if (rc is None):
reagentLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"update: "+str(dep)+" -> quickpkg error. Cannot continue.")
print_error(red(" *")+" quickpkg error for "+red(dep))
print_error(red(" ***")+" Fatal error, cannot continue")
sys.exit(251)
dbconn.closeDB()
dbconn.closeDB()
enzymeRequestBranch = etpConst['branch']
#_atoms = []

3
server/reagent
View File

@@ -41,8 +41,9 @@ def print_help():
print_info(" --version\t\tprint version")
print_info(" --nocolor\t\tdisable colorized output")
print_info(blue("Tools available: "))
print_info(" \t"+green("update")+yellow("\t\t Update Entropy Database analyzing the system for new installed packages."))
print_info(" \t"+green("update")+yellow("\t\t Update Entropy Database analyzing the system for new installed packages"))
print_info(" \t\t"+red("--branch=<branch name>")+"\t\t Choose which branch to assign to the packages")
print_info(" \t\t"+red("--seekstore")+"\t\t\t Skip differential COUNTERS scanning and analyze STORE directory")
print_info(" \t"+green("database")+yellow("\t Entropy database tool manager"))
print_info(" \t\t"+red("--initialize")+"\t\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]")
print_info(" \t\t"+red("statistics")+"\t\t\t Show Entropy database statistics.")