sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added support for .md5 files, some various bug fixes

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@282 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-04-12 22:26:19 +00:00
parent aef097c0bd
commit 1c2aebb47e
10 changed files with 111 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO
+1
View File
@@ -7,6 +7,7 @@ TODO list (for developers only):
- mirrorTools: add PASV option (supported/not supported)
- enzyme, reagent: test kernel dependent packages
- enzyme, add a module that integrates revdep-rebuild
- add FXP support to activator ?
- entropy, needed part, add multithreading support
!! Remember that kernel dependent packages have: sys-kernel/linux-core-2.6.20-sabayon-r3 (for example)
handlers/activator
Vendored
+1
View File
@@ -47,6 +47,7 @@ def print_help():
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("packages"))+entropyTools.yellow("\t to manage binary packages"))
entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t ask before making any changes")
entropyTools.print_info(" \t\t"+entropyTools.red("--pretend")+"\t\t just show what would be done")
entropyTools.print_info(" \t\t"+entropyTools.red("--do-packages-check")+"\t\t after the syncronization, also check packages checksum")
entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the binary packages across primary mirrors"))
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t to manage database status and settings [")+entropyTools.bold("database content won't be touched")+entropyTools.yellow("]"))
entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the database across primary mirrors"))
libraries/activatorTools.py
+59 -17
View File
@@ -30,6 +30,7 @@ import sys
import os
import commands
import string
import time
def sync(options, justTidy = False):
@@ -37,11 +38,21 @@ def sync(options, justTidy = False):
if (not justTidy):
# firstly sync the packages
rc = packages(["sync" , "--ask"])
rc = packages([ "sync" , "--ask" ])
# then sync the database, if the packages sync completed successfully
if (rc == False):
sys.exit(401)
database(["sync"])
else:
# if packages are ok, we can sync the database
database(["sync"])
# now check packages checksum
import databaseTools
databaseTools.database(['md5check'])
time.sleep(2)
# ask question
rc = askquestion(" Should I continue with the tidy procedure ?")
if rc == "No":
sys.exit(0)
print_info(green(" * ")+red("Starting to collect packages that would be removed from the repository ..."), back = True)
@@ -70,8 +81,9 @@ def sync(options, justTidy = False):
found = True
break
if (not found):
# then remove
removeList.append(repoBin)
if (not repoBin.endswith(etpConst['packageshashfileext'])): # filter hash files
# then remove
removeList.append(repoBin)
if (removeList == []):
print_info(green(" * ")+red("No packages to remove from the mirrors."))
@@ -114,11 +126,14 @@ def packages(options):
myopts = options[1:]
activatorRequestAsk = False
activatorRequestPretend = False
activatorRequestPackagesCheck = False
for opt in myopts:
if (opt == "--ask"):
activatorRequestAsk = True
elif (opt == "--pretend"):
activatorRequestPretend = True
elif (opt == "--do-packages-check"):
activatorRequestPackagesCheck = True
if (options[0] == "sync"):
print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ..."))
@@ -132,6 +147,7 @@ def packages(options):
currentUri += 1
# readd try
try:
print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror.")))
@@ -140,17 +156,19 @@ def packages(options):
uploadCounter = 0
toBeUploaded = [] # parse etpConst['packagessuploaddir']
for tbz2 in os.listdir(etpConst['packagessuploaddir']):
if tbz2.endswith(".tbz2"):
if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']):
toBeUploaded.append(tbz2)
uploadCounter += 1
if tbz2.endswith(".tbz2"):
uploadCounter += 1
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
localPackagesRepository = [] # parse etpConst['packagesbindir']
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
packageCounter = 0
for tbz2 in os.listdir(etpConst['packagesbindir']):
if tbz2.endswith(".tbz2"):
if tbz2.endswith(".tbz2") or tbz2.endswith(etpConst['packageshashfileext']):
localPackagesRepository.append(tbz2)
packageCounter += 1
if tbz2.endswith(".tbz2"):
packageCounter += 1
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
@@ -306,8 +324,9 @@ def packages(options):
print_info(bold("\t[") + yellow("REMOTE DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedDownloadQueue.append([item,fileSize])
else:
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
print_info(bold("\t[") + green("LOCAL COPY") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
if (not item.endswith(etpConst['packageshashfileext'])):
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
print_info(bold("\t[") + green("LOCAL COPY") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
# file exists locally and remotely (where is fine == fully uploaded)
simpleCopyQueue.append(etpConst['packagessuploaddir']+"/"+item)
@@ -347,6 +366,7 @@ def packages(options):
for item in detailedRemovalQueue:
print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir'])+red(" ..."))
os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0])
os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]+etpConst['packageshashfileext'])
print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']))
# simple copy queue
@@ -354,6 +374,7 @@ def packages(options):
for item in simpleCopyQueue:
print_info(red(" * Copying file from ") + bold(item) + red(" to ")+bold(etpConst['packagesbindir']))
os.system("cp -p "+item+" "+etpConst['packagesbindir']+"/ > /dev/null")
# md5 copy not needed, already in simpleCopyQueue
# upload queue
if (detailedUploadQueue != []):
@@ -367,10 +388,18 @@ def packages(options):
print_info(counterInfo+red(" Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."))
# is the package in the upload queue?
if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]):
rc = ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
else: # or in the packages queue !
rc = ftp.uploadFile(etpConst['packagesbindir']+"/"+item[0])
if (rc):
uploadItem = etpConst['packagessuploaddir']+"/"+item[0]
else:
uploadItem = etpConst['packagesbindir']+"/"+item[0]
rc = ftp.uploadFile(uploadItem)
if not os.path.isfile(uploadItem+etpConst['packageshashfileext']):
hashfile = createHashFile(uploadItem)
else:
hashfile = uploadItem+etpConst['packageshashfileext']
# upload md5 hash
rcmd5 = ftp.uploadFile(hashfile,ascii = True)
if (rc) and (rcmd5):
successfulUploadCounter += 1
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
@@ -391,9 +420,16 @@ def packages(options):
# skip that, we'll move at the end of the mirrors sync
continue
print_info(counterInfo+red(" Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."))
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
# FIXME: add if condition --> if (rc):
successfulDownloadCounter += 1
# FIXME: test if the .md5 got downloaded
if item[0].endswith(etpConst['packageshashfileext']):
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/", ascii = True)
else:
rc = ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
if (rc):
successfulDownloadCounter += 1
print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
@@ -432,6 +468,12 @@ def packages(options):
else:
sys.exit(470)
# Now we should start to check all the packages in the packages directory
if (activatorRequestPackagesCheck):
import databaseTools
databaseTools.database(['md5check'])
def database(options):
# lock tool
libraries/databaseTools.py
+19 -8
View File
@@ -562,14 +562,17 @@ def database(options):
pkgNotMatch = 0
pkgDownloadedSuccessfully = 0
pkgDownloadedError = 0
worldSelected = False
if (len(mypackages) == 0):
# check world
# create packages list
worldSelected = True
pkgs2check = dbconn.listAllPackages()
elif (mypackages[0] == "world"):
# check world
# create packages list
worldSelected = True
pkgs2check = dbconn.listAllPackages()
else:
# catch the names
@@ -579,7 +582,14 @@ def database(options):
for i in results:
pkgs2check.append(i[0])
entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:"))
# order alphabetically
if (pkgs2check != []):
pkgs2check = entropyTools.alphaSorter(pkgs2check)
if (not worldSelected):
entropyTools.print_info(entropyTools.red(" This is the list of the packages that would be checked:"))
else:
entropyTools.print_info(entropyTools.red(" All the packages in the Entropy Packages repository will be checked."))
toBeDownloaded = []
availList = []
@@ -587,12 +597,12 @@ def database(options):
pkgfile = dbconn.retrievePackageVar(i,"download")
pkgfile = pkgfile.split("/")[len(pkgfile.split("/"))-1]
if (os.path.isfile(etpConst['packagesbindir']+"/"+pkgfile)):
entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i)+" -> "+entropyTools.bold(pkgfile))
if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [PKG AVAILABLE] ")+entropyTools.red(i)+" -> "+entropyTools.bold(pkgfile))
availList.append(pkgfile)
elif (os.path.isfile(etpConst['packagessuploaddir']+"/"+pkgfile)):
entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i)+" -> "+entropyTools.bold(pkgfile))
if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [RUN ACTIVATOR] ")+entropyTools.darkred(i)+" -> "+entropyTools.bold(pkgfile))
else:
entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i)+" -> "+entropyTools.bold(pkgfile))
if (not worldSelected): entropyTools.print_info(entropyTools.green(" - [MUST DOWNLOAD] ")+entropyTools.yellow(i)+" -> "+entropyTools.bold(pkgfile))
toBeDownloaded.append(pkgfile)
rc = entropyTools.askquestion(" Would you like to continue ?")
@@ -632,13 +642,13 @@ def database(options):
brokenPkgsList = []
for pkg in availList:
entropyTools.print_info(entropyTools.red(" Checking MD5 of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True)
entropyTools.print_info(entropyTools.red(" Checking Checksum of ")+entropyTools.yellow(pkg)+entropyTools.red(" ..."), back = True)
storedmd5 = dbconn.retrievePackageVarFromBinaryPackage(pkg,"digest")
result = entropyTools.compareMd5(etpConst['packagesbindir']+"/"+pkg,storedmd5)
if (result):
# match !
pkgMatch += 1
entropyTools.print_info(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.green(" is healthy. Checksum: ")+entropyTools.yellow(storedmd5))
#entropyTools.print_info(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.green(" is healthy. Checksum: ")+entropyTools.yellow(storedmd5), back = True)
else:
pkgNotMatch += 1
entropyTools.print_error(entropyTools.red(" Package ")+entropyTools.yellow(pkg)+entropyTools.red(" is _NOT_ healthy !!!! Stored checksum: ")+entropyTools.yellow(storedmd5))
@@ -730,7 +740,7 @@ class etpDatabase:
entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Entropy database is already locked by you :-)"))
else:
# check if the database is locked REMOTELY
entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Locking and Sync Entropy database ..."), back = True)
entropyTools.print_info(entropyTools.red(" * ")+entropyTools.red(" Locking and Syncing Entropy database ..."), back = True)
for uri in etpConst['activatoruploaduris']:
ftp = mirrorTools.handlerFTP(uri)
ftp.setCWD(etpConst['etpurirelativepath'])
@@ -781,6 +791,7 @@ class etpDatabase:
# if the class is opened readOnly, close and forget
if (self.readOnly):
#self.connection.rollback()
self.cursor.close()
self.connection.close()
return
@@ -805,7 +816,7 @@ class etpDatabase:
self.connection.commit()
self.taintDatabase()
else:
self.connection.rollback() # is it ok?
self.discardChanges() # is it ok?
def taintDatabase(self):
# taint the database status
libraries/entropyConstants.py
+1
View File
@@ -158,6 +158,7 @@ etpConst = {
'etpdatabasetaintfile': ETP_DBFILE+".tainted", # when this file exists, the database is not synced anymore with the online one
'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db"
'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped)
'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file
'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit
'databaselogfile': ETP_LOG_DIR+"/database.log", # database operations log file
'distcc-status': False, # used by Enzyme, if True distcc is enabled
libraries/entropyTools.py
+14 -1
View File
@@ -81,6 +81,19 @@ def md5sum(filepath):
block = readfile.read(1024)
return m.hexdigest()
# This function creates the .hash file related to the given package file
# @returns the complete hash file path
# FIXME: add more hashes, SHA1 for example
def createHashFile(tbz2filepath):
md5hash = md5sum(tbz2filepath)
hashfile = tbz2filepath+etpConst['packageshashfileext']
f = open(hashfile,"w")
tbz2name = os.path.basename(tbz2filepath)
f.write(md5hash+" "+tbz2name+"\n")
f.flush()
f.close()
return hashfile
def compareMd5(filepath,checksum):
checksum = str(checksum)
result = md5sum(filepath)
@@ -1005,7 +1018,7 @@ def print_error(msg, back = False):
print green(">>")+" "+msg
def print_info(msg, back = False):
writechar("\r \r")
writechar("\r \r")
if (back):
writechar("\r"+green(">>")+" "+msg)
return
libraries/enzymeTools.py
+4 -1
View File
@@ -418,13 +418,16 @@ def build(atoms):
#PackagesDependencies = list(set(PackagesDependencies))
compileError = False
PackagesDependenciesCounter = 0
PackagesDependenciesLength = len(PackagesDependencies)
if PackagesDependencies != []:
#print
print_info(yellow(" *")+" Building packages...")
for dep in PackagesDependencies:
outfile = etpConst['packagestmpdir']+"/.emerge-"+dep.split("/")[len(dep.split("/"))-1]+"-"+str(getRandomNumber())
print_info(green(" *")+" Compiling: "+red(dep)+" ... ")
PackagesDependenciesCounter += 1
print_info(green(" * ")+bold("(")+blue(str(PackagesDependenciesCounter))+green("/")+red(str(PackagesDependenciesLength))+bold(")")+" Compiling: "+red(dep)+" ... ")
mountProc()
if (not enzymeRequestVerbose):
libraries/mirrorTools.py
+8 -3
View File
@@ -185,6 +185,9 @@ class handlerFTP:
self.myFileSize = round(float(os.stat(file)[6])/1024,1)
self.mykByteCount = 0
if self.isFileAvailable(filename+".tmp"):
self.deleteFile(filename+".tmp")
if (ascii):
rc = self.ftpconn.storlines("STOR "+filename+".tmp",f)
else:
@@ -196,11 +199,13 @@ class handlerFTP:
return True
else:
return False
except socket.error: # connection reset by peer
except: # connection reset by peer
entropyTools.print_info(entropyTools.red("Upload issue, retrying..."))
self.reconnectHost() # reconnect
self.deleteFile(filename)
self.deleteFile(filename+".tmp")
if self.isFileAvailable(filename):
self.deleteFile(filename)
if self.isFileAvailable(filename+".tmp"):
self.deleteFile(filename+".tmp")
f.close()
continue
libraries/portageTools.py
+1
View File
@@ -425,6 +425,7 @@ def compareLibraryLists(pkgBinaryFiles,newPkgBinaryFiles):
print "DEBUG:"
print pkgBinaryFiles
print newPkgBinaryFiles
# check for version bumps
for pkg in pkgBinaryFiles:
libraries/reagentTools.py
+3
View File
@@ -107,7 +107,10 @@ def enzyme(options):
rc, newFileName = generator(tbz2path, enzymeRequestBump, dbconn)
if (rc):
etpCreated += 1
# create .hash file
hashFilePath = createHashFile(tbz2path)
os.system("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+newFileName+" -f")
os.system("mv "+hashFilePath+" "+etpConst['packagessuploaddir']+"/ -f")
else:
etpNotCreated += 1
os.system("rm -rf "+tbz2path)