sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

some heavy work on activator

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@201 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-03-27 23:02:26 +00:00
parent eb185e9675
commit 8175b17efe
4 changed files with 157 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
TODO
View File

@@ -1,9 +1,13 @@
TODO list (for developers only):
- build() and world(), on enzyme, add the support for whitelist+cron
- build(), on enzyme, add license blacklist (packages that cannot be shipped in a binary form)
- system-wide: add free space check
- activator: for the tidy module I need that reagent includes the SLOT variable.
- activator tasks:
- add/remove packages from the database
- cleanup policy to prune old binaries
- tidy tool: cleanup policy to prune old binaries
- sync tool removal
- database update and management between me and reagent
- Sabayon Linux USE flags: remove all server related use flags
@@ -11,11 +15,11 @@ Project Status:
- entropy: not yet started, nothing to say then
- enzyme: first release up and working (more or less)
- reagent: first release up and working.
- activator: 50%
- activator: 60%
============
- client part: not yet defined apart from equilibrium name.
Features plan:
- enzyme: distcc support
- enzyme: distcc support on cross platforms and on automake
- activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one.

2
handlers/activator vendored
View File

@@ -47,6 +47,8 @@ def print_help():
entropyTools.print_info(" \t\t"+entropyTools.red("--download-etp")+"\t\t to download all the Entropy tree (will overwrite)")
entropyTools.print_info(" \t\t"+entropyTools.red("--show-stats")+"\t\t shows the list of validated package/etp-file couples")
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("packages"))+entropyTools.yellow("\t to manage binary packages"))
entropyTools.print_info(" \t\t"+entropyTools.red("--ask")+"\t\t\t ask before making any changes")
entropyTools.print_info(" \t\t"+entropyTools.red("--pretend")+"\t\t just show what would be done")
entropyTools.print_info(" \t\t"+entropyTools.green("sync")+entropyTools.red("\t\t\t to sync the binary packages across primary mirrors"))
entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("database"))+entropyTools.yellow("\t to manage database status and settings"))
entropyTools.print_info(" \t\t"+entropyTools.green("lock")+entropyTools.red("\t\t\t to lock the database(s) status ["+entropyTools.yellow("DANGEROUS")+entropyTools.red("]")))

143
libraries/activatorTools.py
View File

@@ -41,10 +41,6 @@ def sync(options):
#print ftp.getFileSize("index.htm")
list = ftp.getRoughList()
for item in list:
if item.find("index.htm") != -1:
# extact the size
print item
#print ftp.spawnFTPCommand("mdtm index.htm")
ftp.closeFTPConnection()
@@ -79,13 +75,21 @@ def sync(options):
def packages(options):
# FIXME: add support for --ask
# FIXME: complete this part
# Options available for all the packages submodules
myopts = options[1:]
activatorRequestAsk = False
activatorRequestPretend = False
for opt in myopts:
if (opt == "--ask"):
activatorRequestAsk = True
elif (opt == "--pretend"):
activatorRequestPretend = True
if (options[0] == "sync"):
print_info(green(" * ")+red("Starting ")+bold("binary")+yellow(" packages")+red(" syncronization across servers ..."))
for uri in etpConst['activatoruploaduris']:
print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+" mirror."))
print_info(green(" * ")+yellow("Working on ")+bold(extractFTPHostFromUri(uri)+red(" mirror.")))
print_info(green(" * ")+yellow("Local Statistics"))
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagessuploaddir'])+red(" ..."), back = True)
uploadCounter = 0
@@ -104,13 +108,14 @@ def packages(options):
packageCounter += 1
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
print_info(green(" * ")+yellow("Remote statistics"))
print_info(green(" * ")+yellow("Fetching remote statistics..."), back = True)
ftp = activatorFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
remotePackages = ftp.listFTPdir()
remotePackagesInfo = ftp.getRoughList()
ftp.closeFTPConnection()
print_info(green(" * ")+yellow("Remote statistics"))
remoteCounter = 0
for tbz2 in remotePackages:
if tbz2.endswith(".tbz2"):
@@ -118,16 +123,128 @@ def packages(options):
print_info(green(" * ")+red("Remote packages:\t\t")+bold(str(remoteCounter))+red(" files stored."))
print_info(green(" * ")+yellow("Calculating..."))
# now it's time to compare what I have to upload
for remotePackage in remotePackages:
pkgfound = True
for localPackage in toBeUploaded:
uploadQueue = []
downloadQueue = []
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
# --> UPLOAD
for localPackage in toBeUploaded:
pkgfound = False
for remotePackage in remotePackages:
if localPackage == remotePackage:
pkgfound = True
# it's already on the mirror, but... is its size correct??
remoteSize = ftp.getFileSize(remotePackage)
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+localPackage)[6])
remoteSize = 0
for file in remotePackagesInfo:
if file.split()[8] == remotePackage:
remoteSize = int(file.split()[4])
if (localSize != remoteSize) and (localSize != 0):
# size does not match, adding to the upload queue
uploadQueue.append(localPackage)
break
if (not pkgfound):
# this means that the local package does not exist
# so, we need to download it
uploadQueue.append(localPackage)
# Fill downloadQueue and if something weird is found, add the packages to uploadQueue
for remotePackage in remotePackages:
pkgfound = False
for localPackage in toBeDownloaded:
if localPackage == remotePackage:
pkgfound = True
# it's already on the mirror, but... is its size correct??
localSize = int(os.stat(etpConst['packagesbindir']+"/"+localPackage)[6])
remoteSize = 0
for file in remotePackagesInfo:
if file.split()[8] == remotePackage:
remoteSize = int(file.split()[4])
if (localSize != remoteSize) and (localSize != 0):
# size does not match, adding to the download queue
downloadQueue.append(remotePackage)
break
if (not pkgfound):
# this means that the local package does not exist
# so, we need to download it
downloadQueue.append(remotePackage)
# filter duplicates
uploadQueue = list(set(uploadQueue))
downloadQueue = list(set(downloadQueue))
moveQueue = []
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0):
print_info(green(" * ")+red("Nothing to syncronize. Queues empty."))
sys.exit(0)
totalUploadSize = 0
totalDownloadSize = 0
print_info(green(" * ")+yellow("Queue tasks:"))
detailedUploadQueue = []
detailedDownloadQueue = []
for item in uploadQueue:
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
totalUploadSize += int(fileSize)
print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedUploadQueue.append([item,fileSize])
for item in downloadQueue:
fileSize = "0"
for remotePackage in remotePackagesInfo:
if remotePackage.split()[8] == item:
fileSize = remotePackage.split()[4]
break
totalDownloadSize += int(fileSize)
print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedDownloadQueue.append([item,fileSize])
print_info(red(" * ")+blue("Packages that would be ")+red("uploaded:\t\t")+bold(str(len(uploadQueue))))
print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue))))
print_info(red(" * ")+blue("Total upload ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize))))
if (activatorRequestAsk):
rc = askquestion("\n Would you like to run the steps above ?")
if rc == "No":
print "\n"
continue
elif (activatorRequestPretend):
continue
# upload queue
if (detailedUploadQueue != []):
ftp = activatorFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
for item in detailedUploadQueue:
print_info(red(" * Uploading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] to ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
# now move the file into etpConst['packagesbindir']
os.system("mv "+etpConst['packagessuploaddir']+"/"+item[0]+" "+etpConst['packagesbindir']+"/")
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
# for the download queue, also check in the upload directory
if (detailedDownloadQueue != []):
ftp = activatorFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
for item in detailedDownloadQueue:
if os.path.isfile(etpConst['packagessuploaddir']+"/"+item[0]):
localSize = int(os.stat(etpConst['packagessuploaddir']+"/"+item[0])[6])
remoteSize = int(item[1])
if localSize == remoteSize:
print_info(red(" * Moving file ")+bold(item[0])+red(" to ")+bold(etpConst['packagesbindir'])+red(" ..."),back = True)
os.system("mv "+etpConst['packagessuploaddir']+"/"+item[0]+" "+etpConst['packagesbindir']+"/")
continue
print_info(red(" * Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
# Now I should do some tidy
print "Now it should be time for some tidy...?"
def database(options):

18
libraries/entropyTools.py
View File

@@ -880,6 +880,14 @@ class activatorFTP:
# not supported by dreamhost.com
def getFileSize(self,file):
return self.ftpconn.size(file)
def getFileSizeCompat(self,file):
list = getRoughList()
for item in list:
if item.find(file) != -1:
# extact the size
return item.split()[4]
return ""
def bufferizer(self,buf):
self.FTPbuffer.append(buf)
@@ -1202,6 +1210,16 @@ def digestFile(filepath):
digest.update(line)
return digest.hexdigest()
def bytesIntoHuman(bytes):
bytes = str(bytes)
kbytes = str(int(bytes)/1024)
if len(kbytes) > 3:
kbytes = str(int(kbytes)/1024)
kbytes += "MB"
else:
kbytes += "kB"
return kbytes
# hide password from full ftp URI
def hideFTPpassword(uri):
ftppassword = uri.split("@")[:len(uri.split("@"))-1]