sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

activator packages sync fixes

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@202 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2007-03-28 09:24:31 +00:00
parent 8175b17efe
commit d31b54cca7
3 changed files with 131 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TODO
+31 -2
View File
@@ -8,11 +8,15 @@ TODO list (for developers only):
- tidy tool: cleanup policy to prune old binaries
- sync tool removal
- database update and management between me and reagent
- create a .md5 for every in upload package
- add a counter to the upload command
- reagent tasks:
- reagent should support a forced revision bumping
- Sabayon Linux USE flags: remove all server related use flags
Project Status:
- entropy: not yet started, nothing to say then
- entropy: will handle all the three tools below
- enzyme: first release up and working (more or less)
- reagent: first release up and working.
- activator: 60%
@@ -22,4 +26,29 @@ Project Status:
Features plan:
- enzyme: distcc support on cross platforms and on automake
- activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one.
- activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one.
- activator: add support for ssh (where supported) to we can md5sum the uploaded files?
Have a look at if this is solved:
Traceback (most recent call last):
File "activator", line 100, in ?
activatorTools.packages(options[1:])
File "../libraries/activatorTools.py", line 222, in packages
ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0])
File "../libraries/entropyTools.py", line 852, in uploadFile
rc = self.ftpconn.storbinary("STOR "+file,f)
File "/usr/lib/python2.4/ftplib.py", line 421, in storbinary
return self.voidresp()
File "/usr/lib/python2.4/ftplib.py", line 221, in voidresp
resp = self.getresp()
File "/usr/lib/python2.4/ftplib.py", line 207, in getresp
resp = self.getmultiline()
File "/usr/lib/python2.4/ftplib.py", line 193, in getmultiline
line = self.getline()
File "/usr/lib/python2.4/ftplib.py", line 180, in getline
line = self.file.readline()
File "/usr/lib/python2.4/socket.py", line 332, in readline
data = self._sock.recv(self._rbufsize)
socket.error: (104, 'Connection reset by peer')
libraries/activatorTools.py
+74 -31
View File
@@ -99,12 +99,12 @@ def packages(options):
toBeUploaded.append(tbz2)
uploadCounter += 1
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
toBeDownloaded = [] # parse etpConst['packagesbindir']
localPackagesRepository = [] # parse etpConst['packagesbindir']
print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True)
packageCounter = 0
for tbz2 in os.listdir(etpConst['packagesbindir']):
if tbz2.endswith(".tbz2"):
toBeDownloaded.append(tbz2)
localPackagesRepository.append(tbz2)
packageCounter += 1
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
@@ -125,6 +125,7 @@ def packages(options):
print_info(green(" * ")+yellow("Calculating..."))
uploadQueue = []
downloadQueue = []
removalQueue = []
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
# --> UPLOAD
@@ -149,10 +150,10 @@ def packages(options):
# so, we need to download it
uploadQueue.append(localPackage)
# Fill downloadQueue and if something weird is found, add the packages to uploadQueue
# Fill downloadQueue and removalQueue
for remotePackage in remotePackages:
pkgfound = False
for localPackage in toBeDownloaded:
for localPackage in localPackagesRepository:
if localPackage == remotePackage:
pkgfound = True
# it's already on the mirror, but... is its size correct??
@@ -162,7 +163,9 @@ def packages(options):
if file.split()[8] == remotePackage:
remoteSize = int(file.split()[4])
if (localSize != remoteSize) and (localSize != 0):
# size does not match, adding to the download queue
# size does not match, remove first
removalQueue.append(localPackage)
# then add to the download queue
downloadQueue.append(remotePackage)
break
@@ -170,40 +173,73 @@ def packages(options):
# this means that the local package does not exist
# so, we need to download it
downloadQueue.append(remotePackage)
# Fill removalQueue and downloadQueue
# if the online package does not exist anymore, we have to remove it locally
for localPackage in localPackagesRepository:
pkgfound = False
for remotePackage in remotePackages:
if localPackage == remotePackage:
pkgfound = True
break
if (not pkgfound):
# this means that the local package does not exist
# so, we need to download it
removalQueue.append(localPackage)
# filter duplicates
uploadQueue = list(set(uploadQueue))
removalQueue = list(set(removalQueue))
downloadQueue = list(set(downloadQueue))
moveQueue = []
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0):
print_info(green(" * ")+red("Nothing to syncronize. Queues empty."))
sys.exit(0)
totalUploadSize = 0
uploadQueue = list(set(uploadQueue))
if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0):
print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queues empty.")))
continue
totalRemovalSize = 0
totalDownloadSize = 0
totalUploadSize = 0
print_info(green(" * ")+yellow("Queue tasks:"))
detailedUploadQueue = []
detailedRemovalQueue = []
detailedDownloadQueue = []
detailedUploadQueue = []
for item in removalQueue:
fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6]
totalRemovalSize += int(fileSize)
print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedRemovalQueue.append([item,fileSize])
import time
time.sleep(10)
for item in downloadQueue:
# if the package is already in the upload directory, do not add the size
if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item):
fileSize = "0"
for remotePackage in remotePackagesInfo:
if remotePackage.split()[8] == item:
fileSize = remotePackage.split()[4]
break
totalDownloadSize += int(fileSize)
print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedDownloadQueue.append([item,fileSize])
for item in uploadQueue:
fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6]
totalUploadSize += int(fileSize)
print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedUploadQueue.append([item,fileSize])
for item in downloadQueue:
fileSize = "0"
for remotePackage in remotePackagesInfo:
if remotePackage.split()[8] == item:
fileSize = remotePackage.split()[4]
break
totalDownloadSize += int(fileSize)
print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize)))
detailedDownloadQueue.append([item,fileSize])
print_info(red(" * ")+blue("Packages that would be ")+red("uploaded:\t\t")+bold(str(len(uploadQueue))))
print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue))))
print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue))))
print_info(red(" * ")+blue("Total upload ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue))))
print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize))))
print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize))))
print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize))))
if (activatorRequestAsk):
rc = askquestion("\n Would you like to run the steps above ?")
@@ -212,7 +248,14 @@ def packages(options):
continue
elif (activatorRequestPretend):
continue
# removal queue
if (detailedRemovalQueue != []):
for item in detailedRemovalQueue:
print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True)
os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0])
print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir']))
# upload queue
if (detailedUploadQueue != []):
ftp = activatorFTP(uri)
@@ -225,7 +268,7 @@ def packages(options):
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
# for the download queue, also check in the upload directory
# download queue
if (detailedDownloadQueue != []):
ftp = activatorFTP(uri)
ftp.setCWD(etpConst['binaryurirelativepath'])
@@ -240,9 +283,9 @@ def packages(options):
print_info(red(" * Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True)
ftp.downloadFile(item[0],etpConst['packagesbindir']+"/")
print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri)))
print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri)))
ftp.closeFTPConnection()
# Now I should do some tidy
print "Now it should be time for some tidy...?"
libraries/entropyTools.py
+26 -15
View File
@@ -780,6 +780,14 @@ class activatorFTP:
self.ftpconn.login(self.ftpuser,self.ftppassword)
# change to our dir
self.ftpconn.cwd(self.ftpdir)
self.currentdir = self.ftpdir
# this can be used in case of exceptions
def reconnectHost(self):
self.ftpconn = FTP(self.ftphost)
self.ftpconn.login(self.ftpuser,self.ftppassword)
self.ftpconn.cwd(self.currentdir)
def getFTPHost(self):
return self.ftphost
@@ -795,6 +803,7 @@ class activatorFTP:
def setCWD(self,dir):
self.ftpconn.cwd(dir)
self.currentdir = dir
def getFileMtime(self,path):
rc = self.ftpconn.sendcmd("mdtm "+path)
@@ -846,31 +855,33 @@ class activatorFTP:
return False
def uploadFile(self,file,ascii = False):
if (not ascii):
for i in range(10): # ten tries
f = open(file)
file = file.split("/")[len(file.split("/"))-1]
rc = self.ftpconn.storbinary("STOR "+file,f)
return rc
else:
f = open(file)
file = file.split("/")[len(file.split("/"))-1]
rc = self.ftpconn.storlines("STOR "+file,f)
return rc
try:
if (ascii):
rc = self.ftpconn.storlines("STOR "+file,f)
else:
rc = self.ftpconn.storbinary("STOR "+file,f)
return rc
except socket.error: # connection reset by peer
print_info(red("Upload issue, retrying..."))
self.reconnectHost() # reconnect
self.deleteFile(file)
f.close()
continue
def downloadFile(self,filepath,downloaddir,ascii = False):
file = filepath.split("/")[len(filepath.split("/"))-1]
if (not ascii):
f = open(downloaddir+"/"+file,"wb")
rc = self.ftpconn.retrbinary('RETR '+file,f.write)
f.flush()
f.close()
return rc
else:
f = open(downloaddir+"/"+file,"w")
rc = self.ftpconn.retrlines('RETR '+file,f.write)
f.flush()
f.close()
return rc
f.flush()
f.close()
return rc
# also used to move files
# FIXME: beautify !
@@ -1605,7 +1616,7 @@ def print_error(msg):
print red(">>")+" "+msg
def print_info(msg, back = False):
writechar("\r \r")
writechar("\r \r")
if (back):
writechar("\r"+green(">>")+" "+msg)
return