diff --git a/TODO b/TODO index e8c6057ce..079e51909 100644 --- a/TODO +++ b/TODO @@ -8,11 +8,15 @@ TODO list (for developers only): - tidy tool: cleanup policy to prune old binaries - sync tool removal - database update and management between me and reagent + - create a .md5 for every in upload package + - add a counter to the upload command +- reagent tasks: + - reagent should support a forced revision bumping - Sabayon Linux USE flags: remove all server related use flags Project Status: -- entropy: not yet started, nothing to say then +- entropy: will handle all the three tools below - enzyme: first release up and working (more or less) - reagent: first release up and working. - activator: 60% @@ -22,4 +26,29 @@ Project Status: Features plan: - enzyme: distcc support on cross platforms and on automake -- activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one. \ No newline at end of file +- activator: add stable/ repository and the cron-aware module that moves files from the unstable/default repo to the stable one. +- activator: add support for ssh (where supported) to we can md5sum the uploaded files? + + + +Have a look at if this is solved: +Traceback (most recent call last): + File "activator", line 100, in ? + activatorTools.packages(options[1:]) + File "../libraries/activatorTools.py", line 222, in packages + ftp.uploadFile(etpConst['packagessuploaddir']+"/"+item[0]) + File "../libraries/entropyTools.py", line 852, in uploadFile + rc = self.ftpconn.storbinary("STOR "+file,f) + File "/usr/lib/python2.4/ftplib.py", line 421, in storbinary + return self.voidresp() + File "/usr/lib/python2.4/ftplib.py", line 221, in voidresp + resp = self.getresp() + File "/usr/lib/python2.4/ftplib.py", line 207, in getresp + resp = self.getmultiline() + File "/usr/lib/python2.4/ftplib.py", line 193, in getmultiline + line = self.getline() + File "/usr/lib/python2.4/ftplib.py", line 180, in getline + line = self.file.readline() + File "/usr/lib/python2.4/socket.py", line 332, in readline + data = self._sock.recv(self._rbufsize) +socket.error: (104, 'Connection reset by peer') diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index f05d4cfa5..4e6e9d35d 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -99,12 +99,12 @@ def packages(options): toBeUploaded.append(tbz2) uploadCounter += 1 print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready.")) - toBeDownloaded = [] # parse etpConst['packagesbindir'] + localPackagesRepository = [] # parse etpConst['packagesbindir'] print_info(green(" * ")+red("Calculating packages in ")+bold(etpConst['packagesbindir'])+red(" ..."), back = True) packageCounter = 0 for tbz2 in os.listdir(etpConst['packagesbindir']): if tbz2.endswith(".tbz2"): - toBeDownloaded.append(tbz2) + localPackagesRepository.append(tbz2) packageCounter += 1 print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready.")) @@ -125,6 +125,7 @@ def packages(options): print_info(green(" * ")+yellow("Calculating...")) uploadQueue = [] downloadQueue = [] + removalQueue = [] # Fill uploadQueue and if something weird is found, add the packages to downloadQueue # --> UPLOAD @@ -149,10 +150,10 @@ def packages(options): # so, we need to download it uploadQueue.append(localPackage) - # Fill downloadQueue and if something weird is found, add the packages to uploadQueue + # Fill downloadQueue and removalQueue for remotePackage in remotePackages: pkgfound = False - for localPackage in toBeDownloaded: + for localPackage in localPackagesRepository: if localPackage == remotePackage: pkgfound = True # it's already on the mirror, but... is its size correct?? @@ -162,7 +163,9 @@ def packages(options): if file.split()[8] == remotePackage: remoteSize = int(file.split()[4]) if (localSize != remoteSize) and (localSize != 0): - # size does not match, adding to the download queue + # size does not match, remove first + removalQueue.append(localPackage) + # then add to the download queue downloadQueue.append(remotePackage) break @@ -170,40 +173,73 @@ def packages(options): # this means that the local package does not exist # so, we need to download it downloadQueue.append(remotePackage) - + + # Fill removalQueue and downloadQueue + # if the online package does not exist anymore, we have to remove it locally + for localPackage in localPackagesRepository: + pkgfound = False + for remotePackage in remotePackages: + if localPackage == remotePackage: + pkgfound = True + break + + if (not pkgfound): + # this means that the local package does not exist + # so, we need to download it + removalQueue.append(localPackage) + + # filter duplicates - uploadQueue = list(set(uploadQueue)) + removalQueue = list(set(removalQueue)) downloadQueue = list(set(downloadQueue)) - moveQueue = [] - - if (len(uploadQueue) == 0) and (len(downloadQueue) == 0): - print_info(green(" * ")+red("Nothing to syncronize. Queues empty.")) - sys.exit(0) - - - totalUploadSize = 0 + uploadQueue = list(set(uploadQueue)) + + if (len(uploadQueue) == 0) and (len(downloadQueue) == 0) and (len(removalQueue) == 0): + print_info(green(" * ")+red("Nothing to syncronize for ")+bold(extractFTPHostFromUri(uri)+red(". Queues empty."))) + continue + + + totalRemovalSize = 0 totalDownloadSize = 0 + totalUploadSize = 0 + print_info(green(" * ")+yellow("Queue tasks:")) - detailedUploadQueue = [] + detailedRemovalQueue = [] detailedDownloadQueue = [] + detailedUploadQueue = [] + + for item in removalQueue: + fileSize = os.stat(etpConst['packagesbindir']+"/"+item)[6] + totalRemovalSize += int(fileSize) + print_info(bold("\t[") + red("REMOVAL") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedRemovalQueue.append([item,fileSize]) + import time + time.sleep(10) + + for item in downloadQueue: + # if the package is already in the upload directory, do not add the size + if not os.path.isfile(etpConst['packagessuploaddir']+"/"+item): + fileSize = "0" + for remotePackage in remotePackagesInfo: + if remotePackage.split()[8] == item: + fileSize = remotePackage.split()[4] + break + totalDownloadSize += int(fileSize) + print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) + detailedDownloadQueue.append([item,fileSize]) + for item in uploadQueue: fileSize = os.stat(etpConst['packagessuploaddir']+"/"+item)[6] totalUploadSize += int(fileSize) print_info(bold("\t[") + red("UPLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) detailedUploadQueue.append([item,fileSize]) - for item in downloadQueue: - fileSize = "0" - for remotePackage in remotePackagesInfo: - if remotePackage.split()[8] == item: - fileSize = remotePackage.split()[4] - break - totalDownloadSize += int(fileSize) - print_info(bold("\t[") + yellow("DOWNLOAD") + bold("] ") + red(item.split(".tbz2")[0]) + bold(".tbz2 ") + blue(bytesIntoHuman(fileSize))) - detailedDownloadQueue.append([item,fileSize]) - print_info(red(" * ")+blue("Packages that would be ")+red("uploaded:\t\t")+bold(str(len(uploadQueue)))) + + print_info(red(" * ")+blue("Packages that would be ")+red("removed:\t\t")+bold(str(len(removalQueue)))) print_info(red(" * ")+blue("Packages that would be ")+yellow("downloaded:\t")+bold(str(len(downloadQueue)))) - print_info(red(" * ")+blue("Total upload ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize)))) + print_info(red(" * ")+blue("Packages that would be ")+green("uploaded:\t\t")+bold(str(len(uploadQueue)))) + print_info(red(" * ")+blue("Total removal ")+red("size:\t\t\t")+bold(bytesIntoHuman(str(totalRemovalSize)))) print_info(red(" * ")+blue("Total download ")+yellow("size:\t\t\t")+bold(bytesIntoHuman(str(totalDownloadSize)))) + print_info(red(" * ")+blue("Total upload ")+green("size:\t\t\t")+bold(bytesIntoHuman(str(totalUploadSize)))) if (activatorRequestAsk): rc = askquestion("\n Would you like to run the steps above ?") @@ -212,7 +248,14 @@ def packages(options): continue elif (activatorRequestPretend): continue - + + # removal queue + if (detailedRemovalQueue != []): + for item in detailedRemovalQueue: + print_info(red(" * Removing file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(etpConst['packagesbindir']) +red(" ..."),back = True) + os.system("rm -f "+etpConst['packagesbindir']+"/"+item[0]) + print_info(red(" * Removal completed for ")+bold(etpConst['packagesbindir'])) + # upload queue if (detailedUploadQueue != []): ftp = activatorFTP(uri) @@ -225,7 +268,7 @@ def packages(options): print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) ftp.closeFTPConnection() - # for the download queue, also check in the upload directory + # download queue if (detailedDownloadQueue != []): ftp = activatorFTP(uri) ftp.setCWD(etpConst['binaryurirelativepath']) @@ -240,9 +283,9 @@ def packages(options): print_info(red(" * Downloading file ")+bold(item[0]) + red(" [")+blue(bytesIntoHuman(item[1]))+red("] from ")+ bold(extractFTPHostFromUri(uri)) +red(" ..."),back = True) ftp.downloadFile(item[0],etpConst['packagesbindir']+"/") - print_info(red(" * Upload completed for ")+bold(extractFTPHostFromUri(uri))) + print_info(red(" * Download completed for ")+bold(extractFTPHostFromUri(uri))) ftp.closeFTPConnection() - + # Now I should do some tidy print "Now it should be time for some tidy...?" diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index d424371fe..8351eeba4 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -780,6 +780,14 @@ class activatorFTP: self.ftpconn.login(self.ftpuser,self.ftppassword) # change to our dir self.ftpconn.cwd(self.ftpdir) + self.currentdir = self.ftpdir + + + # this can be used in case of exceptions + def reconnectHost(self): + self.ftpconn = FTP(self.ftphost) + self.ftpconn.login(self.ftpuser,self.ftppassword) + self.ftpconn.cwd(self.currentdir) def getFTPHost(self): return self.ftphost @@ -795,6 +803,7 @@ class activatorFTP: def setCWD(self,dir): self.ftpconn.cwd(dir) + self.currentdir = dir def getFileMtime(self,path): rc = self.ftpconn.sendcmd("mdtm "+path) @@ -846,31 +855,33 @@ class activatorFTP: return False def uploadFile(self,file,ascii = False): - if (not ascii): + for i in range(10): # ten tries f = open(file) file = file.split("/")[len(file.split("/"))-1] - rc = self.ftpconn.storbinary("STOR "+file,f) - return rc - else: - f = open(file) - file = file.split("/")[len(file.split("/"))-1] - rc = self.ftpconn.storlines("STOR "+file,f) - return rc + try: + if (ascii): + rc = self.ftpconn.storlines("STOR "+file,f) + else: + rc = self.ftpconn.storbinary("STOR "+file,f) + return rc + except socket.error: # connection reset by peer + print_info(red("Upload issue, retrying...")) + self.reconnectHost() # reconnect + self.deleteFile(file) + f.close() + continue def downloadFile(self,filepath,downloaddir,ascii = False): file = filepath.split("/")[len(filepath.split("/"))-1] if (not ascii): f = open(downloaddir+"/"+file,"wb") rc = self.ftpconn.retrbinary('RETR '+file,f.write) - f.flush() - f.close() - return rc else: f = open(downloaddir+"/"+file,"w") rc = self.ftpconn.retrlines('RETR '+file,f.write) - f.flush() - f.close() - return rc + f.flush() + f.close() + return rc # also used to move files # FIXME: beautify ! @@ -1605,7 +1616,7 @@ def print_error(msg): print red(">>")+" "+msg def print_info(msg, back = False): - writechar("\r \r") + writechar("\r \r") if (back): writechar("\r"+green(">>")+" "+msg) return