diff --git a/TODO b/TODO index 705bd7897..8cc764edb 100644 --- a/TODO +++ b/TODO @@ -1,6 +1,5 @@ TODO list: CLIENT: - - keep outdated packages on the server for a week - find a way to better handle real smartapps deps - add support for database revisions log - add external trigger for splashutils diff --git a/conf/equo.conf b/conf/equo.conf index 80b9dce9e..afde69f1c 100644 --- a/conf/equo.conf +++ b/conf/equo.conf @@ -51,7 +51,7 @@ configprotectmask| # default: unset, internal variable will be used (this is stored into the repository database) # # 2: Installation/Removal protection skip -configprotectskip|/etc/conf.d/net /etc/X11/xorg.conf /usr/kde/3.5/share/config/kdm/kdmrc +configprotectskip|/etc/conf.d/net /etc/X11/xorg.conf /usr/kde/3.5/share/config/kdm/kdmrc /etc/conf.d/clock /etc/conf.d/hostname /etc/conf.d/keymaps # Log level # 0: No Logging diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index a18b78b4e..5aef2b4a6 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -101,8 +101,14 @@ def sync(options, justTidy = False): for repoBin in repoBinaries: if (not repoBin.endswith(etpConst['packageshashfileext'])): if repoBin not in dbBinaries: - removeList.append(repoBin) - + # check if it's expired + filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+repoBin + mtime = getFileUnixMtime(filepath) + delta = int(etpConst['packagesexpirationdays'])*24*3600 + currmtime = time.time() + if currmtime - mtime > delta: # if it's expired + removeList.append(repoBin) + if (not removeList): activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: no packages to remove from the lirrors.") print_info(green(" * ")+red("No packages to remove from the mirrors.")) @@ -615,7 +621,7 @@ def packages(options): totalSuccessfulUri += 1 # trap exceptions, failed to upload/download someting? - except Exception, e: + except Exception, e: # FIXME: only trap proper ftp exceptions print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Showing traceback:")) import traceback diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 7711b1c39..59e237ccf 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -1207,6 +1207,10 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setDigest: setting new digest for idpackage: "+str(idpackage)+" -> "+str(digest)) self.cursor.execute('UPDATE extrainfo SET digest = "'+str(digest)+'" WHERE idpackage = "'+str(idpackage)+'"') + def setDownloadURL(self, idpackage, url): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setDownloadURL: setting new download for idpackage: "+str(idpackage)+" -> "+url) + self.cursor.execute('UPDATE extrainfo SET download = "'+url+'" WHERE idpackage = "'+str(idpackage)+'"') + def setCounter(self, idpackage, counter): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setCounter: setting new counter for idpackage: "+str(idpackage)+" -> "+str(counter)) try: diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 696c82215..f88bc32cd 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -369,6 +369,7 @@ etpConst = { "gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",) }, 'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file + 'packagesexpirationdays': 15, # number of days after a package will be removed from mirrors 'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools 'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info) @@ -411,7 +412,7 @@ etpConst = { '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' ], 'officialrepositoryname': "sabayonlinux.org", # our official repository name - 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database + 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes) 'pidfile': "/var/run/equo.pid", 'applicationlock': False, 'collisionprotect': 1, # collision protection option, read equo.conf for more info diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index abae94d6e..0b5f70ffa 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -704,7 +704,15 @@ def remove_revision(ver): def remove_tag(mydep): colon = mydep.rfind("#") if colon != -1: - mydep = mydep[:colon] + mystring = mydep[:colon] + return mystring + return mydep + +def remove_entropy_revision(mydep): + colon = mydep.rfind("~") + if colon != -1: + mystring = mydep[:colon] + return mystring return mydep dep_gettagCache = {} @@ -1394,6 +1402,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False): if not silent: print_info(yellow(" * ")+red(info_package+"Getting package name/version..."),back = True) tbz2File = package package = package.split(".tbz2")[0] + package = remove_entropy_revision(package) package = remove_tag(package) # FIXME: deprecated - will be removed soonly diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 55d61654b..66229835d 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -28,6 +28,7 @@ from entropyConstants import * from serverConstants import * from entropyTools import * import os +import shutil # Logging initialization import logTools @@ -47,7 +48,6 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran print_warning(package+" does not exist !") packagename = os.path.basename(package) - print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait...")) mydata = extractPkgData(package, enzymeRequestBranch) @@ -55,7 +55,7 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran dbconn = databaseTools.openServerDatabase(readOnly = False, noUpload = True) else: dbconn = dbconnection - + idpk, revision, etpDataUpdated, accepted = dbconn.handlePackage(mydata) # add package info to our official repository etpConst['officialrepositoryname'] @@ -63,25 +63,24 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran dbconn.removePackageFromInstalledTable(idpk) dbconn.addPackageToInstalledTable(idpk,etpConst['officialrepositoryname']) - # return back also the new possible package filename, so that we can make decisions on that - newFileName = os.path.basename(etpDataUpdated['download']) - if dbconnection is None: dbconn.commitChanges() dbconn.closeDB() + packagename = packagename[:-5]+"~"+str(revision)+".tbz2" + if (accepted) and (revision != 0): reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" has been updated to revision: "+str(revision)) print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision))) - return True, newFileName, idpk + return True, idpk elif (accepted) and (revision == 0): reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created or version bumped.") print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created.")) - return True, newFileName, idpk + return True, idpk else: reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" kept intact, no updates needed.") print_info(green(" * ")+red("Package ")+bold(packagename)+red(" does not need to be updated. Current revision: ")+bold(str(revision))) - return False, newFileName, idpk + return False, idpk # This tool is used by Entropy after enzyme, it simply parses the content of etpConst['packagesstoredir'] @@ -261,15 +260,26 @@ def update(options): etpNotCreated = 0 for tbz2 in tbz2files: counter += 1 - tbz2name = tbz2.split("/")[len(tbz2.split("/"))-1] + tbz2name = tbz2.split("/")[-1] print_info(" ("+str(counter)+"/"+str(totalCounter)+") Processing "+tbz2name) tbz2path = etpConst['packagesstoredir']+"/"+tbz2 - rc, newFileName, idpk = generator(tbz2path, dbconn, enzymeRequestBranch) + rc, idpk = generator(tbz2path, dbconn, enzymeRequestBranch) if (rc): etpCreated += 1 - # move the file with its new name - spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName+" -f") - print_info(yellow(" * ")+red("Injecting database information into ")+bold(newFileName)+red(", please wait..."), back = True) + + # add revision to package file + downloadurl = dbconn.retrieveDownloadURL(idpk) + packagerev = dbconn.retrieveRevision(idpk) + downloaddir = os.path.dirname(downloadurl) + downloadfile = os.path.basename(downloadurl) + # remove tbz2 and add revision + downloadfile = downloadfile[:-5]+"~"+str(packagerev)+".tbz2" + downloadurl = downloaddir+"/"+downloadfile + # update url + dbconn.setDownloadURL(idpk,downloadurl) + + shutil.move(tbz2path,etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile) + print_info(yellow(" * ")+red("Injecting database information into ")+bold(downloadfile)+red(", please wait..."), back = True) dbpath = etpConst['packagestmpdir']+"/"+str(getRandomNumber()) while os.path.isfile(dbpath): @@ -283,14 +293,14 @@ def update(options): pkgDbconn.addPackage(data, revision = rev) pkgDbconn.closeDB() # append the database to the new file - aggregateEdb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName, dbfile = dbpath) + aggregateEdb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile, dbfile = dbpath) - digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName) + digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile) dbconn.setDigest(idpk,digest) - hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName) + hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile) # remove garbage os.remove(dbpath) - print_info(yellow(" * ")+red("Database injection complete for ")+newFileName) + print_info(yellow(" * ")+red("Database injection complete for ")+downloadfile) else: etpNotCreated += 1 diff --git a/server/activator b/server/activator index ecfde5bda..72008d0b8 100644 --- a/server/activator +++ b/server/activator @@ -42,7 +42,7 @@ def print_help(): print_info(blue("Tools available: ")) print_info(" \t"+green(bold("sync"))+yellow("\t\t to sync packages, database and also do some tidy")) print_info(" \t\t"+red("--noask")+"\t\t\t do not make any question") - print_info(" \t"+green(bold("tidy"))+yellow("\t\t to just remove binary packages that are not in the Entropy database anymore")) + print_info(" \t"+green(bold("tidy"))+yellow("\t\t to just remove binary packages that are not in database and are expired")) print_info(" \t"+green(bold("packages"))+yellow("\t to manage binary packages")) print_info(" \t\t"+green("sync")+red("\t\t\t to sync the binary packages across primary mirrors")) print_info(" \t\t\t"+red("--ask")+"\t\t\t ask before making any changes")