- added revision to package archives
- remove packages from mirrors only when they are expired -> etpConst['packagesexpirationdays'] - minor changes/updates git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@746 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
@@ -1,6 +1,5 @@
|
||||
TODO list:
|
||||
CLIENT:
|
||||
- keep outdated packages on the server for a week
|
||||
- find a way to better handle real smartapps deps
|
||||
- add support for database revisions log
|
||||
- add external trigger for splashutils
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ configprotectmask|
|
||||
# default: unset, internal variable will be used (this is stored into the repository database)
|
||||
#
|
||||
# 2: Installation/Removal protection skip
|
||||
configprotectskip|/etc/conf.d/net /etc/X11/xorg.conf /usr/kde/3.5/share/config/kdm/kdmrc
|
||||
configprotectskip|/etc/conf.d/net /etc/X11/xorg.conf /usr/kde/3.5/share/config/kdm/kdmrc /etc/conf.d/clock /etc/conf.d/hostname /etc/conf.d/keymaps
|
||||
|
||||
# Log level
|
||||
# 0: No Logging
|
||||
|
||||
@@ -101,8 +101,14 @@ def sync(options, justTidy = False):
|
||||
for repoBin in repoBinaries:
|
||||
if (not repoBin.endswith(etpConst['packageshashfileext'])):
|
||||
if repoBin not in dbBinaries:
|
||||
removeList.append(repoBin)
|
||||
|
||||
# check if it's expired
|
||||
filepath = etpConst['packagesbindir']+"/"+mybranch+"/"+repoBin
|
||||
mtime = getFileUnixMtime(filepath)
|
||||
delta = int(etpConst['packagesexpirationdays'])*24*3600
|
||||
currmtime = time.time()
|
||||
if currmtime - mtime > delta: # if it's expired
|
||||
removeList.append(repoBin)
|
||||
|
||||
if (not removeList):
|
||||
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: no packages to remove from the lirrors.")
|
||||
print_info(green(" * ")+red("No packages to remove from the mirrors."))
|
||||
@@ -615,7 +621,7 @@ def packages(options):
|
||||
totalSuccessfulUri += 1
|
||||
|
||||
# trap exceptions, failed to upload/download someting?
|
||||
except Exception, e:
|
||||
except Exception, e: # FIXME: only trap proper ftp exceptions
|
||||
|
||||
print_error(yellow(" * ")+red("packages: Exception caught: ")+str(e)+red(" . Showing traceback:"))
|
||||
import traceback
|
||||
|
||||
@@ -1207,6 +1207,10 @@ class etpDatabase:
|
||||
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setDigest: setting new digest for idpackage: "+str(idpackage)+" -> "+str(digest))
|
||||
self.cursor.execute('UPDATE extrainfo SET digest = "'+str(digest)+'" WHERE idpackage = "'+str(idpackage)+'"')
|
||||
|
||||
def setDownloadURL(self, idpackage, url):
|
||||
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setDownloadURL: setting new download for idpackage: "+str(idpackage)+" -> "+url)
|
||||
self.cursor.execute('UPDATE extrainfo SET download = "'+url+'" WHERE idpackage = "'+str(idpackage)+'"')
|
||||
|
||||
def setCounter(self, idpackage, counter):
|
||||
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"setCounter: setting new counter for idpackage: "+str(idpackage)+" -> "+str(counter))
|
||||
try:
|
||||
|
||||
@@ -369,6 +369,7 @@ etpConst = {
|
||||
"gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",)
|
||||
},
|
||||
'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file
|
||||
'packagesexpirationdays': 15, # number of days after a package will be removed from mirrors
|
||||
'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools
|
||||
|
||||
'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info)
|
||||
@@ -411,7 +412,7 @@ etpConst = {
|
||||
'/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage'
|
||||
],
|
||||
'officialrepositoryname': "sabayonlinux.org", # our official repository name
|
||||
'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database
|
||||
'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes)
|
||||
'pidfile': "/var/run/equo.pid",
|
||||
'applicationlock': False,
|
||||
'collisionprotect': 1, # collision protection option, read equo.conf for more info
|
||||
|
||||
@@ -704,7 +704,15 @@ def remove_revision(ver):
|
||||
def remove_tag(mydep):
|
||||
colon = mydep.rfind("#")
|
||||
if colon != -1:
|
||||
mydep = mydep[:colon]
|
||||
mystring = mydep[:colon]
|
||||
return mystring
|
||||
return mydep
|
||||
|
||||
def remove_entropy_revision(mydep):
|
||||
colon = mydep.rfind("~")
|
||||
if colon != -1:
|
||||
mystring = mydep[:colon]
|
||||
return mystring
|
||||
return mydep
|
||||
|
||||
dep_gettagCache = {}
|
||||
@@ -1394,6 +1402,7 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False):
|
||||
if not silent: print_info(yellow(" * ")+red(info_package+"Getting package name/version..."),back = True)
|
||||
tbz2File = package
|
||||
package = package.split(".tbz2")[0]
|
||||
package = remove_entropy_revision(package)
|
||||
package = remove_tag(package)
|
||||
|
||||
# FIXME: deprecated - will be removed soonly
|
||||
|
||||
+27
-17
@@ -28,6 +28,7 @@ from entropyConstants import *
|
||||
from serverConstants import *
|
||||
from entropyTools import *
|
||||
import os
|
||||
import shutil
|
||||
|
||||
# Logging initialization
|
||||
import logTools
|
||||
@@ -47,7 +48,6 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran
|
||||
print_warning(package+" does not exist !")
|
||||
|
||||
packagename = os.path.basename(package)
|
||||
|
||||
print_info(yellow(" * ")+red("Processing: ")+bold(packagename)+red(", please wait..."))
|
||||
mydata = extractPkgData(package, enzymeRequestBranch)
|
||||
|
||||
@@ -55,7 +55,7 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran
|
||||
dbconn = databaseTools.openServerDatabase(readOnly = False, noUpload = True)
|
||||
else:
|
||||
dbconn = dbconnection
|
||||
|
||||
|
||||
idpk, revision, etpDataUpdated, accepted = dbconn.handlePackage(mydata)
|
||||
|
||||
# add package info to our official repository etpConst['officialrepositoryname']
|
||||
@@ -63,25 +63,24 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran
|
||||
dbconn.removePackageFromInstalledTable(idpk)
|
||||
dbconn.addPackageToInstalledTable(idpk,etpConst['officialrepositoryname'])
|
||||
|
||||
# return back also the new possible package filename, so that we can make decisions on that
|
||||
newFileName = os.path.basename(etpDataUpdated['download'])
|
||||
|
||||
if dbconnection is None:
|
||||
dbconn.commitChanges()
|
||||
dbconn.closeDB()
|
||||
|
||||
packagename = packagename[:-5]+"~"+str(revision)+".tbz2"
|
||||
|
||||
if (accepted) and (revision != 0):
|
||||
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" has been updated to revision: "+str(revision))
|
||||
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry has been updated. Revision: ")+bold(str(revision)))
|
||||
return True, newFileName, idpk
|
||||
return True, idpk
|
||||
elif (accepted) and (revision == 0):
|
||||
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" newly created or version bumped.")
|
||||
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" entry newly created."))
|
||||
return True, newFileName, idpk
|
||||
return True, idpk
|
||||
else:
|
||||
reagentLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"generator: entry for "+str(packagename)+" kept intact, no updates needed.")
|
||||
print_info(green(" * ")+red("Package ")+bold(packagename)+red(" does not need to be updated. Current revision: ")+bold(str(revision)))
|
||||
return False, newFileName, idpk
|
||||
return False, idpk
|
||||
|
||||
|
||||
# This tool is used by Entropy after enzyme, it simply parses the content of etpConst['packagesstoredir']
|
||||
@@ -261,15 +260,26 @@ def update(options):
|
||||
etpNotCreated = 0
|
||||
for tbz2 in tbz2files:
|
||||
counter += 1
|
||||
tbz2name = tbz2.split("/")[len(tbz2.split("/"))-1]
|
||||
tbz2name = tbz2.split("/")[-1]
|
||||
print_info(" ("+str(counter)+"/"+str(totalCounter)+") Processing "+tbz2name)
|
||||
tbz2path = etpConst['packagesstoredir']+"/"+tbz2
|
||||
rc, newFileName, idpk = generator(tbz2path, dbconn, enzymeRequestBranch)
|
||||
rc, idpk = generator(tbz2path, dbconn, enzymeRequestBranch)
|
||||
if (rc):
|
||||
etpCreated += 1
|
||||
# move the file with its new name
|
||||
spawnCommand("mv "+tbz2path+" "+etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName+" -f")
|
||||
print_info(yellow(" * ")+red("Injecting database information into ")+bold(newFileName)+red(", please wait..."), back = True)
|
||||
|
||||
# add revision to package file
|
||||
downloadurl = dbconn.retrieveDownloadURL(idpk)
|
||||
packagerev = dbconn.retrieveRevision(idpk)
|
||||
downloaddir = os.path.dirname(downloadurl)
|
||||
downloadfile = os.path.basename(downloadurl)
|
||||
# remove tbz2 and add revision
|
||||
downloadfile = downloadfile[:-5]+"~"+str(packagerev)+".tbz2"
|
||||
downloadurl = downloaddir+"/"+downloadfile
|
||||
# update url
|
||||
dbconn.setDownloadURL(idpk,downloadurl)
|
||||
|
||||
shutil.move(tbz2path,etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile)
|
||||
print_info(yellow(" * ")+red("Injecting database information into ")+bold(downloadfile)+red(", please wait..."), back = True)
|
||||
|
||||
dbpath = etpConst['packagestmpdir']+"/"+str(getRandomNumber())
|
||||
while os.path.isfile(dbpath):
|
||||
@@ -283,14 +293,14 @@ def update(options):
|
||||
pkgDbconn.addPackage(data, revision = rev)
|
||||
pkgDbconn.closeDB()
|
||||
# append the database to the new file
|
||||
aggregateEdb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName, dbfile = dbpath)
|
||||
aggregateEdb(tbz2file = etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile, dbfile = dbpath)
|
||||
|
||||
digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName)
|
||||
digest = md5sum(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile)
|
||||
dbconn.setDigest(idpk,digest)
|
||||
hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+newFileName)
|
||||
hashFilePath = createHashFile(etpConst['packagessuploaddir']+"/"+enzymeRequestBranch+"/"+downloadfile)
|
||||
# remove garbage
|
||||
os.remove(dbpath)
|
||||
print_info(yellow(" * ")+red("Database injection complete for ")+newFileName)
|
||||
print_info(yellow(" * ")+red("Database injection complete for ")+downloadfile)
|
||||
|
||||
else:
|
||||
etpNotCreated += 1
|
||||
|
||||
Vendored
+1
-1
@@ -42,7 +42,7 @@ def print_help():
|
||||
print_info(blue("Tools available: "))
|
||||
print_info(" \t"+green(bold("sync"))+yellow("\t\t to sync packages, database and also do some tidy"))
|
||||
print_info(" \t\t"+red("--noask")+"\t\t\t do not make any question")
|
||||
print_info(" \t"+green(bold("tidy"))+yellow("\t\t to just remove binary packages that are not in the Entropy database anymore"))
|
||||
print_info(" \t"+green(bold("tidy"))+yellow("\t\t to just remove binary packages that are not in database and are expired"))
|
||||
print_info(" \t"+green(bold("packages"))+yellow("\t to manage binary packages"))
|
||||
print_info(" \t\t"+green("sync")+red("\t\t\t to sync the binary packages across primary mirrors"))
|
||||
print_info(" \t\t\t"+red("--ask")+"\t\t\t ask before making any changes")
|
||||
|
||||
Reference in New Issue
Block a user