From cd4fc9f4bf650fb0789434a1537fbef7dd12a684 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Sat, 1 Dec 2007 17:24:18 +0000 Subject: [PATCH] - dynamic configuration handling - some fixes here and there - added filesystem root change support git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@792 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- client/cacheTools.py | 3 +- client/equo | 3 +- client/equoTools.py | 8 +- client/rescueTools.py | 1 + libraries/clientConstants.py | 100 ++--- libraries/databaseTools.py | 30 +- libraries/entropyConstants.py | 808 +++++++++++++++++----------------- libraries/logTools.py | 1 + libraries/maskingparser.py | 340 +++++++------- libraries/portageTools.py | 21 +- libraries/serverConstants.py | 278 ++++++------ 11 files changed, 833 insertions(+), 760 deletions(-) diff --git a/client/cacheTools.py b/client/cacheTools.py index 6bbd84052..8923211fb 100644 --- a/client/cacheTools.py +++ b/client/cacheTools.py @@ -26,6 +26,7 @@ from outputTools import * import entropyTools import equoTools import confTools +from databaseTools import openRepositoryDatabase def cache(options): @@ -90,7 +91,7 @@ def generateCache(quiet = False, verbose = False, depcache = True, configcache = for reponame in etpRepositories: if (not quiet): print_info(" "+darkgreen("(")+bold("*")+darkgreen(")")+darkred(" Scanning ")+brown(etpRepositories[reponame]['description']), back = True) # get all packages keys - dbconn = equoTools.openRepositoryDatabase(reponame) + dbconn = openRepositoryDatabase(reponame) pkgdata = dbconn.listAllPackages() pkgdata = set(pkgdata) for info in pkgdata: diff --git a/client/equo b/client/equo index 2c45d8421..8f4fd3247 100644 --- a/client/equo +++ b/client/equo @@ -25,8 +25,9 @@ path.append('../libraries') path.append('../client') path.append('/usr/lib/entropy/client') path.append('/usr/lib/entropy/libraries') -from outputTools import * from entropyConstants import * +from outputTools import * + # CONSTANTS APPNAME = "Equo" diff --git a/client/equoTools.py b/client/equoTools.py index 83c4e8224..0fc290d9b 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -925,7 +925,7 @@ def matchChecksum(infoDict): return 1 return 0 -def removePackage(infoDict): +def removePackage(infoDict, quiet = False): atom = infoDict['removeatom'] content = infoDict['removecontent'] @@ -940,7 +940,7 @@ def removePackage(infoDict): # remove from database if removeidpackage != -1: - print_info(red(" ## ")+blue("Removing from database: ")+red(infoDict['removeatom'])) + if not quiet: print_info(red(" ## ")+blue("Removing from database: ")+red(infoDict['removeatom'])) removePackageFromDatabase(removeidpackage) # Handle gentoo database @@ -960,7 +960,7 @@ def removePackage(infoDict): if etpConst['collisionprotect'] > 0: if clientDbconn.isFileAvailable(file) and os.path.isfile(file): # in this way we filter out directories - print_warning(darkred(" ## ")+red("Collision found during remove of ")+file+red(" - cannot overwrite")) + if not quiet: print_warning(darkred(" ## ")+red("Collision found during remove of ")+file+red(" - cannot overwrite")) equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Collision found during remove of "+file+" - cannot overwrite") continue @@ -989,7 +989,7 @@ def removePackage(infoDict): if (protected): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"[remove] Protecting config file: "+file) - print_warning(darkred(" ## ")+red("[remove] Protecting config file: ")+file) + if not quiet: print_warning(darkred(" ## ")+red("[remove] Protecting config file: ")+file) else: try: os.lstat(file) diff --git a/client/rescueTools.py b/client/rescueTools.py index dd73dc6f0..32e13dcb6 100644 --- a/client/rescueTools.py +++ b/client/rescueTools.py @@ -52,6 +52,7 @@ def database(options): print_error(darkred(" * ")+bold("Portage")+red(" is not available.")) return 1 + print_warning(bold("ATTENTION: ")+red("The installed package database will be generated again using Gentoo one.")) print_warning(red("If you dont know what you're doing just, don't do this. Really. I'm not joking.")) rc = entropyTools.askquestion(" Understood?") diff --git a/libraries/clientConstants.py b/libraries/clientConstants.py index aee44307c..b4d196d95 100644 --- a/libraries/clientConstants.py +++ b/libraries/clientConstants.py @@ -32,54 +32,54 @@ etpInstallTriggers = {} ### structure same as above etpRemovalTriggers = {} -# equo section -if (not os.path.isfile(etpConst['equoconf'])): - print "ERROR: "+etpConst['equoconf']+" does not exist" - sys.exit(50) -else: - f = open(etpConst['equoconf'],"r") - equoconf = f.readlines() - f.close() - for line in equoconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['equoconf'] - sys.exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['equologlevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['equoconf'] - import time - time.sleep(5) +def initConfig_clientConstants(): + # equo section + if (os.path.isfile(etpConst['equoconf'])): + f = open(etpConst['equoconf'],"r") + equoconf = f.readlines() + f.close() + for line in equoconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['equoconf'] + sys.exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['equologlevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['equoconf'] + import time + time.sleep(5) + + if line.startswith("gentoo-compat|") and (len(line.split("|")) == 2): + compatopt = line.split("|")[1].strip() + if compatopt == "disable": + etpConst['gentoo-compat'] = False + else: + etpConst['gentoo-compat'] = True + + if line.startswith("collisionprotect|") and (len(line.split("|")) == 2): + collopt = line.split("|")[1].strip() + if collopt == "0" or collopt == "1" or collopt == "2": + etpConst['collisionprotect'] = int(collopt) + else: + print "WARNING: invalid collisionprotect in: "+etpConst['equoconf'] + + if line.startswith("configprotect|") and (len(line.split("|")) == 2): + configprotect = line.split("|")[1].strip() + for x in configprotect.split(): + etpConst['configprotect'].append(x) + + if line.startswith("configprotectmask|") and (len(line.split("|")) == 2): + configprotect = line.split("|")[1].strip() + for x in configprotect.split(): + etpConst['configprotectmask'].append(x) + + if line.startswith("configprotectskip|") and (len(line.split("|")) == 2): + configprotect = line.split("|")[1].strip() + for x in configprotect.split(): + etpConst['configprotectskip'].append(x) - if line.startswith("gentoo-compat|") and (len(line.split("|")) == 2): - compatopt = line.split("|")[1].strip() - if compatopt == "disable": - etpConst['gentoo-compat'] = False - else: - etpConst['gentoo-compat'] = True - - if line.startswith("collisionprotect|") and (len(line.split("|")) == 2): - collopt = line.split("|")[1].strip() - if collopt == "0" or collopt == "1" or collopt == "2": - etpConst['collisionprotect'] = int(collopt) - else: - print "WARNING: invalid collisionprotect in: "+etpConst['equoconf'] - - if line.startswith("configprotect|") and (len(line.split("|")) == 2): - configprotect = line.split("|")[1].strip() - for x in configprotect.split(): - etpConst['configprotect'].append(x) - - if line.startswith("configprotectmask|") and (len(line.split("|")) == 2): - configprotect = line.split("|")[1].strip() - for x in configprotect.split(): - etpConst['configprotectmask'].append(x) - - if line.startswith("configprotectskip|") and (len(line.split("|")) == 2): - configprotect = line.split("|")[1].strip() - for x in configprotect.split(): - etpConst['configprotectskip'].append(x) +initConfig_clientConstants() \ No newline at end of file diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 7f8ab2198..43ac2a499 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -2371,9 +2371,13 @@ class etpDatabase: return self.fetchall2set(self.cursor.fetchall()) ''' search packages that need the specified library (in neededreference table) specified by keyword ''' - def searchNeeded(self, keyword): + def searchNeeded(self, keyword, like = False): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchNeeded: called for "+keyword) - self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = "'+keyword+'" and needed.idneeded = neededreference.idneeded') + + if like: + self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library LIKE "'+keyword+'" and needed.idneeded = neededreference.idneeded') + else: + self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = "'+keyword+'" and needed.idneeded = neededreference.idneeded') return self.fetchall2set(self.cursor.fetchall()) ''' same as above but with branch support ''' @@ -2495,6 +2499,28 @@ class etpDatabase: self.storeSearchCache((keyword,sensitive,branch),'searchPackagesByName',results) return results + + def searchPackagesByCategory(self, keyword, like = False branch = None): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackagesByCategory: called for "+keyword) + + if (self.xcache): + cached = self.fetchSearchCache((keyword,branch),'searchPackagesByCategory') + if cached != None: return cached + + branchstring = '' + if branch: + branchstring = ' and branch = "'+branch+'"' + + if like: + self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category LIKE "'+keyword+'" and baseinfo.idcategory = categories.idcategory '+branchstring) + else: + self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category = "'+keyword+'" and baseinfo.idcategory = categories.idcategory '+branchstring) + + results = self.cursor.fetchall() + if (self.xcache): + self.storeSearchCache((keyword,branch),'searchPackagesByCategory',results) + return results + def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackagesByNameAndCategory: called for name: "+name+" and category: "+category) diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index dacef71fe..d351f8860 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -23,6 +23,7 @@ import os import random from sys import exit +import maskingparser # Specifications of the content of etpData @@ -282,168 +283,33 @@ CREATE TABLE neededreference ( """ -# Entropy directories specifications -# THIS IS THE KEY PART OF ENTROPY BINARY PACKAGES MANAGEMENT -# DO NOT EDIT THIS UNLESS YOU KNOW WHAT YOU'RE DOING !! -# the ARCHs that we support -ETP_ARCHS = ["x86", "amd64"] # maybe ppc someday -ETP_API = "1" # ETP_ARCH_CONST setup if os.uname()[4] == "x86_64": ETP_ARCH_CONST = "amd64" else: ETP_ARCH_CONST = "x86" +# the ARCHs that we support +etpSys = { + 'archs': ["x86", "amd64"], + 'api': '1', + 'arch': ETP_ARCH_CONST, + 'rootdir': "" +} -ETP_DIR = "/var/lib/entropy" -ETP_TMPDIR = "/tmp" -ETP_RANDOM = str(random.random())[2:7] -ETP_TMPFILE = "/.random-"+ETP_RANDOM+".tmp" -ETP_REPODIR = "/packages/"+ETP_ARCH_CONST -ETP_PORTDIR = "/usr/portage" -ETP_DISTFILESDIR = "/distfiles" -ETP_DBDIR = "/database/"+ETP_ARCH_CONST -ETP_DBFILE = "packages.db" -ETP_DBCLIENTFILE = "equo.db" -ETP_CLIENT_REPO_DIR = "/client" -ETP_UPLOADDIR = "/upload/"+ETP_ARCH_CONST -ETP_STOREDIR = "/store/"+ETP_ARCH_CONST -ETP_TRIGGERSDIR = "/triggers/"+ETP_ARCH_CONST -ETP_SMARTAPPSDIR = "/smartapps/"+ETP_ARCH_CONST -ETP_SMARTPACKAGESDIR = "/smartpackages/"+ETP_ARCH_CONST -ETP_XMLDIR = "/xml/" -ETP_CONF_DIR = "/etc/entropy" -ETP_ROOT_DIR = "/" -ETP_LOG_DIR = ETP_DIR+"/"+"logs" -ETP_SYSLOG_DIR = "/var/log/entropy/" -ETP_VAR_DIR = "/var/tmp/entropy" +# static logging stuff ETP_LOGLEVEL_NORMAL = 1 ETP_LOGLEVEL_VERBOSE = 2 ETP_LOGPRI_INFO = "[ INFO ]" ETP_LOGPRI_WARNING = "[ WARNING ]" ETP_LOGPRI_ERROR = "[ ERROR ]" -etpConst = { - 'packagestmpdir': ETP_DIR+ETP_TMPDIR, # etpConst['packagestmpdir'] --> temp directory - 'packagestmpfile': ETP_DIR+ETP_TMPDIR+ETP_TMPFILE, # etpConst['packagestmpfile'] --> general purpose tmp file - 'packagesbindir': ETP_DIR+ETP_REPODIR, # etpConst['packagesbindir'] --> repository where the packages will be stored - # by the clients: to query if a package has been already downloaded - # by the servers or rsync mirrors: to store already uploaded packages to the main rsync server - 'smartappsdir': ETP_DIR+ETP_SMARTAPPSDIR, # etpConst['smartappsdir'] location where smart apps files are places - 'smartpackagesdir': ETP_DIR+ETP_SMARTPACKAGESDIR, # etpConst['smartpackagesdir'] location where smart packages files are places - 'triggersdir': ETP_DIR+ETP_TRIGGERSDIR, # etpConst['triggersdir'] location where external triggers are placed - 'packagesstoredir': ETP_DIR+ETP_STOREDIR, # etpConst['packagesstoredir'] --> directory where .tbz2 files are stored waiting for being processed by reagent - 'packagessuploaddir': ETP_DIR+ETP_UPLOADDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror - 'portagetreedir': ETP_PORTDIR, # directory where is stored our local portage tree - 'distfilesdir': ETP_PORTDIR+ETP_DISTFILESDIR, # directory where our sources are downloaded - 'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration - 'entropyconf': ETP_CONF_DIR+"/entropy.conf", # entropy.conf file - 'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file - 'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file - 'serverconf': ETP_CONF_DIR+"/server.conf", # server.conf file (generic server side settings) - 'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file - 'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file - 'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file - 'remoteconf': ETP_CONF_DIR+"/remote.conf", # remote.conf file - 'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # spmbackend.conf file - 'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file - 'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf) - 'activatordownloaduris': [], # list of URIs that activator can use to fetch data - 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. - 'etpurirelativepath': "database/"+ETP_ARCH_CONST+"/", # database relative path - - 'entropyworkdir': ETP_DIR, # Entropy workdir - 'entropyunpackdir': ETP_VAR_DIR, # Entropy unpack directory - - 'etpdatabaserevisionfile': ETP_DBFILE+".revision", # the local/remote database revision file - 'etpdatabasehashfile': ETP_DBFILE+".md5", # its checksum - 'etpdatabaselockfile': ETP_DBFILE+".lock", # the remote database lock file - 'etpdatabasedownloadlockfile': ETP_DBFILE+".download.lock", # the remote database download lock file - 'etpdatabasetaintfile': ETP_DBFILE+".tainted", # when this file exists, the database is not synced anymore with the online one - 'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db" - 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) - 'etpdatabasefilebzip2': ETP_DBFILE+".bz2", # Entropy sqlite database file (bzipped2) - 'etpdatabasefileformat': "bz2", # Entropy default compressed database format - 'etpdatabasesupportedcformats': ["bz2","gz"], # Entropy compressed databases format support - 'etpdatabasecompressclasses': { - "bz2": ("bz2.BZ2File","unpackBzip2","etpdatabasefilebzip2",), - "gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",) - }, - 'rss-feed': True, # enable/disable packages RSS feed feature - 'rss-name': "packages.rss", # default name of the RSS feed - 'rss-base-url': "http://packages.sabayonlinux.org/", # default URL to the entropy web interface (overridden in reagent.conf) - 'rss-website-url': "http://www.sabayonlinux.org/", # default URL to the Operating System website (overridden in reagent.conf) - 'rss-dump-name': "rss_database_actions", # xml file where will be dumped etpRSSMessages dictionary - 'rss-max-entries': 10000, # maximum rss entries - 'rss-managing-editor': "lxnay@sabayonlinux.org", # updates submitter - - 'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file - 'packagesexpirationfileext': ".expired", # Extension of the file that "contains" expiration mtime - 'packagesexpirationdays': 15, # number of days after a package will be removed from mirrors - 'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools - 'proxy': {}, # proxy configuration information, used system wide - - 'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info) - 'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info) - 'remoteloglevel': 1, # Remote handlers (/handlers) log level (default: 1 - see remote.conf for more info) - 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) - 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) - 'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info) - 'equologlevel': 1, # # Equo log level (default: 1 - see equo.conf for more info) - 'spmbackendloglevel': 1, # # Source Package Manager backend log level (default: 1 - see entropy.conf for more info) - 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their stuff - - 'syslogdir': ETP_SYSLOG_DIR, # Entropy system tools log directory - 'mirrorslogfile': ETP_SYSLOG_DIR+"/mirrors.log", # Mirrors operations log file - 'remotelogfile': ETP_SYSLOG_DIR+"/remote.log", # Mirrors operations log file - 'spmbackendlogfile': ETP_SYSLOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file - 'databaselogfile': ETP_SYSLOG_DIR+"/database.log", # Database operations log file - 'reagentlogfile': ETP_SYSLOG_DIR+"/reagent.log", # Reagent operations log file - 'activatorlogfile': ETP_SYSLOG_DIR+"/activator.log", # Activator operations log file - 'entropylogfile': ETP_SYSLOG_DIR+"/entropy.log", # Activator operations log file - 'equologfile': ETP_SYSLOG_DIR+"/equo.log", # Activator operations log file - - 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file FIXME: remove this? - 'etpdatabasedir': ETP_DIR+ETP_DBDIR, - 'etpdatabasefilepath': ETP_DIR+ETP_DBDIR+"/"+ETP_DBFILE, - 'etpdatabaseclientdir': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR, - 'etpdatabaseclientfilepath': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR+"/"+ETP_DBCLIENTFILE, # path to equo.db - client side database file - 'dbnamerepoprefix': "repo_", # prefix of the name of self.dbname in etpDatabase class for the repositories - - 'etpapi': ETP_API, # Entropy database API revision - 'currentarch': ETP_ARCH_CONST, # contains the current running architecture - 'supportedarchs': ETP_ARCHS, # Entropy supported Archs - - 'branches': [], # available branches, this only exists for the server part, these settings will be overridden by server.conf ones - 'branch': "3.5", # default choosen branch (overridden by setting in repositories.conf) - 'keywords': set([ETP_ARCH_CONST,"~"+ETP_ARCH_CONST]), # default allowed package keywords - 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) - 'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem - 'filesystemdirsmask': [ - '/var/cache','/var/db','/var/empty','/var/lib/portage','/var/lib/entropy','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage', - '/var/lib/scrollkeeper', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', - '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' - ], - 'officialrepositoryname': "sabayonlinux.org", # our official repository name - 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes) - 'pidfile': "/var/run/equo.pid", - 'applicationlock': False, - 'collisionprotect': 1, # collision protection option, read equo.conf for more info - 'configprotect': [], # list of user specified CONFIG_PROTECT directories (see Gentoo manual to understand the meaining of this parameter) - 'configprotectmask': [], # list of user specified CONFIG_PROTECT_MASK directories - 'configprotectskip': [], # list of user specified configuration files that should be ignored and kept as they are - 'dbconfigprotect': [], # installed database CONFIG_PROTECT directories - 'dbconfigprotectmask': [], # installed database CONFIG_PROTECT_MASK directories - 'configprotectcounter': 0, # this will be used to show the number of updated files at the end of the processes - 'entropyversion': "1.0", # default Entropy release version - 'systemname': "Sabayon Linux", # default system name (overidden by entropy.conf settings) - 'product': "standard", # Product identificator (standard, professional...) - 'errorstatus': ETP_CONF_DIR+"/code", - - 'dumpstoragedir': ETP_DIR+ETP_XMLDIR, # data storage directory, useful to speed up equo across multiple issued commands - - # packages keywords/mask/unmask settings - 'packagemasking': {}, # package masking information dictionary filled by maskingparser.py - +### file transfer settings +etpFileTransfer = { + 'datatransfer': 0, + 'oldgather': 0, + 'gather': 0, + 'elapsed': 0.0, + 'transferpollingtime': float(1)/4 # 0.25secs = 4Hz } # disk caching dictionary @@ -488,263 +354,25 @@ etpRSSMessages = { 'commitmessage': "" # commit message from the guy who is going to submit a repository update } -### Application disk cache +# Handlers used by entropy to run and retrieve data remotely, using php helpers +etpHandlers = {} + +# CACHING dictionaries dbCacheStore = {} atomMatchCache = {} atomClientMatchCache = {} generateDependsTreeCache = {} idpackageValidatorCache = {} -def const_resetCache(): - dbCacheStore.clear() - atomMatchCache.clear() - atomClientMatchCache.clear() - generateDependsTreeCache.clear() - idpackageValidatorCache.clear() - -# handle Entropy Version -ETP_REVISION_FILE = "../libraries/revision" -if os.path.isfile(ETP_REVISION_FILE): - f = open(ETP_REVISION_FILE,"r") - myrev = f.readline().strip() - etpConst['entropyversion'] = myrev - -# handle pid file -piddir = os.path.dirname(etpConst['pidfile']) -if not os.path.exists(piddir): - if os.getuid() == 0: - os.makedirs(piddir) - else: - print "you need to run this as root at least once." - exit(100) -# PID creation -pid = os.getpid() -if os.path.exists(etpConst['pidfile']): - f = open(etpConst['pidfile'],"r") - foundPid = f.readline().strip() - f.close() - if foundPid != str(pid): - # is foundPid still running ? - import commands - pids = commands.getoutput("pidof python").split("\n")[0].split() - try: - pids.index(foundPid) - etpConst['applicationlock'] = True - except: - # if root, write new pid - if os.getuid() == 0: - f = open(etpConst['pidfile'],"w") - f.write(str(pid)) - f.flush() - f.close() - pass -else: - if os.getuid() == 0: - f = open(etpConst['pidfile'],"w") - f.write(str(pid)) - f.flush() - f.close() - else: - print "you need to run this as root at least once." - exit(100) - -### file transfer settings -etpFileTransfer = { - 'datatransfer': 0, - 'oldgather': 0, - 'gather': 0, - 'elapsed': 0.0, - 'transferpollingtime': float(1)/4 # 0.25secs = 4Hz -} - -# Create paths -if not os.path.isdir(etpConst['entropyworkdir']): - if os.getuid() == 0: - import re - for x in etpConst: - if (type(etpConst[x]) is str): - - if (not etpConst[x]) or (etpConst[x].endswith(".conf")) or (not etpConst[x].startswith("/")) or (etpConst[x].endswith(".cfg")) or (etpConst[x].endswith(".tmp")) or (etpConst[x].find(".db") != -1) or (etpConst[x].find(".log") != -1): - continue - - if etpConst[x].find("%ARCH%") != -1: - for i in ETP_ARCHS: - try: - mdir = re.subn("%ARCH%",i, etpConst[x])[0] - os.makedirs(mdir,0755) - os.chown(mdir,0,0) - except OSError: - pass - else: - try: - os.makedirs(etpConst[x],0755) - os.chown(etpConst[x],0,0) - except OSError: - pass - else: - print "you need to run this as root at least once." - exit(100) - - -# entropy section -if os.path.isfile(etpConst['entropyconf']): - f = open(etpConst['entropyconf'],"r") - entropyconf = f.readlines() - f.close() - for line in entropyconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['entropyconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['entropyloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['entropyconf'] - import time - time.sleep(5) - - elif line.startswith("ftp-proxy|") and (len(line.split("|")) == 2): - ftpproxy = line.split("|")[1].strip() - for x in ftpproxy.split(): - etpConst['proxy']['ftp'] = ftpproxy - elif line.startswith("http-proxy|") and (len(line.split("|")) == 2): - httpproxy = line.split("|")[1].strip() - for x in httpproxy.split(): - etpConst['proxy']['http'] = httpproxy - elif line.startswith("system-name|") and (len(line.split("|")) == 2): - etpConst['systemname'] = line.split("|")[1].strip() - - # Client packages/database repositories etpRepositories = {} etpRepositoriesOrder = set() -ordercount = 0 -if os.path.isfile(etpConst['repositoriesconf']): - f = open(etpConst['repositoriesconf'],"r") - repositoriesconf = f.readlines() - f.close() - - # setup product first - for line in repositoriesconf: - if (line.strip().find("product|") != -1) and (not line.strip().startswith("#")) and (len(line.strip().split("|")) == 2): - etpConst['product'] = line.strip().split("|")[1] - - for line in repositoriesconf: - line = line.strip() - # populate etpRepositories - if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5): - reponame = line.split("|")[1] - repodesc = line.split("|")[2] - repopackages = line.split("|")[3] - repodatabase = line.split("|")[4] - dbformat = etpConst['etpdatabasefileformat'] - dbformatcolon = repodatabase.rfind("#") - if dbformatcolon != -1: - if dbformat in etpConst['etpdatabasesupportedcformats']: - try: - dbformat = repodatabase[dbformatcolon+1:] - except: - pass - repodatabase = repodatabase[:dbformatcolon] - if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")): - etpRepositories[reponame] = {} - ordercount += 1 - etpRepositoriesOrder.add((ordercount,reponame)) - etpRepositories[reponame]['description'] = repodesc - etpRepositories[reponame]['packages'] = [] - for x in repopackages.split(): - etpRepositories[reponame]['packages'].append(x+"/"+etpConst['product']) - etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch'] - etpRepositories[reponame]['dbcformat'] = dbformat - etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch'] - # initialize CONFIG_PROTECT - will be filled the first time the db will be opened - etpRepositories[reponame]['configprotect'] = None - etpRepositories[reponame]['configprotectmask'] = None - elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): - branch = line.split("|")[1] - etpConst['branch'] = branch - if not os.path.isdir(etpConst['packagesbindir']+"/"+branch): - if os.getuid() == 0: - os.makedirs(etpConst['packagesbindir']+"/"+branch) - else: - print "ERROR: please run this as root at least once or create: "+str(etpConst['packagesbindir']+"/"+branch) - exit(49) - -# align etpConst['binaryurirelativepath'] and etpConst['etpurirelativepath'] with etpConst['product'] -etpConst['binaryurirelativepath'] = etpConst['product']+"/"+etpConst['binaryurirelativepath'] -etpConst['etpurirelativepath'] = etpConst['product']+"/"+etpConst['etpurirelativepath'] - -# check for packages and upload directories -if os.getuid() == 0: - for x in etpConst['branches']: - if not os.path.isdir(etpConst['packagesbindir']+"/"+x): - os.makedirs(etpConst['packagesbindir']+"/"+x) - if not os.path.isdir(etpConst['packagessuploaddir']+"/"+x): - os.makedirs(etpConst['packagessuploaddir']+"/"+x) - -# database section -if os.path.isfile(etpConst['databaseconf']): - f = open(etpConst['databaseconf'],"r") - databaseconf = f.readlines() - f.close() - for line in databaseconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['databaseconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['databaseloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['databaseconf'] - import time - time.sleep(5) - -# Handlers used by entropy to run and retrieve data remotely, using php helpers -etpHandlers = { - 'md5sum': "md5sum.php?arch="+ETP_ARCH_CONST+"&package=", # md5sum handler - 'errorsend': "http://svn.sabayonlinux.org/entropy/"+etpConst['product']+"/handlers/error_report.php?arch="+ETP_ARCH_CONST+"&stacktrace=", -} # remote section etpRemoteSupport = {} etpRemoteFailures = {} # dict of excluded mirrors due to failures, it contains mirror name and failure count | > 5 == ignore mirror -if (os.path.isfile(etpConst['remoteconf'])): - f = open(etpConst['remoteconf'],"r") - remoteconf = f.readlines() - f.close() - for line in remoteconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "WARNING: invalid loglevel in: "+etpConst['remoteconf'] - if (loglevel > -1) and (loglevel < 3): - etpConst['remoteloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['remoteconf'] - if line.startswith("handler|") and (len(line.split("|")) > 2): - servername = line.split("|")[1].strip() - url = line.split("|")[2].strip() - if not url.endswith("/"): - url = url+"/" - url += etpConst['product']+"/handlers/" - etpRemoteSupport[servername] = url - -# generate masking dictionary -# MUST BE INSTANTIANTED BEFORE ANY DB CONNECTION BEGINS -import maskingparser -etpConst['packagemasking'] = maskingparser.parse() -# merge universal keywords -for x in etpConst['packagemasking']['keywords']['universal']: - etpConst['keywords'].add(x) +etpConst = {} # Portage /var/db///* # you never know if gentoo devs change these things @@ -771,3 +399,397 @@ dbCONTENTS = "CONTENTS" dbCOUNTER = "COUNTER" edbCOUNTER = "/var/cache/edb/counter" + +### Application disk cache +def const_resetCache(): + dbCacheStore.clear() + atomMatchCache.clear() + atomClientMatchCache.clear() + generateDependsTreeCache.clear() + idpackageValidatorCache.clear() + + +# =============================================================================================== +# BEGINNING OF DYNAMIC SECTION +# =============================================================================================== + +def initConfig_entropyConstants(rootdir): + + ETP_DIR = rootdir+"/var/lib/entropy" + ETP_TMPDIR = rootdir+"/tmp" + ETP_RANDOM = str(random.random())[2:7] + ETP_TMPFILE = "/.random-"+ETP_RANDOM+".tmp" + ETP_REPODIR = "/packages/"+ETP_ARCH_CONST + ETP_PORTDIR = rootdir+"/usr/portage" + ETP_DISTFILESDIR = "/distfiles" + ETP_DBDIR = "/database/"+ETP_ARCH_CONST + ETP_DBFILE = "packages.db" + ETP_DBCLIENTFILE = "equo.db" + ETP_CLIENT_REPO_DIR = "/client" + ETP_UPLOADDIR = "/upload/"+ETP_ARCH_CONST + ETP_STOREDIR = "/store/"+ETP_ARCH_CONST + ETP_TRIGGERSDIR = "/triggers/"+ETP_ARCH_CONST + ETP_SMARTAPPSDIR = "/smartapps/"+ETP_ARCH_CONST + ETP_SMARTPACKAGESDIR = "/smartpackages/"+ETP_ARCH_CONST + ETP_XMLDIR = "/xml/" + ETP_LOG_DIR = ETP_DIR+"/"+"logs" + ETP_CONF_DIR = rootdir+"/etc/entropy" + ETP_SYSLOG_DIR = rootdir+"/var/log/entropy/" + ETP_VAR_DIR = rootdir+"/var/tmp/entropy" + + + const_resetCache() + etpConst.clear() + myConst = { + 'packagestmpdir': ETP_DIR+ETP_TMPDIR, # etpConst['packagestmpdir'] --> temp directory + 'packagestmpfile': ETP_DIR+ETP_TMPDIR+ETP_TMPFILE, # etpConst['packagestmpfile'] --> general purpose tmp file + 'packagesbindir': ETP_DIR+ETP_REPODIR, # etpConst['packagesbindir'] --> repository where the packages will be stored + # by the clients: to query if a package has been already downloaded + # by the servers or rsync mirrors: to store already uploaded packages to the main rsync server + 'smartappsdir': ETP_DIR+ETP_SMARTAPPSDIR, # etpConst['smartappsdir'] location where smart apps files are places + 'smartpackagesdir': ETP_DIR+ETP_SMARTPACKAGESDIR, # etpConst['smartpackagesdir'] location where smart packages files are places + 'triggersdir': ETP_DIR+ETP_TRIGGERSDIR, # etpConst['triggersdir'] location where external triggers are placed + 'packagesstoredir': ETP_DIR+ETP_STOREDIR, # etpConst['packagesstoredir'] --> directory where .tbz2 files are stored waiting for being processed by reagent + 'packagessuploaddir': ETP_DIR+ETP_UPLOADDIR, # etpConst['packagessuploaddir'] --> directory where .tbz2 files are stored waiting for being uploaded to our main mirror + 'portagetreedir': ETP_PORTDIR, # directory where is stored our local portage tree + 'distfilesdir': ETP_PORTDIR+ETP_DISTFILESDIR, # directory where our sources are downloaded + 'confdir': ETP_CONF_DIR, # directory where entropy stores its configuration + 'entropyconf': ETP_CONF_DIR+"/entropy.conf", # entropy.conf file + 'repositoriesconf': ETP_CONF_DIR+"/repositories.conf", # repositories.conf file + 'activatorconf': ETP_CONF_DIR+"/activator.conf", # activator.conf file + 'serverconf': ETP_CONF_DIR+"/server.conf", # server.conf file (generic server side settings) + 'reagentconf': ETP_CONF_DIR+"/reagent.conf", # reagent.conf file + 'databaseconf': ETP_CONF_DIR+"/database.conf", # database.conf file + 'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file + 'remoteconf': ETP_CONF_DIR+"/remote.conf", # remote.conf file + 'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # spmbackend.conf file + 'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file + 'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf) + 'activatordownloaduris': [], # list of URIs that activator can use to fetch data + 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. + 'etpurirelativepath': "database/"+ETP_ARCH_CONST+"/", # database relative path + + 'entropyworkdir': ETP_DIR, # Entropy workdir + 'entropyunpackdir': ETP_VAR_DIR, # Entropy unpack directory + + 'etpdatabaserevisionfile': ETP_DBFILE+".revision", # the local/remote database revision file + 'etpdatabasehashfile': ETP_DBFILE+".md5", # its checksum + 'etpdatabaselockfile': ETP_DBFILE+".lock", # the remote database lock file + 'etpdatabasedownloadlockfile': ETP_DBFILE+".download.lock", # the remote database download lock file + 'etpdatabasetaintfile': ETP_DBFILE+".tainted", # when this file exists, the database is not synced anymore with the online one + 'etpdatabasefile': ETP_DBFILE, # Entropy sqlite database file ETP_DIR+ETP_DBDIR+"/packages.db" + 'etpdatabasefilegzip': ETP_DBFILE+".gz", # Entropy sqlite database file (gzipped) + 'etpdatabasefilebzip2': ETP_DBFILE+".bz2", # Entropy sqlite database file (bzipped2) + 'etpdatabasefileformat': "bz2", # Entropy default compressed database format + 'etpdatabasesupportedcformats': ["bz2","gz"], # Entropy compressed databases format support + 'etpdatabasecompressclasses': { + "bz2": ("bz2.BZ2File","unpackBzip2","etpdatabasefilebzip2",), + "gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",) + }, + 'rss-feed': True, # enable/disable packages RSS feed feature + 'rss-name': "packages.rss", # default name of the RSS feed + 'rss-base-url': "http://packages.sabayonlinux.org/", # default URL to the entropy web interface (overridden in reagent.conf) + 'rss-website-url': "http://www.sabayonlinux.org/", # default URL to the Operating System website (overridden in reagent.conf) + 'rss-dump-name': "rss_database_actions", # xml file where will be dumped etpRSSMessages dictionary + 'rss-max-entries': 10000, # maximum rss entries + 'rss-managing-editor': "lxnay@sabayonlinux.org", # updates submitter + + 'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file + 'packagesexpirationfileext': ".expired", # Extension of the file that "contains" expiration mtime + 'packagesexpirationdays': 15, # number of days after a package will be removed from mirrors + 'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools + 'proxy': {}, # proxy configuration information, used system wide + + 'databaseloglevel': 1, # Database log level (default: 1 - see database.conf for more info) + 'mirrorsloglevel': 1, # Mirrors log level (default: 1 - see mirrors.conf for more info) + 'remoteloglevel': 1, # Remote handlers (/handlers) log level (default: 1 - see remote.conf for more info) + 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) + 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) + 'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info) + 'equologlevel': 1, # # Equo log level (default: 1 - see equo.conf for more info) + 'spmbackendloglevel': 1, # # Source Package Manager backend log level (default: 1 - see entropy.conf for more info) + 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their stuff + + 'syslogdir': ETP_SYSLOG_DIR, # Entropy system tools log directory + 'mirrorslogfile': ETP_SYSLOG_DIR+"/mirrors.log", # Mirrors operations log file + 'remotelogfile': ETP_SYSLOG_DIR+"/remote.log", # Mirrors operations log file + 'spmbackendlogfile': ETP_SYSLOG_DIR+"/spmbackend.log", # Source Package Manager backend configuration log file + 'databaselogfile': ETP_SYSLOG_DIR+"/database.log", # Database operations log file + 'reagentlogfile': ETP_SYSLOG_DIR+"/reagent.log", # Reagent operations log file + 'activatorlogfile': ETP_SYSLOG_DIR+"/activator.log", # Activator operations log file + 'entropylogfile': ETP_SYSLOG_DIR+"/entropy.log", # Activator operations log file + 'equologfile': ETP_SYSLOG_DIR+"/equo.log", # Activator operations log file + + 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file FIXME: remove this? + 'etpdatabasedir': ETP_DIR+ETP_DBDIR, + 'etpdatabasefilepath': ETP_DIR+ETP_DBDIR+"/"+ETP_DBFILE, + 'etpdatabaseclientdir': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR, + 'etpdatabaseclientfilepath': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR+"/"+ETP_DBCLIENTFILE, # path to equo.db - client side database file + 'dbnamerepoprefix': "repo_", # prefix of the name of self.dbname in etpDatabase class for the repositories + + 'etpapi': etpSys['api'], # Entropy database API revision + 'currentarch': etpSys['arch'], # contains the current running architecture + 'supportedarchs': etpSys['archs'], # Entropy supported Archs + + 'branches': [], # available branches, this only exists for the server part, these settings will be overridden by server.conf ones + 'branch': "3.5", # default choosen branch (overridden by setting in repositories.conf) + 'keywords': set([etpSys['arch'],"~"+etpSys['arch']]), # default allowed package keywords + 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) + 'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem + 'filesystemdirsmask': [ + '/var/cache','/var/db','/var/empty','/var/lib/portage','/var/lib/entropy','/var/log','/var/mail','/var/tmp','/var/www', '/usr/portage', + '/var/lib/scrollkeeper', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', + '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' + ], + 'officialrepositoryname': "sabayonlinux.org", # our official repository name + 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes) + 'pidfile': "/var/run/equo.pid", + 'applicationlock': False, + 'collisionprotect': 1, # collision protection option, read equo.conf for more info + 'configprotect': [], # list of user specified CONFIG_PROTECT directories (see Gentoo manual to understand the meaining of this parameter) + 'configprotectmask': [], # list of user specified CONFIG_PROTECT_MASK directories + 'configprotectskip': [], # list of user specified configuration files that should be ignored and kept as they are + 'dbconfigprotect': [], # installed database CONFIG_PROTECT directories + 'dbconfigprotectmask': [], # installed database CONFIG_PROTECT_MASK directories + 'configprotectcounter': 0, # this will be used to show the number of updated files at the end of the processes + 'entropyversion': "1.0", # default Entropy release version + 'systemname': "Sabayon Linux", # default system name (overidden by entropy.conf settings) + 'product': "standard", # Product identificator (standard, professional...) + 'errorstatus': ETP_CONF_DIR+"/code", + 'systemroot': rootdir, # default system root + + 'dumpstoragedir': ETP_DIR+ETP_XMLDIR, # data storage directory, useful to speed up equo across multiple issued commands + + # packages keywords/mask/unmask settings + 'packagemasking': {}, # package masking information dictionary filled by maskingparser.py + + } + for item in myConst: + if (type(myConst[item]) is set) or (type(myConst[item]) is dict): + etpConst[item] = myConst[item].copy() + else: + etpConst[item] = myConst[item] + del myConst + + + # handle Entropy Version + ETP_REVISION_FILE = "../libraries/revision" + if os.path.isfile(ETP_REVISION_FILE): + f = open(ETP_REVISION_FILE,"r") + myrev = f.readline().strip() + etpConst['entropyversion'] = myrev + + # handle pid file + piddir = os.path.dirname(etpConst['pidfile']) + if not os.path.exists(piddir): + if os.getuid() == 0: + os.makedirs(piddir) + else: + print "you need to run this as root at least once." + exit(100) + # PID creation + pid = os.getpid() + if os.path.exists(etpConst['pidfile']): + f = open(etpConst['pidfile'],"r") + foundPid = f.readline().strip() + f.close() + if foundPid != str(pid): + # is foundPid still running ? + import commands + pids = commands.getoutput("pidof python").split("\n")[0].split() + try: + pids.index(foundPid) + etpConst['applicationlock'] = True + except: + # if root, write new pid + if os.getuid() == 0: + f = open(etpConst['pidfile'],"w") + f.write(str(pid)) + f.flush() + f.close() + pass + else: + if os.getuid() == 0: + f = open(etpConst['pidfile'],"w") + f.write(str(pid)) + f.flush() + f.close() + else: + print "you need to run this as root at least once." + exit(100) + + # Create paths + if not os.path.isdir(etpConst['entropyworkdir']): + if os.getuid() == 0: + for x in etpConst: + if (type(etpConst[x]) is str): + + if (not etpConst[x]) or (etpConst[x].endswith(".conf")) or (not etpConst[x].startswith("/")) or (etpConst[x].endswith(".cfg")) or (etpConst[x].endswith(".tmp")) or (etpConst[x].find(".db") != -1) or (etpConst[x].find(".log") != -1): + continue + + try: + os.makedirs(etpConst[x],0755) + os.chown(etpConst[x],0,0) + except OSError: + pass + else: + print "you need to run this as root at least once." + exit(100) + + + # entropy section + if os.path.isfile(etpConst['entropyconf']): + f = open(etpConst['entropyconf'],"r") + entropyconf = f.readlines() + f.close() + for line in entropyconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['entropyconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['entropyloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['entropyconf'] + + elif line.startswith("ftp-proxy|") and (len(line.split("|")) == 2): + ftpproxy = line.split("|")[1].strip() + for x in ftpproxy.split(): + etpConst['proxy']['ftp'] = ftpproxy + elif line.startswith("http-proxy|") and (len(line.split("|")) == 2): + httpproxy = line.split("|")[1].strip() + for x in httpproxy.split(): + etpConst['proxy']['http'] = httpproxy + elif line.startswith("system-name|") and (len(line.split("|")) == 2): + etpConst['systemname'] = line.split("|")[1].strip() + + + etpHandlers = { + 'md5sum': "md5sum.php?arch="+etpConst['currentarch']+"&package=", # md5sum handler + 'errorsend': "http://svn.sabayonlinux.org/entropy/"+etpConst['product']+"/handlers/error_report.php?arch="+etpConst['currentarch']+"&stacktrace=", + } + + etpRepositories.clear() + etpRepositoriesOrder.clear() + ordercount = 0 + if os.path.isfile(etpConst['repositoriesconf']): + f = open(etpConst['repositoriesconf'],"r") + repositoriesconf = f.readlines() + f.close() + + # setup product first + for line in repositoriesconf: + if (line.strip().find("product|") != -1) and (not line.strip().startswith("#")) and (len(line.strip().split("|")) == 2): + etpConst['product'] = line.strip().split("|")[1] + + for line in repositoriesconf: + line = line.strip() + # populate etpRepositories + if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5): + reponame = line.split("|")[1] + repodesc = line.split("|")[2] + repopackages = line.split("|")[3] + repodatabase = line.split("|")[4] + dbformat = etpConst['etpdatabasefileformat'] + dbformatcolon = repodatabase.rfind("#") + if dbformatcolon != -1: + if dbformat in etpConst['etpdatabasesupportedcformats']: + try: + dbformat = repodatabase[dbformatcolon+1:] + except: + pass + repodatabase = repodatabase[:dbformatcolon] + if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")): + etpRepositories[reponame] = {} + ordercount += 1 + etpRepositoriesOrder.add((ordercount,reponame)) + etpRepositories[reponame]['description'] = repodesc + etpRepositories[reponame]['packages'] = [] + for x in repopackages.split(): + etpRepositories[reponame]['packages'].append(x+"/"+etpConst['product']) + etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['product']+"/"+etpConst['currentarch'] + etpRepositories[reponame]['dbcformat'] = dbformat + etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['product']+"/database/"+etpConst['currentarch'] + # initialize CONFIG_PROTECT - will be filled the first time the db will be opened + etpRepositories[reponame]['configprotect'] = None + etpRepositories[reponame]['configprotectmask'] = None + elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): + branch = line.split("|")[1] + etpConst['branch'] = branch + if not os.path.isdir(etpConst['packagesbindir']+"/"+branch): + if os.getuid() == 0: + os.makedirs(etpConst['packagesbindir']+"/"+branch) + else: + print "ERROR: please run this as root at least once or create: "+str(etpConst['packagesbindir']+"/"+branch) + exit(49) + + # align etpConst['binaryurirelativepath'] and etpConst['etpurirelativepath'] with etpConst['product'] + etpConst['binaryurirelativepath'] = etpConst['product']+"/"+etpConst['binaryurirelativepath'] + etpConst['etpurirelativepath'] = etpConst['product']+"/"+etpConst['etpurirelativepath'] + + # check for packages and upload directories + if os.getuid() == 0: + for x in etpConst['branches']: + if not os.path.isdir(etpConst['packagesbindir']+"/"+x): + os.makedirs(etpConst['packagesbindir']+"/"+x) + if not os.path.isdir(etpConst['packagessuploaddir']+"/"+x): + os.makedirs(etpConst['packagessuploaddir']+"/"+x) + + # database section + if os.path.isfile(etpConst['databaseconf']): + f = open(etpConst['databaseconf'],"r") + databaseconf = f.readlines() + f.close() + for line in databaseconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['databaseconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['databaseloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['databaseconf'] + + + etpRemoteSupport.clear() + etpRemoteFailures.clear() + if (os.path.isfile(etpConst['remoteconf'])): + f = open(etpConst['remoteconf'],"r") + remoteconf = f.readlines() + f.close() + for line in remoteconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "WARNING: invalid loglevel in: "+etpConst['remoteconf'] + if (loglevel > -1) and (loglevel < 3): + etpConst['remoteloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['remoteconf'] + + if line.startswith("handler|") and (len(line.split("|")) > 2): + servername = line.split("|")[1].strip() + url = line.split("|")[2].strip() + if not url.endswith("/"): + url = url+"/" + url += etpConst['product']+"/handlers/" + etpRemoteSupport[servername] = url + + # generate masking dictionary + myparser = maskingparser.parser(etpConst,etpCache) + etpConst['packagemasking'] = myparser.parse() + # merge universal keywords + for x in etpConst['packagemasking']['keywords']['universal']: + etpConst['keywords'].add(x) + + +# load config +initConfig_entropyConstants(etpSys['rootdir']) diff --git a/libraries/logTools.py b/libraries/logTools.py index 11574bb19..89090eb46 100644 --- a/libraries/logTools.py +++ b/libraries/logTools.py @@ -27,6 +27,7 @@ from sys import stderr from time import strftime +from entropyConstants import * class LogFile: def __init__ (self, level = 0, filename = None, header = "[LOG]"): diff --git a/libraries/maskingparser.py b/libraries/maskingparser.py index 400d6b939..2526ac3e8 100644 --- a/libraries/maskingparser.py +++ b/libraries/maskingparser.py @@ -19,183 +19,191 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ''' - -from entropyConstants import ETP_CONF_DIR, etpConst, etpCache, os - -etpMaskFiles = { - 'keywords': ETP_CONF_DIR+"/packages/package.keywords", # keywording configuration files - 'unmask': ETP_CONF_DIR+"/packages/package.unmask", # unmasking configuration files - 'mask': ETP_CONF_DIR+"/packages/package.mask", # masking configuration files -} - -etpMtimeFiles = { - 'keywords_mtime': etpConst['dumpstoragedir']+"/keywords.mtime", # keywording configuration files mtime - 'unmask_mtime': etpConst['dumpstoragedir']+"/unmask.mtime", # unmasking configuration files mtime - 'mask_mtime': etpConst['dumpstoragedir']+"/mask.mtime", # masking configuration files mtime -} +import os ''' This function parses files in etpMaskFiles and returns collected data to caller (dict) ''' -def parse(): - data = {} - for item in etpMaskFiles: - data[item] = eval(item+'_parser')() - return data + +class parser: + + def __init__(self,etpConst,etpCache): + + self.etpConst = etpConst + self.etpCache = etpCache + + + def parse(self): + + self.etpMaskFiles = { + 'keywords': self.etpConst['confdir']+"/packages/package.keywords", # keywording configuration files + 'unmask': self.etpConst['confdir']+"/packages/package.unmask", # unmasking configuration files + 'mask': self.etpConst['confdir']+"/packages/package.mask", # masking configuration files + } + self.etpMtimeFiles = { + 'keywords_mtime': self.etpConst['dumpstoragedir']+"/keywords.mtime", # keywording configuration files mtime + 'unmask_mtime': self.etpConst['dumpstoragedir']+"/unmask.mtime", # unmasking configuration files mtime + 'mask_mtime': self.etpConst['dumpstoragedir']+"/mask.mtime", # masking configuration files mtime + } + + data = {} + for item in self.etpMaskFiles: + data[item] = eval('self.'+item+'_parser')() + return data -''' - parser of package.keywords file -''' -def keywords_parser(): - - __validateEntropyCache(etpMaskFiles['keywords'],etpMtimeFiles['keywords_mtime']) - - data = { - 'universal': set(), - 'packages': {}, - 'repositories': {}, - } - if os.path.isfile(etpMaskFiles['keywords']): - f = open(etpMaskFiles['keywords'],"r") - content = f.readlines() - f.close() - # filter comments and white lines - content = [x.strip() for x in content if not x.startswith("#") and x.strip()] - for line in content: - keywordinfo = line.split() - # skip wrong lines - if len(keywordinfo) > 3: - print ">> "+line+" << is invalid!!" - continue - if len(keywordinfo) == 1: # inversal keywording, check if it's not repo= - # repo=? - if keywordinfo[0].startswith("repo="): + ''' + parser of package.keywords file + ''' + def keywords_parser(self): + + self.__validateEntropyCache(self.etpMaskFiles['keywords'],self.etpMtimeFiles['keywords_mtime']) + + data = { + 'universal': set(), + 'packages': {}, + 'repositories': {}, + } + if os.path.isfile(self.etpMaskFiles['keywords']): + f = open(self.etpMaskFiles['keywords'],"r") + content = f.readlines() + f.close() + # filter comments and white lines + content = [x.strip() for x in content if not x.startswith("#") and x.strip()] + for line in content: + keywordinfo = line.split() + # skip wrong lines + if len(keywordinfo) > 3: print ">> "+line+" << is invalid!!" continue - # atom? is it worth it? it would take a little bit to parse uhm... >50 entries...!? - kinfo = keywordinfo[0] - if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format - data['universal'].add(keywordinfo[0]) - continue # needed? - if len(keywordinfo) in (2,3): # inversal keywording, check if it's not repo= - # repo=? - if keywordinfo[0].startswith("repo="): - print ">> "+line+" << is invalid!!" - continue - # add to repo? - items = keywordinfo[1:] - if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format - reponame = [x for x in items if x.startswith("repo=") and (len(x.split("=")) == 2)] - if reponame: - reponame = reponame[0].split("=")[1] - if reponame not in data['repositories']: - data['repositories'][reponame] = {} - # repository unmask or package in repository unmask? - if keywordinfo[0] not in data['repositories'][reponame]: - data['repositories'][reponame][keywordinfo[0]] = set() - if len(items) == 1: - # repository unmask - data['repositories'][reponame][keywordinfo[0]].add('*') - else: - if "*" not in data['repositories'][reponame][keywordinfo[0]]: - item = [x for x in items if not x.startswith("repo=")] - data['repositories'][reponame][keywordinfo[0]].add(item[0]) - else: - # it's going to be a faulty line!!?? - if len(items) == 2: # can't have two items and no repo= + if len(keywordinfo) == 1: # inversal keywording, check if it's not repo= + # repo=? + if keywordinfo[0].startswith("repo="): print ">> "+line+" << is invalid!!" continue - # add keyword to packages - if keywordinfo[0] not in data['packages']: - data['packages'][keywordinfo[0]] = set() - data['packages'][keywordinfo[0]].add(items[0]) - return data - - -def unmask_parser(): - __validateEntropyCache(etpMaskFiles['unmask'],etpMtimeFiles['unmask_mtime']) + # atom? is it worth it? it would take a little bit to parse uhm... >50 entries...!? + kinfo = keywordinfo[0] + if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format + data['universal'].add(keywordinfo[0]) + continue # needed? + if len(keywordinfo) in (2,3): # inversal keywording, check if it's not repo= + # repo=? + if keywordinfo[0].startswith("repo="): + print ">> "+line+" << is invalid!!" + continue + # add to repo? + items = keywordinfo[1:] + if keywordinfo[0] == "**": keywordinfo[0] = "" # convert into entropy format + reponame = [x for x in items if x.startswith("repo=") and (len(x.split("=")) == 2)] + if reponame: + reponame = reponame[0].split("=")[1] + if reponame not in data['repositories']: + data['repositories'][reponame] = {} + # repository unmask or package in repository unmask? + if keywordinfo[0] not in data['repositories'][reponame]: + data['repositories'][reponame][keywordinfo[0]] = set() + if len(items) == 1: + # repository unmask + data['repositories'][reponame][keywordinfo[0]].add('*') + else: + if "*" not in data['repositories'][reponame][keywordinfo[0]]: + item = [x for x in items if not x.startswith("repo=")] + data['repositories'][reponame][keywordinfo[0]].add(item[0]) + else: + # it's going to be a faulty line!!?? + if len(items) == 2: # can't have two items and no repo= + print ">> "+line+" << is invalid!!" + continue + # add keyword to packages + if keywordinfo[0] not in data['packages']: + data['packages'][keywordinfo[0]] = set() + data['packages'][keywordinfo[0]].add(items[0]) + return data - data = set() - if os.path.isfile(etpMaskFiles['unmask']): - f = open(etpMaskFiles['unmask'],"r") - content = f.readlines() + + def unmask_parser(self): + self.__validateEntropyCache(self.etpMaskFiles['unmask'],self.etpMtimeFiles['unmask_mtime']) + + data = set() + if os.path.isfile(self.etpMaskFiles['unmask']): + f = open(self.etpMaskFiles['unmask'],"r") + content = f.readlines() + f.close() + # filter comments and white lines + content = [x.strip() for x in content if not x.startswith("#") and x.strip()] + for line in content: + # FIXME: need validation? probably not since atomMatch handles it all + # and doesn't care about badly formatted atoms + data.add(line) + return data + + def mask_parser(self): + self.__validateEntropyCache(self.etpMaskFiles['mask'],self.etpMtimeFiles['mask_mtime']) + + data = set() + if os.path.isfile(self.etpMaskFiles['mask']): + f = open(self.etpMaskFiles['mask'],"r") + content = f.readlines() + f.close() + # filter comments and white lines + content = [x.strip() for x in content if not x.startswith("#") and x.strip()] + for line in content: + # FIXME: need validation? probably not since atomMatch handles it all + # and doesn't care about badly formatted atoms + data.add(line) + return data + + ''' + internal functions + ''' + + def __removeRepoCache(self): + if os.path.isdir(self.etpConst['dumpstoragedir']): + content = os.listdir(self.etpConst['dumpstoragedir']) + for file in content: + if file.startswith(self.etpCache['dbMatch']+self.etpConst['dbnamerepoprefix']) and file.endswith(".dmp"): + os.remove(self.etpConst['dumpstoragedir']+"/"+file) + else: + os.makedirs(self.etpConst['dumpstoragedir']) + + def __saveFileMtime(self,toread,tosaveinto): + + if not os.path.isfile(toread): + currmtime = 0.0 + else: + currmtime = os.path.getmtime(toread) + + if not os.path.isdir(self.etpConst['dumpstoragedir']): + os.makedirs(self.etpConst['dumpstoragedir']) + + f = open(tosaveinto,"w") + f.write(str(currmtime)) + f.flush() f.close() - # filter comments and white lines - content = [x.strip() for x in content if not x.startswith("#") and x.strip()] - for line in content: - # FIXME: need validation? probably not since atomMatch handles it all - # and doesn't care about badly formatted atoms - data.add(line) - return data - -def mask_parser(): - __validateEntropyCache(etpMaskFiles['mask'],etpMtimeFiles['mask_mtime']) + - data = set() - if os.path.isfile(etpMaskFiles['mask']): - f = open(etpMaskFiles['mask'],"r") - content = f.readlines() - f.close() - # filter comments and white lines - content = [x.strip() for x in content if not x.startswith("#") and x.strip()] - for line in content: - # FIXME: need validation? probably not since atomMatch handles it all - # and doesn't care about badly formatted atoms - data.add(line) - return data - -''' - internal functions -''' - -def __removeRepoCache(): - if os.path.isdir(etpConst['dumpstoragedir']): - content = os.listdir(etpConst['dumpstoragedir']) - for file in content: - if file.startswith(etpCache['dbMatch']+etpConst['dbnamerepoprefix']) and file.endswith(".dmp"): - os.remove(etpConst['dumpstoragedir']+"/"+file) - else: - os.makedirs(etpConst['dumpstoragedir']) - -def __saveFileMtime(toread,tosaveinto): - - if not os.path.isfile(toread): - currmtime = 0.0 - else: - currmtime = os.path.getmtime(toread) - - if not os.path.isdir(etpConst['dumpstoragedir']): - os.makedirs(etpConst['dumpstoragedir']) - - f = open(tosaveinto,"w") - f.write(str(currmtime)) - f.flush() - f.close() - - -def __validateEntropyCache(maskfile,mtimefile): - # handle on-disk cache validation - # in this case, repositories cache - # if package.keywords is changed, we must destroy cache - if not os.path.isfile(mtimefile): - # we can't know if package.keywords has been updated - # remove repositories caches - __removeRepoCache() - __saveFileMtime(maskfile,mtimefile) - else: - # check mtime - try: - f = open(mtimefile,"r") - mtime = float(f.readline().strip()) - # compare with current mtime + def __validateEntropyCache(self,maskfile,mtimefile): + # handle on-disk cache validation + # in this case, repositories cache + # if package.keywords is changed, we must destroy cache + if not os.path.isfile(mtimefile): + # we can't know if package.keywords has been updated + # remove repositories caches + self.__removeRepoCache() + self.__saveFileMtime(maskfile,mtimefile) + else: + # check mtime try: - currmtime = os.path.getmtime(maskfile) - except OSError: - currmtime = 0.0 - if mtime != currmtime: - __removeRepoCache() - __saveFileMtime(maskfile,mtimefile) - except: - __removeRepoCache() - __saveFileMtime(maskfile,mtimefile) + f = open(mtimefile,"r") + mtime = float(f.readline().strip()) + # compare with current mtime + try: + currmtime = os.path.getmtime(maskfile) + except OSError: + currmtime = 0.0 + if mtime != currmtime: + self.__removeRepoCache() + self.__saveFileMtime(maskfile,mtimefile) + except: + self.__removeRepoCache() + self.__saveFileMtime(maskfile,mtimefile) diff --git a/libraries/portageTools.py b/libraries/portageTools.py index 9174d5a63..c20061c05 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -136,8 +136,9 @@ def getAtomCategory(atom): # please always force =pkgcat/pkgname-ver if possible def getInstalledAtom(atom): + mypath = etpConst['systemroot']+"/" portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getInstalledAtom: called -> "+str(atom)) - rc = portage.db['/']['vartree'].dep_match(str(atom)) + rc = portage.db[mypath]['vartree'].dep_match(str(atom)) if (rc != []): if (len(rc) == 1): return rc[0] @@ -147,10 +148,11 @@ def getInstalledAtom(atom): return None def getPackageSlot(atom): + mypath = etpConst['systemroot']+"/" portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getPackageSlot: called. ") if atom.startswith("="): atom = atom[1:] - rc = portage.db['/']['vartree'].getslot(atom) + rc = portage.db[mypath]['vartree'].getslot(atom) if rc != "": portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getPackageSlot: slot found -> "+str(atom)+" -> "+str(rc)) return rc @@ -159,8 +161,9 @@ def getPackageSlot(atom): return None def getEbuildDbPath(atom): + mypath = etpConst['systemroot']+"/" portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getEbuildDbPath: called -> "+atom) - return portage.db['/']['vartree'].getebuildpath(atom) + return portage.db[mypath]['vartree'].getebuildpath(atom) def getEbuildTreePath(atom): portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getEbuildTreePath: called -> "+atom) @@ -174,7 +177,8 @@ def getEbuildTreePath(atom): def getInstalledAtoms(atom): portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getInstalledAtoms: called -> "+atom) - rc = portage.db['/']['vartree'].dep_match(str(atom)) + mypath = etpConst['systemroot']+"/" + rc = portage.db[mypath]['vartree'].dep_match(str(atom)) if (rc != []): return rc else: @@ -490,16 +494,19 @@ def synthetizeRoughDependencies(roughDependencies, useflags = None): return dependencies, conflicts def getPortageAppDbPath(): - rc = getPortageEnv("ROOT")+portage_const.VDB_PATH + rc = getPortageEnv("ROOT")+etpConst['systemroot']+portage_const.VDB_PATH if (not rc.endswith("/")): return rc+"/" return rc # Collect installed packages -def getInstalledPackages(): +def getInstalledPackages(dbdir = None): portageLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"getInstalledPackages: called.") import os - appDbDir = getPortageAppDbPath() + if not dbdir: + appDbDir = getPortageAppDbPath() + else: + appDbDir = dbdir dbDirs = os.listdir(appDbDir) installedAtoms = [] for pkgsdir in dbDirs: diff --git a/libraries/serverConstants.py b/libraries/serverConstants.py index 14f15a737..697fcaecc 100644 --- a/libraries/serverConstants.py +++ b/libraries/serverConstants.py @@ -24,142 +24,148 @@ import os from sys import exit from entropyConstants import * -# activator section -if (os.path.isfile(etpConst['activatorconf'])): - try: - if (os.stat(etpConst['activatorconf'])[0] != 33152): - os.chmod(etpConst['activatorconf'],0600) - except: - pass - # fill etpConst['activatoruploaduris'] and etpConst['activatordownloaduris'] - f = open(etpConst['activatorconf'],"r") - actconffile = f.readlines() - f.close() - for line in actconffile: - line = line.strip() - if line.startswith("mirror-upload|") and (len(line.split("mirror-upload|")) == 2): - uri = line.split("mirror-upload|")[1] - if uri.endswith("/"): - uri = uri[:len(uri)-1] - etpConst['activatoruploaduris'].append(uri) - if line.startswith("mirror-download|") and (len(line.split("mirror-download|")) == 2): - uri = line.split("mirror-download|")[1] - if uri.endswith("/"): - uri = uri[:len(uri)-1] - etpConst['activatordownloaduris'].append(uri) - if line.startswith("database-format|") and (len(line.split("database-format|")) == 2): - format = line.split("database-format|")[1] - if format in etpConst['etpdatabasesupportedcformats']: - etpConst['etpdatabasefileformat'] = format - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['activatorconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['activatorloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['activatorconf'] - import time - time.sleep(5) - -# reagent section -if (os.path.isfile(etpConst['reagentconf'])): - f = open(etpConst['reagentconf'],"r") - reagentconf = f.readlines() - f.close() - for line in reagentconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['reagentconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['reagentloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['reagentconf'] - import time - time.sleep(5) - elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): - feed = line.split("rss-feed|")[1] - if feed in ("enable","enabled","true","1"): - etpConst['rss-feed'] = True - elif feed in ("disable","disabled","false","0"): - etpConst['rss-feed'] = False - elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2): - feedname = line.split("rss-name|")[1].strip() - etpConst['rss-name'] = feedname - elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2): - etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip() - if not etpConst['rss-base-url'][-1] == "/": - etpConst['rss-base-url'] += "/" - elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2): - etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip() - elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2): - etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip() - elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2): - try: - entries = int(line.split("max-rss-entries|")[1].strip()) - etpConst['rss-max-entries'] = entries - except: - pass - -# mirrors section -if (os.path.isfile(etpConst['mirrorsconf'])): - f = open(etpConst['mirrorsconf'],"r") - databaseconf = f.readlines() - f.close() - for line in databaseconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['mirrorsconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['mirrorsloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['mirrorsconf'] - import time - time.sleep(5) -# spmbackend section -if (os.path.isfile(etpConst['spmbackendconf'])): - f = open(etpConst['spmbackendconf'],"r") - spmconf = f.readlines() - f.close() - for line in spmconf: - if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): - loglevel = line.split("loglevel|")[1] - try: - loglevel = int(loglevel) - except: - print "ERROR: invalid loglevel in: "+etpConst['spmbackendconf'] - exit(51) - if (loglevel > -1) and (loglevel < 3): - etpConst['spmbackendloglevel'] = loglevel - else: - print "WARNING: invalid loglevel in: "+etpConst['spmbackendconf'] - import time - time.sleep(5) +def initConfig_serverConstants(): + + # activator section + if (os.path.isfile(etpConst['activatorconf'])): + try: + if (os.stat(etpConst['activatorconf'])[0] != 33152): + os.chmod(etpConst['activatorconf'],0600) + except: + pass + # fill etpConst['activatoruploaduris'] and etpConst['activatordownloaduris'] + f = open(etpConst['activatorconf'],"r") + actconffile = f.readlines() + f.close() + for line in actconffile: + line = line.strip() + if line.startswith("mirror-upload|") and (len(line.split("mirror-upload|")) == 2): + uri = line.split("mirror-upload|")[1] + if uri.endswith("/"): + uri = uri[:len(uri)-1] + etpConst['activatoruploaduris'].append(uri) + if line.startswith("mirror-download|") and (len(line.split("mirror-download|")) == 2): + uri = line.split("mirror-download|")[1] + if uri.endswith("/"): + uri = uri[:len(uri)-1] + etpConst['activatordownloaduris'].append(uri) + if line.startswith("database-format|") and (len(line.split("database-format|")) == 2): + format = line.split("database-format|")[1] + if format in etpConst['etpdatabasesupportedcformats']: + etpConst['etpdatabasefileformat'] = format + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['activatorconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['activatorloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['activatorconf'] + import time + time.sleep(5) + + # reagent section + if (os.path.isfile(etpConst['reagentconf'])): + f = open(etpConst['reagentconf'],"r") + reagentconf = f.readlines() + f.close() + for line in reagentconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['reagentconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['reagentloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['reagentconf'] + import time + time.sleep(5) + elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2): + feed = line.split("rss-feed|")[1] + if feed in ("enable","enabled","true","1"): + etpConst['rss-feed'] = True + elif feed in ("disable","disabled","false","0"): + etpConst['rss-feed'] = False + elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2): + feedname = line.split("rss-name|")[1].strip() + etpConst['rss-name'] = feedname + elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2): + etpConst['rss-base-url'] = line.split("rss-base-url|")[1].strip() + if not etpConst['rss-base-url'][-1] == "/": + etpConst['rss-base-url'] += "/" + elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2): + etpConst['rss-website-url'] = line.split("rss-website-url|")[1].strip() + elif line.startswith("managing-editor|") and (len(line.split("managing-editor|")) == 2): + etpConst['rss-managing-editor'] = line.split("managing-editor|")[1].strip() + elif line.startswith("max-rss-entries|") and (len(line.split("max-rss-entries|")) == 2): + try: + entries = int(line.split("max-rss-entries|")[1].strip()) + etpConst['rss-max-entries'] = entries + except: + pass + + # mirrors section + if (os.path.isfile(etpConst['mirrorsconf'])): + f = open(etpConst['mirrorsconf'],"r") + databaseconf = f.readlines() + f.close() + for line in databaseconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['mirrorsconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['mirrorsloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['mirrorsconf'] + import time + time.sleep(5) + + + # spmbackend section + if (os.path.isfile(etpConst['spmbackendconf'])): + f = open(etpConst['spmbackendconf'],"r") + spmconf = f.readlines() + f.close() + for line in spmconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['spmbackendconf'] + exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['spmbackendloglevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['spmbackendconf'] + import time + time.sleep(5) + + + # generic settings section + if (os.path.isfile(etpConst['serverconf'])): + f = open(etpConst['serverconf'],"r") + spmconf = f.readlines() + f.close() + for line in spmconf: + if line.startswith("branches|") and (len(line.split("branches|")) == 2): + branches = line.split("branches|")[1] + etpConst['branches'] = [] + for branch in branches.split(): + etpConst['branches'].append(branch) + if etpConst['branch'] not in etpConst['branches']: + etpConst['branches'].append(etpConst['branch']) - -# generic settings section -if (os.path.isfile(etpConst['serverconf'])): - f = open(etpConst['serverconf'],"r") - spmconf = f.readlines() - f.close() - for line in spmconf: - if line.startswith("branches|") and (len(line.split("branches|")) == 2): - branches = line.split("branches|")[1] - etpConst['branches'] = [] - for branch in branches.split(): - etpConst['branches'].append(branch) - if etpConst['branch'] not in etpConst['branches']: - etpConst['branches'].append(etpConst['branch']) +initConfig_serverConstants() \ No newline at end of file