From 7df0cdd108b92811d5d58cd51f8aa228f99579fa Mon Sep 17 00:00:00 2001 From: lxnay Date: Thu, 26 Jul 2007 22:38:23 +0000 Subject: [PATCH] some work on equo, now the database controller is more flexible git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@335 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 + client/equo | 26 +++++++++++------- client/equoTools.py | 48 ++++++++++++++++++++++++++++++-- conf/equo.conf | 21 ++++++++++++++ libraries/databaseTools.py | 52 ++++++++++++++++++++++++++++++----- libraries/entropyConstants.py | 38 ++++++++++++++++++++++++- 6 files changed, 166 insertions(+), 20 deletions(-) create mode 100644 conf/equo.conf diff --git a/TODO b/TODO index 02b4f091b..b800a8cf1 100644 --- a/TODO +++ b/TODO @@ -4,6 +4,7 @@ TODO list: - enzyme, build(): add an option to force the continuation of the emerge queue even if a package fails - enzyme, add a module that integrates revdep-rebuild - test stabilization code +- equo, when we'll ship the first release with etp, use the packages.db for the client.db Project Status: - entropy: will handle all the three tools below diff --git a/client/equo b/client/equo index 7fc37ccd6..732514a0c 100644 --- a/client/equo +++ b/client/equo @@ -24,6 +24,7 @@ import sys import string import equoTools import entropyTools +from entropyConstants import * from outputTools import * # CONSTANTS @@ -37,17 +38,18 @@ def print_help(): print_info(" --verbose\t\tbe more verbose") print_info(" --nocolor\t\tdisable colorized output") print_info(blue("Tools available: ")) - print_info(" \t"+green(bold("repo"))+yellow("\t\t to handle package repositories")) - print_info(" \t\t"+green("sync")+red("\t\t\t sync repositories")) - print_info(" \t\t"+green("show")+red("\t\t\t show enabled repositories")) - print_info(" \t\t"+green("status")+red("\t\t\t show respositories status")) - print_info(" \t"+green(bold("package"))+yellow("\t\t to handle packages")) - print_info(" \t\t"+green("search")+red("\t\t\t to search a package inside the database")) - print_info(" \t\t"+red("--ask")+"\t\t\t ask before making any changes") - print_info(" \t\t"+red("--pretend")+"\t\t just show what would be done") + print_info(" \t"+darkgreen(bold("repo"))+brown("\t\t to handle package repositories")) + print_info(" \t\t"+darkgreen("sync")+red("\t\t\t sync repositories")) + print_info(" \t\t"+darkgreen("show")+red("\t\t\t show enabled repositories")) + print_info(" \t\t"+darkgreen("status")+red("\t\t\t show respositories status")) + print_info(" \t"+darkgreen(bold("package"))+brown("\t\t to handle packages")) + print_info(" \t\t"+darkgreen("search")+red("\t\t\t to search a package inside the database")) + print_info(" \t\t"+darkgreen("install")+red("\t\t\t to install a package")) + print_info(" \t\t\t"+red("--ask")+"\t\t ask before making any changes") + print_info(" \t\t\t"+red("--pretend")+"\t just show what would be done") - print_info(" \t"+green(bold("database"))+yellow("\t to handle installed packages database")) - print_info(" \t\t"+green("void")+red("\t\t\t void void void void void void void")) + print_info(" \t"+darkgreen(bold("database"))+brown("\t to handle installed packages database")) + print_info(" \t\t"+darkgreen("generate")+red("\t\t generate installed packages database")) options = sys.argv[1:] @@ -81,4 +83,8 @@ elif (options[0] == "repo"): elif (options[0] == "package"): rc = equoTools.package(options[1:]) + sys.exit(rc) + +elif (options[0] == "database"): + rc = equoTools.database(options[1:]) sys.exit(rc) \ No newline at end of file diff --git a/client/equoTools.py b/client/equoTools.py index 1c4d2c8f7..c2b7550f7 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -29,7 +29,7 @@ sys.path.append('../libraries') from entropyConstants import * from outputTools import * from remoteTools import downloadData -from entropyTools import unpackGzip,compareMd5,bytesIntoHuman,convertUnixTimeToHumanTime +from entropyTools import unpackGzip,compareMd5,bytesIntoHuman,convertUnixTimeToHumanTime,askquestion,getRandomNumber ######################################################## @@ -197,6 +197,19 @@ def syncRepositories(reponames = []): return 0 +def backupClientDatabase(): + if os.path.isfile(etpConst['etpdatabaseclientfilepath']): + import shutil + rnd = getRandomNumber() + source = etpConst['etpdatabaseclientfilepath'] + dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd) + shutil.copy2(source,dest) + user = os.stat(source)[4] + group = os.stat(source)[5] + os.chown(dest,user,group) + shutil.copystat(source,dest) + return dest + return "" ######################################################## #### @@ -230,6 +243,37 @@ def package(options): return rc +def database(options): + + if len(options) < 1: + return 0 + + if (options[0] == "generate"): + print_warning(bold("####### ATTENTION -> ")+red("The installed package database will be regenerated, this will take a LOT of time.")) + print_warning(bold("####### ATTENTION -> ")+red("Sabayon Linux Officially Repository MUST be on top of the repositories list in ")+etpConst['repositoriesconf']) + rc = askquestion(" Can I continue ?") + if rc == "No": + sys.exit(0) + rc = askquestion(" Are you REALLY sure ?") + if rc == "No": + sys.exit(0) + rc = askquestion(" Do you even know what you're doing ?") + if rc == "No": + sys.exit(0) + + # ok, he/she knows it... hopefully + # if exist, copy old database + print etpConst['etpdatabaseclientfilepath'] + print_info(red(" @@ ")+blue("Creating backup of the previous database, if exists.")+red(" @@")) + newfile = backupClientDatabase() + if (newfile): + print_info(red(" @@ ")+blue("Database copied to file ")+newfile+red(" @@")) + + # Now reinitialize it + # dbconn = etpDatabase(readOnly = False, noUpload = True) -> specify client mode and file + # dbconn.initializeDatabase() + + def searchPackage(packages): from databaseTools import etpDatabase @@ -276,7 +320,7 @@ def searchPackage(packages): if foundBranchIndex > myBranchIndex: # package found in branch more unstable than the selected one, for us, it does not exist continue - + # now fetch essential info pkgatom = dbconn.retrieveAtom(id) pkgname = dbconn.retrieveName(id) diff --git a/conf/equo.conf b/conf/equo.conf new file mode 100644 index 000000000..09739bd9d --- /dev/null +++ b/conf/equo.conf @@ -0,0 +1,21 @@ +# Project Entropy 1.0 Equo configuration file + +# +# synthax for gentoo-compat: +# +# gentoo-compat: Enable/Disable Gentoo Portage compatibility +# gentoo-compat|enable or disable (no spaces!) +# +# example: +# gentoo-compat|enable +# or: +# gentoo-compat|disable +# +# NOTE: once you disable it, YOU SHOULD NOT RE-ENABLE IT AGAIN ! +gentoo-compat|enable + +# Log level +# 0: No Logging +# 1: Normal Logging +# 2: Verbose Logging +loglevel|1 diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index fe1522808..e3e73de12 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -730,22 +730,31 @@ class databaseStatus: class etpDatabase: - def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath']): + def __init__(self, readOnly = False, noUpload = False, dbFile = etpConst['etpdatabasefilepath'], clientDatabase = False): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase.__init__ called.") self.readOnly = readOnly self.noUpload = noUpload + self.clientDatabase = clientDatabase - if (self.readOnly): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened readonly") + if (self.clientDatabase): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened by Entropy client, file: "+str(dbFile)) # if the database is opened readonly, we don't need to lock the online status self.connection = sqlite.connect(dbFile) self.cursor = self.connection.cursor() # set the table read only return - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened in read/write mode") + if (self.readOnly): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened readonly, file: "+str(dbFile)) + # if the database is opened readonly, we don't need to lock the online status + self.connection = sqlite.connect(dbFile) + self.cursor = self.connection.cursor() + # set the table read only + return + + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"etpDatabase: database opened in read/write mode, file: "+str(dbFile)) # check if the database is locked locally if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): @@ -804,7 +813,7 @@ class etpDatabase: # ok done... now sync the new db, if needed entropyTools.syncRemoteDatabases(self.noUpload) - self.connection = sqlite.connect(etpConst['etpdatabasefilepath']) + self.connection = sqlite.connect(dbFile) self.cursor = self.connection.cursor() def closeDB(self): @@ -816,6 +825,14 @@ class etpDatabase: self.cursor.close() self.connection.close() return + + # if it's equo that's calling the function, just save changes and quit + if (self.clientDatabase): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"closeDB: closing database opened by Entropy Client.") + self.commitChanges() + self.cursor.close() + self.connection.close() + return dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"closeDB: closing database opened in read/write.") @@ -844,6 +861,9 @@ class etpDatabase: self.discardChanges() # is it ok? def taintDatabase(self): + if (self.clientDatabase): # if it's equo to open it, this should be avoided + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"taintDatabase: called by Entropy client, won't do anything.") + return dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"taintDatabase: called.") # taint the database status f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'],"w") @@ -853,6 +873,9 @@ class etpDatabase: entropyTools.dbStatus.setDatabaseTaint(True) def untaintDatabase(self): + if (self.clientDatabase): # if it's equo to open it, this should be avoided + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"untaintDatabase: called by Entropy client, won't do anything.") + return dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"untaintDatabase: called.") entropyTools.dbStatus.setDatabaseTaint(False) # untaint the database status @@ -899,6 +922,10 @@ class etpDatabase: # otherwise it fires up updatePackage def handlePackage(self, etpData, forceBump = False): + if (self.readOnly): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlePackage: Cannot handle this in read only.") + raise Exception, "What are you trying to do?" + # prepare versiontag versiontag = "" if (etpData['versiontag']): @@ -914,6 +941,10 @@ class etpDatabase: # default add an unstable package def addPackage(self, etpData, revision = 0, wantedBranch = "unstable"): + if (self.readOnly): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: Cannot handle this in read only.") + raise Exception, "What are you trying to do?" + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addPackage: called.") # Handle package name @@ -1115,6 +1146,10 @@ class etpDatabase: # returns False,revision if not def updatePackage(self, etpData, forceBump = False): + if (self.readOnly): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"updatePackage: Cannot handle this in read only.") + raise Exception, "What are you trying to do?" + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"updatePackage: called.") # prepare versiontag @@ -1184,9 +1219,12 @@ class etpDatabase: return x,y,z - # You must provide the full atom to this function - # FIXME: this must be fixed to work with branches def removePackage(self,idpackage): + + if (self.readOnly): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"removePackage: Cannot handle this in read only.") + raise Exception, "What are you trying to do?" + key = self.retrieveAtom(idpackage) branch = self.retrieveBranch(idpackage) idpackage = str(idpackage) diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 78499e91c..0547fe13f 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -205,6 +205,7 @@ ETP_PORTDIR = "/portage" ETP_DISTFILESDIR = "/distfiles" ETP_DBDIR = "/database/"+ETP_ARCH_CONST ETP_DBFILE = "packages.db" +ETP_DBCLIENTFILE = "equo.db" ETP_CLIENT_REPO_DIR = "/client" ETP_UPLOADDIR = "/upload/"+ETP_ARCH_CONST ETP_STOREDIR = "/store/"+ETP_ARCH_CONST @@ -247,6 +248,7 @@ etpConst = { 'spmbackendconf': ETP_CONF_DIR+"/spmbackend.conf", # Source Package Manager backend configuration (Portage now) 'mirrorsconf': ETP_CONF_DIR+"/mirrors.conf", # mirrors.conf file 'remoteconf': ETP_CONF_DIR+"/remote.conf", # remote.conf file + 'equoconf': ETP_CONF_DIR+"/equo.conf", # equo.conf file 'activatoruploaduris': [], # list of URIs that activator can use to upload files (parsed from activator.conf) 'activatordownloaduris': [], # list of URIs that activator can use to fetch data 'binaryurirelativepath': "packages/"+ETP_ARCH_CONST+"/", # Relative remote path for the binary repository. @@ -268,6 +270,7 @@ etpConst = { 'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info) 'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info) 'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info) + 'equologlevel': 1, # # Equo log level (default: 1 - see equo.conf for more info) 'spmbackendloglevel': 1, # # Source Package Manager backend log level (default: 1 - see entropy.conf for more info) 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit @@ -280,6 +283,7 @@ etpConst = { 'reagentlogfile': ETP_SYSLOG_DIR+"/reagent.log", # Reagent operations log file 'activatorlogfile': ETP_SYSLOG_DIR+"/activator.log", # Activator operations log file 'entropylogfile': ETP_SYSLOG_DIR+"/entropy.log", # Activator operations log file + 'equologfile': ETP_SYSLOG_DIR+"/equo.log", # Activator operations log file 'distcc-status': False, # used by Enzyme, if True distcc is enabled @@ -287,6 +291,7 @@ etpConst = { 'etpdatabasedir': ETP_DIR+ETP_DBDIR, 'etpdatabasefilepath': ETP_DIR+ETP_DBDIR+"/"+ETP_DBFILE, 'etpdatabaseclientdir': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR, + 'etpdatabaseclientfilepath': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR+"/"+ETP_DBCLIENTFILE, # path to equo.db - client side database file 'etpapi': ETP_API, # Entropy database API revision 'headertext': ETP_HEADER_TEXT, # header text that can be outputted to a file @@ -297,6 +302,7 @@ etpConst = { 'branches': ["stable","unstable"], # available branches, do not scramble! 'branch': "unstable", # choosen branch + 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) } # Handlers used by entropy to run and retrieve data remotely, using php helpers @@ -574,7 +580,37 @@ else: print "WARNING: invalid loglevel in: "+etpConst['databaseconf'] import time time.sleep(5) - + +# equo section +if (not os.path.isfile(etpConst['equoconf'])): + print "ERROR: "+etpConst['equoconf']+" does not exist" + sys.exit(50) +else: + f = open(etpConst['equoconf'],"r") + equoconf = f.readlines() + f.close() + for line in equoconf: + if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2): + loglevel = line.split("loglevel|")[1] + try: + loglevel = int(loglevel) + except: + print "ERROR: invalid loglevel in: "+etpConst['equoconf'] + sys.exit(51) + if (loglevel > -1) and (loglevel < 3): + etpConst['equologlevel'] = loglevel + else: + print "WARNING: invalid loglevel in: "+etpConst['equoconf'] + import time + time.sleep(5) + + if line.startswith("gentoo-compat|") and (len(line.split("|")) == 2): + compatopt = line.split("|")[1].strip() + if compatopt == "disable": + etpConst['gentoo-compat'] = False + else: + etpConst['gentoo-compat'] = True + # mirrors section if (not os.path.isfile(etpConst['mirrorsconf'])): print "ERROR: "+etpConst['mirrorsconf']+" does not exist"