From a8b63adb7ffc81714711fb2ee81fd09cde8d6208 Mon Sep 17 00:00:00 2001 From: lxnay Date: Sat, 31 Mar 2007 08:34:42 +0000 Subject: [PATCH] nice improvements in the database management. Tool for creating empty Entropy databases added. git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@244 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 - handlers/reagent | 3 ++- libraries/activatorTools.py | 10 +++++----- libraries/databaseTools.py | 27 ++++++++++++++++++++++----- libraries/entropyConstants.py | 5 ++++- libraries/entropyTools.py | 10 +++++++--- libraries/portageTools.py | 13 ++++++------- 7 files changed, 46 insertions(+), 23 deletions(-) diff --git a/TODO b/TODO index b602478de..246f8d720 100644 --- a/TODO +++ b/TODO @@ -4,7 +4,6 @@ TODO list (for developers only): - build(), on enzyme, if a package is not going to be installed anymore from an overlay, it must be notified - build(), on enzyme, quickpkg a package before running emerge(), in this case, if the API has been broken, we can easily query the user and fix it - system-wide: add free space check -- reagent: tool to create the empty Entropy DB - activator: for the tidy module I need that reagent includes the SLOT variable. - activator: add support for stable/unstable branch - activator tasks: diff --git a/handlers/reagent b/handlers/reagent index 70432447c..5ddb6ffb0 100644 --- a/handlers/reagent +++ b/handlers/reagent @@ -50,7 +50,8 @@ def print_help(): entropyTools.print_info(" \t\t"+entropyTools.green("search")+"\t\t\t Search a package inside the Entropy packages database") entropyTools.print_info(" \t\t"+entropyTools.green("dump-package-info")+"\t Dump the stored information of package atom(s) to a file") entropyTools.print_info(" \t\t"+entropyTools.green("inject-package-info")+"\t Inject an Entropy dump inside the database (file path is mandatory)") - entropyTools.print_info(" \t\t"+entropyTools.green("restore-package-info")+"\t Reinitialize a package entry looking at the current install.") + entropyTools.print_info(" \t\t"+entropyTools.green("restore-package-info")+"\t Reinitialize a package entry looking at the current install") + entropyTools.print_info(" \t\t"+entropyTools.green("create-empty-database")+"\t Create an empty Entropy database file in the specified ") entropyTools.print_info(" \t"+entropyTools.green(entropyTools.bold("cleanup"))+entropyTools.yellow("\t\t to clean temporary files")) options = sys.argv[1:] diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index add62042a..a66abbf80 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -342,10 +342,10 @@ def database(options): print_info(green(" * ")+red("Checking database status ..."), back = True) - dbTaintFile = False + dbLockFile = False # does the taint file exist? - if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']): - dbTaintFile = True + if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): + dbLockFile = True # are online mirrors locked? mirrorsLocked = False @@ -361,14 +361,14 @@ def database(options): # if the mirrors are locked, we need to change if we have # the taint file in place. Because in this case, the one # that tainted the db, was me. - if (dbTaintFile): + if (dbLockFile): print_info(green(" * ")+red("Updating mirrors with new information ...")) # it's safe to sync syncRemoteDatabases() # remove the online lock file lockDatabases(False) # remove the taint file - os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile']) + os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: print print_error(green(" * ")+red("At the moment, mirrors are locked, someone is working on their databases, try again later ...")) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 39f480658..3c113471d 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -56,14 +56,15 @@ def database(options): dbconn.initializeDatabase() entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Reinitializing Entropy database using Portage database...")) + from portageTools import getInstalledPackages, quickpkg # now run quickpkg for all the packages and then extract data - installedAtoms, atomsnumber = entropyTools.getInstalledPackages() + installedAtoms, atomsnumber = getInstalledPackages() currCounter = 0 import reagentTools for atom in installedAtoms: currCounter += 1 entropyTools.print_info(entropyTools.green(" (")+ entropyTools.blue(str(currCounter))+"/"+entropyTools.red(str(atomsnumber))+entropyTools.green(") ")+entropyTools.red("Analyzing ")+entropyTools.bold(atom)+entropyTools.red(" ...")) - entropyTools.quickpkg(atom,etpConst['packagestmpdir']) + quickpkg(atom,etpConst['packagestmpdir']) # file is etpConst['packagestmpdir']+"/atomscan/"+pkgnamever.tbz2 etpData = reagentTools.extractPkgData(etpConst['packagestmpdir']+"/"+atom.split("/")[1]+".tbz2") # fill the db entry @@ -250,8 +251,23 @@ def database(options): dbconn.closeDB() entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Done.")) - - + elif (options[0] == "create-empty-database"): + mypath = options[1:] + if len(mypath) == 0: + entropyTools.print_error(entropyTools.yellow(" * ")+entropyTools.red("Not enough parameters")) + sys.exit(303) + if (os.path.dirname(mypath[0]) != '') and (not os.path.isdir(os.path.dirname(mypath[0]))): + entropyTools.print_error(entropyTools.green(" * ")+entropyTools.red("Supplied directory does not exist.")) + sys.exit(304) + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Initializing an empty database file with Entropy structure ..."),back = True) + connection = sqlite.connect(mypath[0]) + cursor = connection.cursor() + cursor.execute(etpSQLInitDestroyAll) + cursor.execute(etpSQLInit) + connection.commit() + cursor.close() + connection.close() + entropyTools.print_info(entropyTools.green(" * ")+entropyTools.red("Entropy database file ")+entropyTools.bold(mypath[0])+entropyTools.red(" successfully initialized.")) ############ # Functions and Classes @@ -323,7 +339,7 @@ class etpDatabase: for uri in etpConst['activatoruploaduris']: ftp = handlerFTP(uri) ftp.setCWD(etpConst['etpurirelativepath']) - if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) and (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])): + if (ftp.isFileAvailable(etpConst['etpdatabaselockfile'])) and (not os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])): import time entropyTools.print_info(entropyTools.red(" * ")+entropyTools.bold("WARNING")+entropyTools.red(": online database is already locked. Waiting up to 2 minutes..."), back = True) unlocked = False @@ -432,6 +448,7 @@ class etpDatabase: # never use this unless you know what you're doing def initializeDatabase(self): + self.cursor.execute(etpSQLInitDestroyAll) self.cursor.execute(etpSQLInit) self.commitChanges() diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 39b65c02e..139545d43 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -59,6 +59,9 @@ etpData = { # Entropy database SQL initialization Schema and data structure # MUST BE KEPT IN SYNC with etpData above +etpSQLInitDestroyAll = """ +DROP TABLE IF EXISTS etpData; +""" etpSQLInit = """ CREATE TABLE etpData ( atom VARCHAR(75) PRIMARY KEY, @@ -154,7 +157,7 @@ etpConst = { 'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their shit 'distcc-status': False, # used by Enzyme, if True distcc is enabled 'distccconf': "/etc/distcc/hosts", # distcc hosts configuration file - 'etpdatabasedir': ETP_DIR+ETP_DBDIR, # FIXME: REMOVE THIS ! + 'etpdatabasedir': ETP_DIR+ETP_DBDIR, # 'etpdatabasefilepath': ETP_DIR+ETP_DBDIR+"/"+ETP_DBFILE, 'etpapi': ETP_API, # Entropy database API revision 'headertext': ETP_HEADER_TEXT, # header text that can be outputted to a file diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 76535a6cc..5435b3318 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -375,7 +375,7 @@ def downloadDatabase(uri): dbfile.flush() dbfile.close() del dbcont - print_info(green(" * ")+red("Decompression of ")+bold(etpConst['etpdatabasefilegzip'])+red(" completed ...")) + print_info(green(" * ")+red("Decompression of ")+bold(etpConst['etpdatabasefilegzip'])+red(" completed.")) # downloading revision file print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True) @@ -494,21 +494,25 @@ def lockDatabases(lock = True, mirrorList = []): ftp.closeFTPConnection() continue if (lock): - f = open(etpConst['packagestmpdir']+"/"+etpConst['etpdatabaselockfile'],"w") + f = open(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],"w") f.write("database locked\n") f.flush() f.close() - rc = ftp.uploadFile(etpConst['packagestmpdir']+"/"+etpConst['etpdatabaselockfile'],ascii= True) + rc = ftp.uploadFile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'],ascii= True) if (rc.startswith("226")): print_info(green(" * ")+red("Succesfully locked ")+bold(extractFTPHostFromUri(uri))+red(" mirror.")) else: outstat = True print "\n" print_warning(red(" * ")+red("A problem occured while locking ")+bold(extractFTPHostFromUri(uri))+red(" mirror. Please have a look.")) + if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): + os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: rc = ftp.deleteFile(etpConst['etpdatabaselockfile']) if (rc): print_info(green(" * ")+red("Succesfully unlocked ")+bold(extractFTPHostFromUri(uri))+red(" mirror.")) + if os.path.isfile(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']): + os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile']) else: outstat = True print "\n" diff --git a/libraries/portageTools.py b/libraries/portageTools.py index 1010e6de6..bb6c87fba 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -677,6 +677,12 @@ def synthetizeRoughDependencies(roughDependencies, useflags = None): return dependencies, conflicts +def getPortageAppDbPath(): + rc = getPortageEnv("ROOT")+portage_const.VDB_PATH + if (not rc.endswith("/")): + return rc+"/" + return rc + # Collect installed packages def getInstalledPackages(): import os @@ -692,13 +698,6 @@ def getInstalledPackages(): installedAtoms.append(pkgatom) return installedAtoms, len(installedAtoms) -def getPortageAppDbPath(): - rc = getPortageEnv("ROOT")+portage_const.VDB_PATH - if (not rc.endswith("/")): - return rc+"/" - return rc - - def packageSearch(keyword): SearchDirs = []