From 76d8d9596c6d2a9c8effc656bdeda313afcdd60c Mon Sep 17 00:00:00 2001 From: lxnay Date: Sat, 11 Oct 2008 21:11:07 +0000 Subject: [PATCH] Entropy/TODO: - update TODO Entropy/Equo: - add installed packages database backup/restore tools Entropy/EquoInterface: - add backup/restore database helper functions Entropy/EntropyDatabaseInterface: - fix getIDPackageFromDownload() Entropy/entropyTools: - add compress_file() function (uncompress_file() already present) git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@2518 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 2 +- client/equo.py | 28 ++++---- client/text_rescue.py | 38 +++++++++++ libraries/entropy.py | 120 ++++++++++++++++++++++++++++++++-- libraries/entropyConstants.py | 1 + libraries/entropyTools.py | 15 +++++ 6 files changed, 183 insertions(+), 21 deletions(-) diff --git a/TODO b/TODO index 65266987d..4688a44a1 100644 --- a/TODO +++ b/TODO @@ -3,13 +3,13 @@ TODO list - text ADS - p2p way to fight security attacks (through UGC and triple md5 verification) - force a package to keep the current version + - masking/unmasking functions - server-side messages collection - community repos: add repository handling on dependencies to avoid library breakages - External trigger: rewrite support and use bash - split entropy.py into entropy package - UGC: bug report documents? - UGC: ads documents/rotation support? - - database backup tools - implement pdepend support in dependencies sorting - implement package sets (meta packages list) - use expiration as a base to have and handle multiple packages in repo? diff --git a/client/equo.py b/client/equo.py index 3cc6ff0c2..2900bd513 100644 --- a/client/equo.py +++ b/client/equo.py @@ -171,17 +171,19 @@ myopts = [ (3,'--savedir',1,_('save new metadata into the specified directory')), None, (1,'database',1,_('handles installed packages database')), - (2,'check',2,_('check System Database for errors')), - #(2,'generate',2,'generate installed packages database using Portage database (Portage needed)'), - (2,'resurrect',1,_('generate installed packages database using files on the system [last hope]')), - (2,'depends',2,_('regenerate depends caching table')), - (2,'counters',1,_('update/generate counters table (Portage <-> Entropy packages table)')), - (2,'gentoosync',1,_('makes Entropy aware of your Portage-updated packages')), + (2,'check',2,_('check System Database for errors')), + (2,'generate',1,'generate installed packages database using Portage database (Portage needed)'), + (2,'resurrect',1,_('generate installed packages database using files on the system [last hope]')), + (2,'depends',2,_('regenerate depends caching table')), + (2,'counters',1,_('update/generate counters table (Portage <-> Entropy packages table)')), + (2,'gentoosync',1,_('makes Entropy aware of your Portage-updated packages')), + (2,'backup',2,_('backup the current Entropy installed packages database')), + (2,'restore',2,_('restore a previously backed up Entropy installed packages database')), None, (1,'packages',1,_('handles packages helper applications')), - (2,'python-updater',1,_('migrate all Python modules to the latest installed version')), - (2,'--ask',2,_('ask before making any changes')), - (2,'--pretend',1,_('just show what would be done')), + (2,'python-updater',1,_('migrate all Python modules to the latest installed version')), + (2,'--ask',2,_('ask before making any changes')), + (2,'--pretend',1,_('just show what would be done')), None, (1,'community',1,_('handles community-side features')), @@ -267,10 +269,10 @@ myopts = [ None, (1,'cache',2,_('handles Entropy cache')), - (2,'clean',2,_('clean Entropy cache')), - (2,'generate',1,_('regenerate Entropy cache')), - (2,'--verbose',1,_('show more details')), - (2,'--quiet',2,_('print results in a scriptable way')), + (2,'clean',2,_('clean Entropy cache')), + (2,'generate',1,_('regenerate Entropy cache')), + (2,'--verbose',1,_('show more details')), + (2,'--quiet',2,_('print results in a scriptable way')), None, (1,'cleanup',2,_('remove downloaded packages and clean temp. directories')), None, diff --git a/client/text_rescue.py b/client/text_rescue.py index b82a0c788..f9899cf29 100644 --- a/client/text_rescue.py +++ b/client/text_rescue.py @@ -597,6 +597,44 @@ def database(options): Equo._resources_run_remove_lock() return 0 + elif (options[0] == "backup"): + + status = Equo.backupDatabase(etpConst['etpdatabaseclientfilepath']) + if status: + return 0 + return 1 + + elif (options[0] == "restore"): + + dblist = Equo.list_backedup_client_databases() + if not dblist: + print_info(brown(" @@ ")+blue("%s." % (_("No backed up databases found"),))) + return 1 + + mydblist = [] + for mydb in dblist: + ts = Equo.entropyTools.getFileUnixMtime(mydb) + mytime = Equo.entropyTools.convertUnixTimeToHumanTime(ts) + mydblist.append("[%s] %s" % (mytime,mydb,)) + + def fake_cb(s): + return s + + input_params = [ + ('db',('combo',(_('Select the database you want to restore'),mydblist),),fake_cb,True) + ] + + data = Equo.inputBox(red(_("Entropy installed packages database restore tool")), input_params, cancel_button = True) + if data == None: + return 1 + myid, dbx = data['db'] + dbpath = dblist[myid] + + status = Equo.restoreDatabase(dbpath, etpConst['etpdatabaseclientfilepath']) + if status: + return 0 + return 1 + else: return -10 diff --git a/libraries/entropy.py b/libraries/entropy.py index 13b6ccae3..2bec4e99d 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -851,6 +851,12 @@ class EquoInterface(TextInterface): etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotect']] etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotectmask']] + def listAllAvailableBranches(self): + branches = set() + for repo in self.validRepositories: + dbconn = self.openRepositoryDatabase(repo) + branches.update(dbconn.listAllBranches()) + return branches def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False, indexing_override = None, skipChecks = False): if xcache == None: @@ -888,13 +894,114 @@ class EquoInterface(TextInterface): dbc.initializeDatabase() return dbc - def listAllAvailableBranches(self): - branches = set() - for repo in self.validRepositories: - dbconn = self.openRepositoryDatabase(repo) - branches.update(dbconn.listAllBranches()) - return branches + def backupDatabase(self, dbpath, backup_dir = None, silent = False, compress_level = 9): + if compress_level not in range(1,10): + compress_level = 9 + + backup_dir = os.path.dirname(dbpath) + if not backup_dir: backup_dir = os.path.dirname(dbpath) + dbname = os.path.basename(dbpath) + bytes_required = 1024000*300 + if not (os.access(backup_dir,os.W_OK) and \ + os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \ + os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)): + if not silent: + mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),) + self.updateProgress( + mytxt, + importance = 1, + type = "error", + header = red(" @@ ") + ) + return False + + def get_ts(): + from datetime import datetime + ts = datetime.fromtimestamp(time.time()) + return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second) + + comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),) + comp_dbpath = os.path.join(backup_dir,comp_dbname) + if not silent: + mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + import bz2 + try: + self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level) + except: + if not silent: + self.entropyTools.printTraceback() + return False + + if not silent: + mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + return True + + def restoreDatabase(self, backup_path, db_destination, silent = False): + + bytes_required = 1024000*300 + if not (os.access(db_destination,os.W_OK) and \ + os.path.isfile(db_destination) and os.path.isfile(backup_path) and \ + os.access(backup_path,os.R_OK) and self.entropyTools.check_required_space(os.path.dirname(db_destination), bytes_required)): + if not silent: + mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")),blue(backup_path),darkred(_("permission denied")),) + self.updateProgress( + mytxt, + importance = 1, + type = "error", + header = red(" @@ ") + ) + return False + + if not silent: + mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")),blue(os.path.basename(backup_path)),blue(db_destination),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + + import bz2 + try: + self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File) + except: + if not silent: + self.entropyTools.printTraceback() + return False + + if not silent: + mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")),blue(db_destination),) + self.updateProgress( + mytxt, + importance = 1, + type = "info", + header = blue(" @@ "), + back = True + ) + return True + + def list_backedup_client_databases(self): + client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath']) + return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \ + if x.startswith(etpConst['dbbackupprefix']) and \ + os.access(os.path.join(client_dbdir,x),os.R_OK) + ] ''' Cache stuff :: begin @@ -30169,7 +30276,6 @@ class EntropyDatabaseInterface: return -1 def getIDPackageFromDownload(self, download_relative_path, endswith = False): - if branch == None: branch = self.db_branch if endswith: self.cursor.execute('SELECT baseinfo.idpackage FROM baseinfo,extrainfo WHERE extrainfo.download LIKE (?)', ("%"+download_relative_path,)) else: diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index de948f7f7..5aaf882d9 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -626,6 +626,7 @@ def const_defaultSettings(rootdir): 'etpdatabaseclientdir': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR, 'etpdatabaseclientfilepath': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR+"/"+ETP_DBCLIENTFILE, # path to equo.db - client side database file 'dbnamerepoprefix': "repo_", # prefix of the name of self.dbname in EntropyDatabaseInterface class for the repositories + 'dbbackupprefix': 'etp_backup_', # prefix of database backups 'etpapi': etpSys['api'], # Entropy database API revision 'currentarch': etpSys['arch'], # contains the current running architecture diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index 308a5f0ca..1a109891f 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -445,6 +445,21 @@ def uncompress_file(file_path, destination_path, opener): f_out.close() f_in.close() +def compress_file(file_path, destination_path, opener, compress_level = None): + f_in = open(file_path,"rb") + if compress_level != None: + f_out = opener(destination_path,"wb",compresslevel = compress_level) + else: + f_out = opener(destination_path,"wb") + data = f_in.read(8192) + while data: + f_out.write(data) + data = f_in.read(8192) + if hasattr(f_out,'flush'): + f_out.flush() + f_out.close() + f_in.close() + def unpackGzip(gzipfilepath): import gzip filepath = gzipfilepath[:-3] # remove .gz