Entropy/TODO:

- update TODO
Entropy/Equo:
- add installed packages database backup/restore tools
Entropy/EquoInterface:
- add backup/restore database helper functions
Entropy/EntropyDatabaseInterface:
- fix getIDPackageFromDownload()
Entropy/entropyTools:
- add compress_file() function (uncompress_file() already present)


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@2518 cd1c1023-2f26-0410-ae45-c471fc1f0318

TODO

@@ -3,13 +3,13 @@ TODO list
   - text ADS
   - p2p way to fight security attacks (through UGC and triple md5 verification)
   - force a package to keep the current version
+  - masking/unmasking functions
   - server-side messages collection
   - community repos: add repository handling on dependencies to avoid library breakages
   - External trigger: rewrite support and use bash
   - split entropy.py into entropy package
   - UGC: bug report documents?
   - UGC: ads documents/rotation support?
-  - database backup tools
   - implement pdepend support in dependencies sorting
   - implement package sets (meta packages list)
   - use expiration as a base to have and handle multiple packages in repo?

client/equo.py

@@ -171,17 +171,19 @@ myopts = [
     (3,'--savedir',1,_('save new metadata into the specified directory')),
     None,
     (1,'database',1,_('handles installed packages database')),
-    (2,'check',2,_('check System Database for errors')),
-    #(2,'generate',2,'generate installed packages database using Portage database (Portage needed)'),
-    (2,'resurrect',1,_('generate installed packages database using files on the system [last hope]')),
-    (2,'depends',2,_('regenerate depends caching table')),
-    (2,'counters',1,_('update/generate counters table (Portage <-> Entropy packages table)')),
-    (2,'gentoosync',1,_('makes Entropy aware of your Portage-updated packages')),
+        (2,'check',2,_('check System Database for errors')),
+        (2,'generate',1,'generate installed packages database using Portage database (Portage needed)'),
+        (2,'resurrect',1,_('generate installed packages database using files on the system [last hope]')),
+        (2,'depends',2,_('regenerate depends caching table')),
+        (2,'counters',1,_('update/generate counters table (Portage <-> Entropy packages table)')),
+        (2,'gentoosync',1,_('makes Entropy aware of your Portage-updated packages')),
+        (2,'backup',2,_('backup the current Entropy installed packages database')),
+        (2,'restore',2,_('restore a previously backed up Entropy installed packages database')),
     None,
     (1,'packages',1,_('handles packages helper applications')),
-    (2,'python-updater',1,_('migrate all Python modules to the latest installed version')),
-    (2,'--ask',2,_('ask before making any changes')),
-    (2,'--pretend',1,_('just show what would be done')),
+        (2,'python-updater',1,_('migrate all Python modules to the latest installed version')),
+        (2,'--ask',2,_('ask before making any changes')),
+        (2,'--pretend',1,_('just show what would be done')),
     None,
     (1,'community',1,_('handles community-side features')),
 
@@ -267,10 +269,10 @@ myopts = [
 
     None,
     (1,'cache',2,_('handles Entropy cache')),
-    (2,'clean',2,_('clean Entropy cache')),
-    (2,'generate',1,_('regenerate Entropy cache')),
-    (2,'--verbose',1,_('show more details')),
-    (2,'--quiet',2,_('print results in a scriptable way')),
+        (2,'clean',2,_('clean Entropy cache')),
+        (2,'generate',1,_('regenerate Entropy cache')),
+        (2,'--verbose',1,_('show more details')),
+        (2,'--quiet',2,_('print results in a scriptable way')),
     None,
     (1,'cleanup',2,_('remove downloaded packages and clean temp. directories')),
     None,

client/text_rescue.py

@@ -597,6 +597,44 @@ def database(options):
         Equo._resources_run_remove_lock()
         return 0
 
+    elif (options[0] == "backup"):
+
+        status = Equo.backupDatabase(etpConst['etpdatabaseclientfilepath'])
+        if status:
+            return 0
+        return 1
+
+    elif (options[0] == "restore"):
+
+        dblist = Equo.list_backedup_client_databases()
+        if not dblist:
+            print_info(brown(" @@ ")+blue("%s." % (_("No backed up databases found"),)))
+            return 1
+
+        mydblist = []
+        for mydb in dblist:
+            ts = Equo.entropyTools.getFileUnixMtime(mydb)
+            mytime = Equo.entropyTools.convertUnixTimeToHumanTime(ts)
+            mydblist.append("[%s] %s" % (mytime,mydb,))
+
+        def fake_cb(s):
+            return s
+
+        input_params = [
+            ('db',('combo',(_('Select the database you want to restore'),mydblist),),fake_cb,True)
+        ]
+
+        data = Equo.inputBox(red(_("Entropy installed packages database restore tool")), input_params, cancel_button = True)
+        if data == None:
+            return 1
+        myid, dbx = data['db']
+        dbpath = dblist[myid]
+
+        status = Equo.restoreDatabase(dbpath, etpConst['etpdatabaseclientfilepath'])
+        if status:
+            return 0
+        return 1
+
     else:
         return -10
 

libraries/entropy.py

@@ -851,6 +851,12 @@ class EquoInterface(TextInterface):
         etpRepositories[repoid]['configprotect'] += [etpConst['systemroot']+x for x in etpConst['configprotect'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotect']]
         etpRepositories[repoid]['configprotectmask'] += [etpConst['systemroot']+x for x in etpConst['configprotectmask'] if etpConst['systemroot']+x not in etpRepositories[repoid]['configprotectmask']]
 
+    def listAllAvailableBranches(self):
+        branches = set()
+        for repo in self.validRepositories:
+            dbconn = self.openRepositoryDatabase(repo)
+            branches.update(dbconn.listAllBranches())
+        return branches
 
     def openGenericDatabase(self, dbfile, dbname = None, xcache = None, readOnly = False, indexing_override = None, skipChecks = False):
         if xcache == None:
@@ -888,13 +894,114 @@ class EquoInterface(TextInterface):
         dbc.initializeDatabase()
         return dbc
 
-    def listAllAvailableBranches(self):
-        branches = set()
-        for repo in self.validRepositories:
-            dbconn = self.openRepositoryDatabase(repo)
-            branches.update(dbconn.listAllBranches())
-        return branches
+    def backupDatabase(self, dbpath, backup_dir = None, silent = False, compress_level = 9):
 
+        if compress_level not in range(1,10):
+            compress_level = 9
+
+        backup_dir = os.path.dirname(dbpath)
+        if not backup_dir: backup_dir = os.path.dirname(dbpath)
+        dbname = os.path.basename(dbpath)
+        bytes_required = 1024000*300
+        if not (os.access(backup_dir,os.W_OK) and \
+                os.path.isdir(backup_dir) and os.path.isfile(dbpath) and \
+                os.access(dbpath,os.R_OK) and self.entropyTools.check_required_space(backup_dir, bytes_required)):
+            if not silent:
+                mytxt = "%s: %s, %s" % (darkred(_("Cannot backup selected database")),blue(dbpath),darkred(_("permission denied")),)
+                self.updateProgress(
+                    mytxt,
+                    importance = 1,
+                    type = "error",
+                    header = red(" @@ ")
+                )
+            return False
+
+        def get_ts():
+            from datetime import datetime
+            ts = datetime.fromtimestamp(time.time())
+            return "%s%s%s_%sh%sm%ss" % (ts.year,ts.month,ts.day,ts.hour,ts.minute,ts.second)
+
+        comp_dbname = "%s%s.%s.bz2" % (etpConst['dbbackupprefix'],dbname,get_ts(),)
+        comp_dbpath = os.path.join(backup_dir,comp_dbname)
+        if not silent:
+            mytxt = "%s: %s ..." % (darkgreen(_("Backing up database to")),blue(comp_dbpath),)
+            self.updateProgress(
+                mytxt,
+                importance = 1,
+                type = "info",
+                header = blue(" @@ "),
+                back = True
+            )
+        import bz2
+        try:
+            self.entropyTools.compress_file(dbpath, comp_dbpath, bz2.BZ2File, compress_level)
+        except:
+            if not silent:
+                self.entropyTools.printTraceback()
+            return False
+
+        if not silent:
+            mytxt = "%s: %s" % (darkgreen(_("Database backed up successfully")),blue(comp_dbpath),)
+            self.updateProgress(
+                mytxt,
+                importance = 1,
+                type = "info",
+                header = blue(" @@ "),
+                back = True
+            )
+        return True
+
+    def restoreDatabase(self, backup_path, db_destination, silent = False):
+
+        bytes_required = 1024000*300
+        if not (os.access(db_destination,os.W_OK) and \
+                os.path.isfile(db_destination) and os.path.isfile(backup_path) and \
+                os.access(backup_path,os.R_OK) and self.entropyTools.check_required_space(os.path.dirname(db_destination), bytes_required)):
+            if not silent:
+                mytxt = "%s: %s, %s" % (darkred(_("Cannot restore selected backup")),blue(backup_path),darkred(_("permission denied")),)
+                self.updateProgress(
+                    mytxt,
+                    importance = 1,
+                    type = "error",
+                    header = red(" @@ ")
+                )
+            return False
+
+        if not silent:
+            mytxt = "%s: %s => %s ..." % (darkgreen(_("Restoring backed up database")),blue(os.path.basename(backup_path)),blue(db_destination),)
+            self.updateProgress(
+                mytxt,
+                importance = 1,
+                type = "info",
+                header = blue(" @@ "),
+                back = True
+            )
+
+        import bz2
+        try:
+            self.entropyTools.uncompress_file(backup_path, db_destination, bz2.BZ2File)
+        except:
+            if not silent:
+                self.entropyTools.printTraceback()
+            return False
+
+        if not silent:
+            mytxt = "%s: %s" % (darkgreen(_("Database restored successfully")),blue(db_destination),)
+            self.updateProgress(
+                mytxt,
+                importance = 1,
+                type = "info",
+                header = blue(" @@ "),
+                back = True
+            )
+        return True
+
+    def list_backedup_client_databases(self):
+        client_dbdir = os.path.dirname(etpConst['etpdatabaseclientfilepath'])
+        return [os.path.join(client_dbdir,x) for x in os.listdir(client_dbdir) \
+                    if x.startswith(etpConst['dbbackupprefix']) and \
+                    os.access(os.path.join(client_dbdir,x),os.R_OK)
+        ]
 
     '''
        Cache stuff :: begin
@@ -30169,7 +30276,6 @@ class EntropyDatabaseInterface:
         return -1
 
     def getIDPackageFromDownload(self, download_relative_path, endswith = False):
-        if branch == None: branch = self.db_branch
         if endswith:
             self.cursor.execute('SELECT baseinfo.idpackage FROM baseinfo,extrainfo WHERE extrainfo.download LIKE (?)', ("%"+download_relative_path,))
         else:

libraries/entropyConstants.py

@@ -626,6 +626,7 @@ def const_defaultSettings(rootdir):
         'etpdatabaseclientdir': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR,
         'etpdatabaseclientfilepath': ETP_DIR+ETP_CLIENT_REPO_DIR+ETP_DBDIR+"/"+ETP_DBCLIENTFILE, # path to equo.db - client side database file
         'dbnamerepoprefix': "repo_", # prefix of the name of self.dbname in EntropyDatabaseInterface class for the repositories
+        'dbbackupprefix': 'etp_backup_', # prefix of database backups
 
         'etpapi': etpSys['api'], # Entropy database API revision
         'currentarch': etpSys['arch'], # contains the current running architecture

libraries/entropyTools.py

@@ -445,6 +445,21 @@ def uncompress_file(file_path, destination_path, opener):
     f_out.close()
     f_in.close()
 
+def compress_file(file_path, destination_path, opener, compress_level = None):
+    f_in = open(file_path,"rb")
+    if compress_level != None:
+        f_out = opener(destination_path,"wb",compresslevel = compress_level)
+    else:
+        f_out = opener(destination_path,"wb")
+    data = f_in.read(8192)
+    while data:
+        f_out.write(data)
+        data = f_in.read(8192)
+    if hasattr(f_out,'flush'):
+        f_out.flush()
+    f_out.close()
+    f_in.close()
+
 def unpackGzip(gzipfilepath):
     import gzip
     filepath = gzipfilepath[:-3] # remove .gz