diff --git a/TODO b/TODO index e8ad4536d..c24e32958 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,8 @@ TODO list: - reagent: complete smartapps section CLIENT: - - implement logTools into new Equo functions (especially messages) --> messages should go into (install/remove packages) equo.log + - on-disk/static cache for removal functions? + - handle libraries reverse lookup during install to assure a consistent system - add multithreading support - during removal/add handle (trigger functions): - complete preremove() @@ -9,7 +10,7 @@ TODO list: - complete preinstall() - complete postinstall() - new kernel? grub configuration at your service (add a trigger) - - add kbuildsycoca trigger + - add kbuildsycoca trigger (using eclasses info) - if something is installed in /boot and this is a not mounted mountpoint, mount it (preinstall trigger) - if something is installed in /usr/lib/xorg/modules trigger eselect - remove TAGS and substitute them with forced SLOT setting (just keep it for the package filename) @@ -19,7 +20,8 @@ TODO list: - add scripts directory that contains pre/post-install-remove scripts that is synced using activator - implement installation messages infrastructure (where are collected all installation messages) - add support for new atoms layout (cpv:SLOT) - - maybe on-disk cache for removal functions? + - on-disk/static cache for removal functions? + - optimize database interface Project Status: - entropy: will handle all the three tools below diff --git a/client/equoTools.py b/client/equoTools.py index 3e65236f5..bc62d83ce 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -1076,11 +1076,13 @@ def fetchFile(url, digest = False): def removePackage(infoDict): - + atom = infoDict['removeatom'] content = infoDict['removecontent'] removeidpackage = infoDict['removeidpackage'] + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package: "+str(atom)) + # load content cache if found empty if etpConst['collisionprotect'] > 0: if (not contentCache): @@ -1098,6 +1100,7 @@ def removePackage(infoDict): # Handle gentoo database if (etpConst['gentoo-compat']): gentooAtom = dep_striptag(atom) # FIXME: tags will be removed + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing package from Gentoo database: "+str(gentooAtom)) removePackageFromGentooDatabase(gentooAtom) # load CONFIG_PROTECT and its mask - client database at this point has been surely opened, so our dicts are already filled @@ -1110,7 +1113,8 @@ def removePackage(infoDict): if etpConst['collisionprotect'] > 0: if file in contentCache: print_warning(darkred(" ## ")+red("Collision found during remove for ")+file.encode(sys.getfilesystemencoding())+" - cannot overwrite") - continue + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"Collision found during remove for "+file.encode(sys.getfilesystemencoding())+" - cannot overwrite") + continue try: del contentCache[file] except: @@ -1140,6 +1144,7 @@ def removePackage(infoDict): pass # some filenames are buggy encoded if (protected): + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"[remove] Protecting config file: "+str(file)) print_warning(darkred(" ## ")+red("[remove] Protecting config file: ")+str(file)) else: try: @@ -1170,7 +1175,9 @@ def installPackage(infoDict): clientDbconn = openClientDatabase() package = infoDict['download'] - + + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing package: "+str(infoDict['atom'])) + # load content cache if found empty if etpConst['collisionprotect'] > 0: if (not contentCache): @@ -1219,6 +1226,7 @@ def installPackage(infoDict): #print os.readlink(rootdir) os.remove(rootdir) elif os.path.isfile(rootdir): # weird + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! "+str(rootdir)+" is a file when it should be a directory !! Removing in 10 seconds...") print_warning(red(" *** ")+bold(rootdir)+red(" is a file when it should be a directory !! Removing in 10 seconds...")) time.sleep(10) os.remove(rootdir) @@ -1237,6 +1245,8 @@ def installPackage(infoDict): if etpConst['collisionprotect'] > 1: if tofile in contentCache: print_warning(darkred(" ## ")+red("Collision found during install for ")+file.encode(sys.getfilesystemencoding())+" - cannot overwrite") + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"WARNING!!! Collision found during install for "+file.encode(sys.getfilesystemencoding())+" - cannot overwrite") + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[collision] Protecting config file: "+str(tofile)) print_warning(darkred(" ## ")+red("[collision] Protecting config file: ")+str(tofile)) continue @@ -1282,6 +1292,7 @@ def installPackage(infoDict): # request new tofile then if (protected): + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Protecting config file: "+str(tofile)) print_warning(darkred(" ## ")+red("Protecting config file: ")+str(tofile)) tofile = allocateMaskedFile(tofile) @@ -1297,6 +1308,7 @@ def installPackage(infoDict): try: packageContent.append(tofile.encode("utf-8")) except: + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Cannot convert filename into UTF-8, skipping: "+str(tofile)) print_warning(darkred(" ## ")+red("Cannot convert filename into UTF-8, skipping ")+str(tofile)) pass except IOError,(errno,strerror): @@ -1328,8 +1340,10 @@ def installPackage(infoDict): # remove old files and gentoo stuff if (infoDict['removeidpackage'] != -1): # doing a diff removal + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Remove old package: "+str(infoDict['removeatom'])) infoDict['removeidpackage'] = -1 # disabling database removal if (etpConst['gentoo-compat']): + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Removing Gentoo database entry for "+str(infoDict['removeatom'])) print_info(red(" ## ")+blue("Cleaning old package files...")+" ## w/Gentoo compatibility") else: print_info(red(" ## ")+blue("Cleaning old package files...")) @@ -1338,6 +1352,7 @@ def installPackage(infoDict): closeClientDatabase(clientDbconn) if (etpConst['gentoo-compat']): + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"Installing new Gentoo database entry: "+str(infoDict['atom'])) rc = installPackageIntoGentooDatabase(infoDict,pkgpath) if (rc >= 0): shutil.rmtree(unpackDir,True) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index d7d034565..9901d6d07 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -83,6 +83,7 @@ def database(options): dbconn = etpDatabase(readOnly = False, noUpload = True) dbconn.initializeDatabase() + #print revisionsMatch # sync packages directory activatorTools.packages(["sync","--ask"]) @@ -806,6 +807,8 @@ class etpDatabase: if (self.packagesRemoved): self.cleanupUseflags() self.cleanupSources() + self.cleanupEclasses() + self.cleanupNeeded() self.cleanupDependencies() dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"closeDB: closing database opened in read/write.") @@ -1051,6 +1054,48 @@ class etpDatabase: etpData['disksize'], ) ) + + # eclasses table + for var in etpData['eclasses']: + + try: + idclass = self.isEclassAvailable(var) + except: + self.createEclassesTable() + idclass = self.isEclassAvailable(var) + + if (idclass == -1): + # create eclass + idclass = self.addEclass(var) + + self.cursor.execute( + 'INSERT into eclasses VALUES ' + '(?,?)' + , ( idpackage, + idclass, + ) + ) + + # needed table + for var in etpData['needed']: + + try: + idneeded = self.isNeededAvailable(var) + except: + self.createNeededTable() + idneeded = self.isNeededAvailable(var) + + if (idclass == -1): + # create eclass + idneeded = self.addNeeded(var) + + self.cursor.execute( + 'INSERT into needed VALUES ' + '(?,?)' + , ( idpackage, + idneeded, + ) + ) # dependencies, a list for dep in etpData['dependencies']: @@ -1110,7 +1155,11 @@ class etpDatabase: ) # create new protect if it doesn't exist - idprotect = self.isProtectAvailable(etpData['config_protect']) + try: + idprotect = self.isProtectAvailable(etpData['config_protect']) + except: + self.createProtectTable() + idprotect = self.isProtectAvailable(etpData['config_protect']) if (idprotect == -1): # create category idprotect = self.addProtect(etpData['config_protect']) @@ -1326,15 +1375,20 @@ class etpDatabase: self.cursor.execute('DELETE FROM keywords WHERE idpackage = '+idpackage) # binkeywords self.cursor.execute('DELETE FROM binkeywords WHERE idpackage = '+idpackage) + + # + # WARNING: exception won't be handled anymore with 1.0 + # + try: # messages self.cursor.execute('DELETE FROM messages WHERE idpackage = '+idpackage) except: - pass # FIXME: temp workaround + pass # systempackage self.cursor.execute('DELETE FROM systempackages WHERE idpackage = '+idpackage) try: - # cpunter + # counter self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage) except: pass @@ -1343,6 +1397,16 @@ class etpDatabase: self.cursor.execute('DELETE FROM sizes WHERE idpackage = '+idpackage) except: pass + try: + # eclasses + self.cursor.execute('DELETE FROM eclasses WHERE idpackage = '+idpackage) + except: + pass + try: + # needed + self.cursor.execute('DELETE FROM needed WHERE idpackage = '+idpackage) + except: + pass # Remove from installedtable if exists self.removePackageFromInstalledTable(idpackage) @@ -1440,6 +1504,30 @@ class etpDatabase: return use raise Exception, "I tried to insert a useflag but then, fetching it returned -1. There's something broken." + def addEclass(self,eclass): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addEclass: adding Eclass -> "+str(eclass)) + self.cursor.execute( + 'INSERT into eclassesreference VALUES ' + '(NULL,?)', (eclass,) + ) + # get info about inserted value and return + myclass = self.isEclassAvailable(eclass) + if myclass != -1: + return myclass + raise Exception, "I tried to insert an eclass but then, fetching it returned -1. There's something broken." + + def addNeeded(self,needed): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addNeeded: adding needed library -> "+str(needed)) + self.cursor.execute( + 'INSERT into neededreference VALUES ' + '(NULL,?)', (needed,) + ) + # get info about inserted value and return + myneeded = self.isNeededAvailable(needed) + if myneeded != -1: + return myneeded + raise Exception, "I tried to insert a needed library but then, fetching it returned -1. There's something broken." + def addLicense(self,license): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"addLicense: adding License -> "+str(license)) self.cursor.execute( @@ -1507,6 +1595,44 @@ class etpDatabase: for row in self.cursor: x = row # really necessary ? + def cleanupEclasses(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"cleanupEclasses: called.") + self.cursor.execute('SELECT idclass FROM eclassesreference') + idclasses = set([]) + for row in self.cursor: + idclasses.add(row[0]) + # now parse them into eclasses table + orphanedClasses = idclasses.copy() + for idclass in idclasses: + self.cursor.execute('SELECT idclass FROM eclasses WHERE idclass = '+str(idclass)) + for row in self.cursor: + orphanedClasses.remove(row[0]) + break + # now we have orphans that can be removed safely + for idoclass in orphanedClasses: + self.cursor.execute('DELETE FROM eclassesreference WHERE idclass = '+str(idoclass)) + for row in self.cursor: + x = row # really necessary ? + + def cleanupNeeded(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"cleanupNeeded: called.") + self.cursor.execute('SELECT idneeded FROM neededreference') + idneededs = set([]) + for row in self.cursor: + idneededs.add(row[0]) + # now parse them into needed table + orphanedNeededs = idneededs.copy() + for idneeded in idneededs: + self.cursor.execute('SELECT idneeded FROM needed WHERE idneeded = '+str(idneeded)) + for row in self.cursor: + orphanedNeededs.remove(row[0]) + break + # now we have orphans that can be removed safely + for idoneeded in orphanedNeededs: + self.cursor.execute('DELETE FROM neededreference WHERE idneeded = '+str(idoneeded)) + for row in self.cursor: + x = row # really necessary ? + def cleanupDependencies(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"cleanupDependencies: called.") self.cursor.execute('SELECT iddependency FROM dependenciesreference') @@ -1613,6 +1739,7 @@ class etpDatabase: data['homepage'] = self.retrieveHomepage(idpackage) data['useflags'] = self.retrieveUseflags(idpackage) data['license'] = self.retrieveLicense(idpackage) + data['eclasses'] = self.retrieveEclasses(idpackage) data['keywords'] = self.retrieveKeywords(idpackage) data['binkeywords'] = self.retrieveBinKeywords(idpackage) @@ -2054,8 +2181,64 @@ class etpDatabase: self.databaseCache[int(idpackage)]['retrieveUseflags'] = flags return flags + def retrieveEclasses(self, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveEclasses: retrieving eclasses for package ID "+str(idpackage)) + + ''' caching ''' + if (self.xcache): + cached = self.databaseCache.get(int(idpackage), None) + if cached: + rslt = self.databaseCache[int(idpackage)].get('retrieveEclasses',None) + if rslt: + return rslt + else: + self.databaseCache[int(idpackage)] = {} + + self.cursor.execute('SELECT "idclass" FROM eclasses WHERE idpackage = "'+str(idpackage)+'"') + idclasses = [] + for row in self.cursor: + idclasses.append(row[0]) + classes = [] + for idclass in idclasses: + self.cursor.execute('SELECT "classname" FROM eclassesreference WHERE idclass = "'+str(idclass)+'"') + for row in self.cursor: + classes.append(row[0]) + + ''' caching ''' + if (self.xcache): + self.databaseCache[int(idpackage)]['retrieveEclasses'] = classes + return classes + + def retrieveNeeded(self, idpackage): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveNeeded: retrieving needed libraries for package ID "+str(idpackage)) + + ''' caching ''' + if (self.xcache): + cached = self.databaseCache.get(int(idpackage), None) + if cached: + rslt = self.databaseCache[int(idpackage)].get('retrieveNeeded',None) + if rslt: + return rslt + else: + self.databaseCache[int(idpackage)] = {} + + self.cursor.execute('SELECT "idneeded" FROM needed WHERE idpackage = "'+str(idpackage)+'"') + idneededs = [] + for row in self.cursor: + idneededs.append(row[0]) + needed = [] + for idneeded in idneededs: + self.cursor.execute('SELECT "library" FROM neededreference WHERE idneeded = "'+str(idneeded)+'"') + for row in self.cursor: + needed.append(row[0]) + + ''' caching ''' + if (self.xcache): + self.databaseCache[int(idpackage)]['retrieveNeeded'] = needed + return needed + def retrieveConflicts(self, idpackage): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveConflicts: retrieving Conflicts for package ID "+str(idpackage)) + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveEclasses: retrieving Conflicts for package ID "+str(idpackage)) ''' caching ''' if (self.xcache): @@ -2615,6 +2798,30 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isUseflagAvailable: "+useflag+" available.") return result + def isEclassAvailable(self,eclass): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isEclassAvailable: called.") + result = -1 + self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = "'+eclass+'"') + for row in self.cursor: + result = row[0] + if result == -1: + dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isEclassAvailable: "+eclass+" not available.") + return result + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isEclassAvailable: "+eclass+" available.") + return result + + def isNeededAvailable(self,needed): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isNeededAvailable: called.") + result = -1 + self.cursor.execute('SELECT idneeded FROM neededreference WHERE needed = "'+needed+'"') + for row in self.cursor: + result = row[0] + if result == -1: + dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isNeededAvailable: "+needed+" not available.") + return result + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isNeededAvailable: "+needed+" available.") + return result + def isCounterAvailable(self,counter): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isCounterAvailable: called.") result = False @@ -3119,12 +3326,42 @@ class etpDatabase: break return sane + # + # FIXME: remove these when 1.0 will be out + # + def createSizesTable(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createSizesTable: called.") self.cursor.execute('DROP TABLE IF EXISTS sizes;') self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );') self.commitChanges() + def createEclassesTable(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createEclassesTable: called.") + self.cursor.execute('DROP TABLE IF EXISTS eclasses;') + self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;') + self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );') + self.cursor.execute('CREATE TABLE eclassesreference ( idclass INTEGER PRIMARY KEY, classname VARCHAR );') + self.commitChanges() + + def createNeededTable(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createNeededTable: called.") + self.cursor.execute('DROP TABLE IF EXISTS needed;') + self.cursor.execute('DROP TABLE IF EXISTS neededreference;') + self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER );') + self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY, library VARCHAR );') + self.commitChanges() + + def createProtectTable(self): + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createProtectTable: called.") + self.cursor.execute('DROP TABLE IF EXISTS configprotect;') + self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;') + self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;') + self.cursor.execute('CREATE TABLE configprotect ( idpackage INTEGER, idprotect INTEGER );') + self.cursor.execute('CREATE TABLE configprotectmask ( idpackage INTEGER, idprotect INTEGER );') + self.cursor.execute('CREATE TABLE configprotectreference ( idprotect INTEGER PRIMARY KEY, protect VARCHAR );') + self.commitChanges() + def createInstalledTable(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"createInstalledTable: called.") self.cursor.execute('DROP TABLE IF EXISTS installedtable;') diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index e6ed1476b..59c12e55e 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -62,7 +62,9 @@ etpData = { 'config_protect_mask': u"", # list of directories that contain files that should be overwritten 'disksize': u"", # size on the hard disk in bytes (integer) 'counter': u"", # aka. COUNTER file - 'messages': u"" # elog content from portage + 'messages': u"", # elog content from portage + 'eclasses': u"", # eclasses used by the ebuild + 'needed': u"" # runtime libraries needed by the package } # Entropy database SQL initialization Schema and data structure @@ -99,6 +101,10 @@ DROP TABLE IF EXISTS dependstable; DROP TABLE IF EXISTS sizes; DROP TABLE IF EXISTS messages; DROP TABLE IF EXISTS counters; +DROP TABLE IF EXISTS eclasses; +DROP TABLE IF EXISTS eclassesreference; +DROP TABLE IF EXISTS needed; +DROP TABLE IF EXISTS neededreference; """ etpSQLInit = """ @@ -249,6 +255,26 @@ CREATE TABLE counters ( idpackage INTEGER ); +CREATE TABLE eclasses ( + idpackage INTEGER, + idclass INTEGER +); + +CREATE TABLE eclassesreference ( + idclass INTEGER PRIMARY KEY, + classname VARCHAR +); + +CREATE TABLE needed ( + idpackage INTEGER, + idneeded INTEGER +); + +CREATE TABLE neededreference ( + idneeded INTEGER PRIMARY KEY, + library VARCHAR +); + """ # ^^ add dependstable? @@ -596,6 +622,7 @@ dbDEPEND = "DEPEND" dbRDEPEND = "RDEPEND" dbPDEPEND = "PDEPEND" dbNEEDED = "NEEDED" +dbINHERITED = "INHERITED" dbOR = "|or|" dbKEYWORDS = "KEYWORDS" dbCONTENTS = "CONTENTS" diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 63c02403e..400512da3 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -358,6 +358,36 @@ def extractPkgData(package, etpBranch = etpConst['branch']): except IOError: pass + print_info(yellow(" * ")+red(info_package+"Getting package eclasses information..."),back = True) + # fill eclasses list + etpData['eclasses'] = [] + try: + f = open(tbz2TmpDir+dbINHERITED,"r") + etpData['eclasses'] = f.readline().strip().split() + f.close() + except IOError: + pass + + print_info(yellow(" * ")+red(info_package+"Getting package needed libraries information..."),back = True) + # fill needed list + etpData['needed'] = set() + try: + f = open(tbz2TmpDir+dbNEEDED,"r") + lines = f.readlines() + f.close() + for line in lines: + line = line.strip() + if line: + needed = line.split() + if len(needed) == 2: + libs = needed[1].split(",") + for lib in libs: + if (lib.find(".so") != -1): + etpData['needed'].add(lib) + except IOError: + pass + etpData['needed'] = list(etpData['needed']) + print_info(yellow(" * ")+red(info_package+"Getting package content..."),back = True) # dbCONTENTS etpData['content'] = []