From 7e4432b4801ac9f41723393477a38b1fdf65ed80 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Wed, 6 Feb 2008 00:09:14 +0000 Subject: [PATCH] added dependencies rescan tool for reagent git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1143 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- libraries/databaseTools.py | 145 ++++++++++++++++++------------------- libraries/portageTools.py | 13 ++-- libraries/reagentTools.py | 86 +++++++++++++++++++++- server/reagent | 3 + 4 files changed, 165 insertions(+), 82 deletions(-) diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 5a303357e..752ea59b9 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -918,20 +918,7 @@ class etpDatabase(TextInterface): ) # content, a list - if not self.doesColumnInTableExist("content","type"): - self.createContentTypeColumn() - for xfile in etpData['content']: - contenttype = etpData['content'][xfile] - if type(xfile) is unicode: - xfile = xfile.encode('raw_unicode_escape') - self.cursor.execute( - 'INSERT into content VALUES ' - '(?,?,?)' - , ( idpackage, - xfile, - contenttype, - ) - ) + self.insertContent(idpackage,etpData['content']) etpData['counter'] = int(etpData['counter']) # cast to integer if etpData['counter'] != -1 and not (etpData['injected']): @@ -1062,54 +1049,32 @@ class etpDatabase(TextInterface): ) # compile messages - try: - for message in etpData['messages']: - self.cursor.execute( - 'INSERT into messages VALUES ' - '(?,?)' - , ( idpackage, - message, - ) - ) - except: - # FIXME: temp workaround, create messages table - self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);") - for message in etpData['messages']: - self.cursor.execute( - 'INSERT into messages VALUES ' - '(?,?)' - , ( idpackage, - message, - ) - ) + if not self.doesTableExist("messages"): + self.createMessagesTable() + for message in etpData['messages']: + self.cursor.execute( + 'INSERT into messages VALUES ' + '(?,?)' + , ( idpackage, + message, + ) + ) - try: - # is it a system package? - if etpData['systempackage']: - self.cursor.execute( - 'INSERT into systempackages VALUES ' - '(?)' - , ( idpackage, - ) - ) - except: - # FIXME: temp workaround, create systempackages table + if not self.doesTableExist("systempackages"): self.createSystemPackagesTable() - # is it a system package? - if etpData['systempackage']: - self.cursor.execute( - 'INSERT into systempackages VALUES ' - '(?)' - , ( idpackage, - ) - ) + # is it a system package? + if etpData['systempackage']: + self.cursor.execute( + 'INSERT into systempackages VALUES ' + '(?)' + , ( idpackage, + ) + ) - # create new protect if it doesn't exist - try: - idprotect = self.isProtectAvailable(etpData['config_protect']) - except: + if not self.doesTableExist("configprotect"): self.createProtectTable() - idprotect = self.isProtectAvailable(etpData['config_protect']) + # create new protect if it doesn't exist + idprotect = self.isProtectAvailable(etpData['config_protect']) if (idprotect == -1): # create category idprotect = self.addProtect(etpData['config_protect']) @@ -1608,6 +1573,28 @@ class etpDatabase(TextInterface): ) ) + def removeContent(self, idpackage): + self.checkReadOnly() + self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,)) + self.commitChanges() + + def insertContent(self, idpackage, content): + self.checkReadOnly() + if not self.doesColumnInTableExist("content","type"): + self.createContentTypeColumn() + for xfile in content: + contenttype = content[xfile] + if type(xfile) is unicode: + xfile = xfile.encode('raw_unicode_escape') + self.cursor.execute( + 'INSERT into content VALUES ' + '(?,?,?)' + , ( idpackage, + xfile, + contenttype, + ) + ) + def insertCounter(self, idpackage, counter): self.checkReadOnly() self.cursor.execute('DELETE FROM counters WHERE counter = (?) OR idpackage = (?)', (counter,idpackage,)) @@ -2741,26 +2728,18 @@ class etpDatabase(TextInterface): def isSystemPackage(self,idpackage): - cache = self.fetchInfoCache(idpackage,'isSystemPackage') - if cache != None: return cache - - try: - self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) - except: # FIXME: remove this for 1.0 + if not self.doesTableExist("systempackages"): try: self.createSystemPackagesTable() - except: - # readonly database? + except exceptionTools.OperationNotPermitted: return False - self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) - - result = self.cursor.fetchone() - rslt = False - if result: - rslt = True - self.storeInfoCache(idpackage,'isSystemPackage',rslt) - return rslt + self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,)) + + result = self.cursor.fetchone() + if result: + return True + return False def isInjected(self,idpackage): @@ -3519,21 +3498,25 @@ class etpDatabase(TextInterface): # def createTreeupdatesTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS treeupdates;') self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );') self.commitChanges() def createTreeupdatesactionsTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS treeupdatesactions;') self.cursor.execute('CREATE TABLE treeupdatesactions ( idupdate INTEGER PRIMARY KEY, repository VARCHAR, command VARCHAR, branch VARCHAR );') self.commitChanges() def createSizesTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS sizes;') self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );') self.commitChanges() def createTreeupdatesactionsBranchColumn(self): + self.checkReadOnly() try: # if database disk image is malformed, won't raise exception here self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN branch VARCHAR;') self.cursor.execute('UPDATE treeupdatesactions SET branch = (?)', (str(etpConst['branch']),)) @@ -3542,6 +3525,7 @@ class etpDatabase(TextInterface): self.commitChanges() def createContentTypeColumn(self): + self.checkReadOnly() try: # if database disk image is malformed, won't raise exception here self.cursor.execute('ALTER TABLE content ADD COLUMN type VARCHAR;') self.cursor.execute('UPDATE content SET type = "0"') @@ -3550,15 +3534,23 @@ class etpDatabase(TextInterface): self.commitChanges() def createTriggerTable(self): + self.checkReadOnly() self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );') self.commitChanges() def createTriggerColumn(self): + self.checkReadOnly() self.cursor.execute('ALTER TABLE baseinfo ADD COLUMN trigger INTEGER;') self.cursor.execute('UPDATE baseinfo SET trigger = 0') self.commitChanges() + def createMessagesTable(self): + self.checkReadOnly() + self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);") + self.commitChanges() + def createEclassesTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS eclasses;') self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;') self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );') @@ -3566,19 +3558,23 @@ class etpDatabase(TextInterface): self.commitChanges() def createNeededTable(self): + self.checkReadOnly() self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER );') self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY, library VARCHAR );') self.commitChanges() def createSystemPackagesTable(self): + self.checkReadOnly() self.cursor.execute('CREATE TABLE systempackages ( idpackage INTEGER PRIMARY KEY );') self.commitChanges() def createInjectedTable(self): + self.checkReadOnly() self.cursor.execute('CREATE TABLE injected ( idpackage INTEGER PRIMARY KEY );') self.commitChanges() def createProtectTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS configprotect;') self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;') self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;') @@ -3588,6 +3584,7 @@ class etpDatabase(TextInterface): self.commitChanges() def createInstalledTable(self): + self.checkReadOnly() self.cursor.execute('DROP TABLE IF EXISTS installedtable;') self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR );') self.commitChanges() diff --git a/libraries/portageTools.py b/libraries/portageTools.py index 24246884b..272eb97c6 100644 --- a/libraries/portageTools.py +++ b/libraries/portageTools.py @@ -422,6 +422,12 @@ def dep_and_select(and_list): else: newlist.append(x) + # now verify if all are satisfied + for x in newlist: + match = getInstalledAtom(x) + if match == None: + return [] + return newlist def dep_or_select(or_list): @@ -436,11 +442,8 @@ def dep_or_select(or_list): do_skip = True elif isinstance(x, list): # and x = dep_and_select(x) - for y in x: # need to match all - match = getInstalledAtom(y) - if match == None: - # skip, can't match all - continue + if not x: + continue # found return x else: diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index cb7d34397..cf3dc3675 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -1049,15 +1049,17 @@ def database(options): # FIXME: this function does not update metadata inside tbz2 # Please implement this !!! + # it would just have to repackage the bins elif (options[0] == "depsregen"): # first of all, sync and lock database if not databaseRequestJustScan: print_info(green(" * ")+red("Remember to flush all the pending uploads. It's always better having a fully synchronized system.")) time.sleep(5) dbconn = Entropy.databaseTools.openServerDatabase(readOnly = False, noUpload = True) + print_info(green(" * ")+red("Starting to regenerate package dependencies in repository")) else: dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True) - print_info(green(" * ")+red("Starting to regenerate package dependencies in repository")) + print_info(green(" * ")+red("Starting to scan and compare package dependencies in repository")) idpackages = dbconn.listAllIdpackages() maxcount = str(len(idpackages)) count = 0 @@ -1112,8 +1114,8 @@ def database(options): dbconn.insertDependencies(idpackage, found_deps) # done ! - # XXX update depends - dependsTableInitialize(dbconn, False) + if not databaseRequestJustScan: + dependsTableInitialize(dbconn, False) print_info(blue(" * Statistics:")) print_info(brown(" Number of checked packages:\t\t")+maxcount) @@ -1122,6 +1124,84 @@ def database(options): print_info(red(" Number of not found packages:\t\t")+str(stats['not_found'])) dbconn.closeDB() + # FIXME: this function does not update metadata inside tbz2 + # Please implement this !!! + # it would just have to repackage the bins + elif (options[0] == "contentregen"): + # first of all, sync and lock database + if not databaseRequestJustScan: + print_info(green(" * ")+red("Remember to flush all the pending uploads. It's always better having a fully synchronized system.")) + time.sleep(5) + dbconn = Entropy.databaseTools.openServerDatabase(readOnly = False, noUpload = True) + print_info(green(" * ")+red("Starting to regenerate package content metadata in repository")) + else: + dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True) + print_info(green(" * ")+red("Starting to scan and compare package content metadata in repository")) + idpackages = dbconn.listAllIdpackages() + maxcount = str(len(idpackages)) + count = 0 + stats = {} + stats['updated'] = 0 + stats['not_found'] = 0 + stats['bad_digest'] = 0 + for idpackage in idpackages: + count += 1 + atom = dbconn.retrieveAtom(idpackage) + branch = dbconn.retrieveBranch(idpackage) + download = dbconn.retrieveDownloadURL(idpackage) + checksum = dbconn.retrieveDigest(idpackage) + # start scanning + countstring = green(" * ")+red("(")+blue(str(count))+"/"+darkgreen(maxcount)+red(") ")+red("[")+blue(branch)+red("] ") + print_info(countstring+red("Scanning ")+brown(atom), back = True) + download = os.path.join(etpConst['entropyworkdir'],download) + download_upload = os.path.join(etpConst['packagessuploaddir'],branch) + download_upload = os.path.join(download_upload,os.path.basename(download)) + if not os.path.isfile(download) and not os.path.isfile(download_upload): + print_warning(countstring+red(" Package Error:")) + print_warning(countstring+" "+blue(download)+" not found!") + print_warning(countstring+" "+blue(download_upload)+" not found!") + stats['not_found'] += 1 + continue + if os.path.isfile(download_upload): + download = download_upload + # verify checksum before starting + if not databaseRequestNoChecksum: + print_info(countstring+red("Verifying checksum ")+brown(atom), back = True) + status = Entropy.entropyTools.compareMd5(download,checksum) + if not status: + print_warning(countstring+red(" Checksum Error:")) + print_warning(countstring+" "+blue(download)+" is corrupted!") + stats['bad_digest'] += 1 + continue + # rescan package + metadata = Entropy.entropyTools.extractPkgData(download, silent = True) + db_content = dbconn.retrieveContent(idpackage) + found_content = set([x for x in metadata['content']]) + del metadata + if db_content != found_content: + if databaseRequestJustScan: + # show difference + print_warning(countstring+red(" Content difference for ")+brown(atom)+(":")) + print_warning(countstring+" repository entries: "+str(len(db_content))) + print_warning(countstring+" scanned entries: "+str(len(found_content))) + else: + stats['updated'] += 1 + print_info(countstring+red("Updating content metadata for ")+brown(atom)) + dbconn.removeContent(idpackage) + dbconn.insertContent(idpackage, found_content) + + # done ! + if not databaseRequestJustScan: + dependsTableInitialize(dbconn, False) + + print_info(blue(" * Statistics:")) + print_info(brown(" Number of checked packages:\t\t")+maxcount) + print_info(green(" Number of updated packages:\t\t")+str(stats['updated'])) + print_info(red(" Number of broken packages:\t\t")+str(stats['bad_digest'])) + print_info(red(" Number of not found packages:\t\t")+str(stats['not_found'])) + dbconn.closeDB() + + # query tools elif (options[0] == "query"): diff --git a/server/reagent b/server/reagent index f282eee28..37290e2e5 100644 --- a/server/reagent +++ b/server/reagent @@ -76,6 +76,9 @@ def print_help(): print_info(" \t\t"+green("depsregen")+"\t\t\t Recalculate and update package dependencies in repository") print_info(" \t\t\t"+red("--justscan")+"\t Just show differences between results") print_info(" \t\t\t"+red("--nochecksum")+"\t Disable checksum verification") + print_info(" \t\t"+green("contentregen")+"\t\t\t Recalculate and update package content metadata in repository") + print_info(" \t\t\t"+red("--justscan")+"\t Just show differences between results") + print_info(" \t\t\t"+red("--nochecksum")+"\t Disable checksum verification") print_info(" \t"+green("spm")+brown("\t\t Source Package Manager tool manager")) print_info(" \t\t"+green("compile")+"\t\t\t Start SPM and compile something")