sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added dependencies rescan tool for reagent

Browse Source 
git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1143 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2008-02-06 00:09:14 +00:00
parent 78091dd7dd
commit 7e4432b480
4 changed files with 165 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/databaseTools.py
+71 -74
View File
@@ -918,20 +918,7 @@ class etpDatabase(TextInterface):
)
# content, a list
if not self.doesColumnInTableExist("content","type"):
self.createContentTypeColumn()
for xfile in etpData['content']:
contenttype = etpData['content'][xfile]
if type(xfile) is unicode:
xfile = xfile.encode('raw_unicode_escape')
self.cursor.execute(
'INSERT into content VALUES '
'(?,?,?)'
, ( idpackage,
xfile,
contenttype,
)
)
self.insertContent(idpackage,etpData['content'])
etpData['counter'] = int(etpData['counter']) # cast to integer
if etpData['counter'] != -1 and not (etpData['injected']):
@@ -1062,54 +1049,32 @@ class etpDatabase(TextInterface):
)
# compile messages
try:
for message in etpData['messages']:
self.cursor.execute(
'INSERT into messages VALUES '
'(?,?)'
, ( idpackage,
message,
)
)
except:
# FIXME: temp workaround, create messages table
self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);")
for message in etpData['messages']:
self.cursor.execute(
'INSERT into messages VALUES '
'(?,?)'
, ( idpackage,
message,
)
)
if not self.doesTableExist("messages"):
self.createMessagesTable()
for message in etpData['messages']:
self.cursor.execute(
'INSERT into messages VALUES '
'(?,?)'
, ( idpackage,
message,
)
)
try:
# is it a system package?
if etpData['systempackage']:
self.cursor.execute(
'INSERT into systempackages VALUES '
'(?)'
, ( idpackage,
)
)
except:
# FIXME: temp workaround, create systempackages table
if not self.doesTableExist("systempackages"):
self.createSystemPackagesTable()
# is it a system package?
if etpData['systempackage']:
self.cursor.execute(
'INSERT into systempackages VALUES '
'(?)'
, ( idpackage,
)
)
# is it a system package?
if etpData['systempackage']:
self.cursor.execute(
'INSERT into systempackages VALUES '
'(?)'
, ( idpackage,
)
)
# create new protect if it doesn't exist
try:
idprotect = self.isProtectAvailable(etpData['config_protect'])
except:
if not self.doesTableExist("configprotect"):
self.createProtectTable()
idprotect = self.isProtectAvailable(etpData['config_protect'])
# create new protect if it doesn't exist
idprotect = self.isProtectAvailable(etpData['config_protect'])
if (idprotect == -1):
# create category
idprotect = self.addProtect(etpData['config_protect'])
@@ -1608,6 +1573,28 @@ class etpDatabase(TextInterface):
)
)
def removeContent(self, idpackage):
self.checkReadOnly()
self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,))
self.commitChanges()
def insertContent(self, idpackage, content):
self.checkReadOnly()
if not self.doesColumnInTableExist("content","type"):
self.createContentTypeColumn()
for xfile in content:
contenttype = content[xfile]
if type(xfile) is unicode:
xfile = xfile.encode('raw_unicode_escape')
self.cursor.execute(
'INSERT into content VALUES '
'(?,?,?)'
, ( idpackage,
xfile,
contenttype,
)
)
def insertCounter(self, idpackage, counter):
self.checkReadOnly()
self.cursor.execute('DELETE FROM counters WHERE counter = (?) OR idpackage = (?)', (counter,idpackage,))
@@ -2741,26 +2728,18 @@ class etpDatabase(TextInterface):
def isSystemPackage(self,idpackage):
cache = self.fetchInfoCache(idpackage,'isSystemPackage')
if cache != None: return cache
try:
self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,))
except: # FIXME: remove this for 1.0
if not self.doesTableExist("systempackages"):
try:
self.createSystemPackagesTable()
except:
# readonly database?
except exceptionTools.OperationNotPermitted:
return False
self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,))
result = self.cursor.fetchone()
rslt = False
if result:
rslt = True
self.storeInfoCache(idpackage,'isSystemPackage',rslt)
return rslt
self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,))
result = self.cursor.fetchone()
if result:
return True
return False
def isInjected(self,idpackage):
@@ -3519,21 +3498,25 @@ class etpDatabase(TextInterface):
#
def createTreeupdatesTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS treeupdates;')
self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );')
self.commitChanges()
def createTreeupdatesactionsTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS treeupdatesactions;')
self.cursor.execute('CREATE TABLE treeupdatesactions ( idupdate INTEGER PRIMARY KEY, repository VARCHAR, command VARCHAR, branch VARCHAR );')
self.commitChanges()
def createSizesTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS sizes;')
self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );')
self.commitChanges()
def createTreeupdatesactionsBranchColumn(self):
self.checkReadOnly()
try: # if database disk image is malformed, won't raise exception here
self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN branch VARCHAR;')
self.cursor.execute('UPDATE treeupdatesactions SET branch = (?)', (str(etpConst['branch']),))
@@ -3542,6 +3525,7 @@ class etpDatabase(TextInterface):
self.commitChanges()
def createContentTypeColumn(self):
self.checkReadOnly()
try: # if database disk image is malformed, won't raise exception here
self.cursor.execute('ALTER TABLE content ADD COLUMN type VARCHAR;')
self.cursor.execute('UPDATE content SET type = "0"')
@@ -3550,15 +3534,23 @@ class etpDatabase(TextInterface):
self.commitChanges()
def createTriggerTable(self):
self.checkReadOnly()
self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );')
self.commitChanges()
def createTriggerColumn(self):
self.checkReadOnly()
self.cursor.execute('ALTER TABLE baseinfo ADD COLUMN trigger INTEGER;')
self.cursor.execute('UPDATE baseinfo SET trigger = 0')
self.commitChanges()
def createMessagesTable(self):
self.checkReadOnly()
self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);")
self.commitChanges()
def createEclassesTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS eclasses;')
self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;')
self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );')
@@ -3566,19 +3558,23 @@ class etpDatabase(TextInterface):
self.commitChanges()
def createNeededTable(self):
self.checkReadOnly()
self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER );')
self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY, library VARCHAR );')
self.commitChanges()
def createSystemPackagesTable(self):
self.checkReadOnly()
self.cursor.execute('CREATE TABLE systempackages ( idpackage INTEGER PRIMARY KEY );')
self.commitChanges()
def createInjectedTable(self):
self.checkReadOnly()
self.cursor.execute('CREATE TABLE injected ( idpackage INTEGER PRIMARY KEY );')
self.commitChanges()
def createProtectTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS configprotect;')
self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;')
self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;')
@@ -3588,6 +3584,7 @@ class etpDatabase(TextInterface):
self.commitChanges()
def createInstalledTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS installedtable;')
self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR );')
self.commitChanges()
libraries/portageTools.py
+8 -5
View File
@@ -422,6 +422,12 @@ def dep_and_select(and_list):
else:
newlist.append(x)
# now verify if all are satisfied
for x in newlist:
match = getInstalledAtom(x)
if match == None:
return []
return newlist
def dep_or_select(or_list):
@@ -436,11 +442,8 @@ def dep_or_select(or_list):
do_skip = True
elif isinstance(x, list): # and
x = dep_and_select(x)
for y in x: # need to match all
match = getInstalledAtom(y)
if match == None:
# skip, can't match all
continue
if not x:
continue
# found
return x
else:
libraries/reagentTools.py
+83 -3
View File
@@ -1049,15 +1049,17 @@ def database(options):
# FIXME: this function does not update metadata inside tbz2
# Please implement this !!!
# it would just have to repackage the bins
elif (options[0] == "depsregen"):
# first of all, sync and lock database
if not databaseRequestJustScan:
print_info(green(" * ")+red("Remember to flush all the pending uploads. It's always better having a fully synchronized system."))
time.sleep(5)
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = False, noUpload = True)
print_info(green(" * ")+red("Starting to regenerate package dependencies in repository"))
else:
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
print_info(green(" * ")+red("Starting to regenerate package dependencies in repository"))
print_info(green(" * ")+red("Starting to scan and compare package dependencies in repository"))
idpackages = dbconn.listAllIdpackages()
maxcount = str(len(idpackages))
count = 0
@@ -1112,8 +1114,8 @@ def database(options):
dbconn.insertDependencies(idpackage, found_deps)
# done !
# XXX update depends
dependsTableInitialize(dbconn, False)
if not databaseRequestJustScan:
dependsTableInitialize(dbconn, False)
print_info(blue(" * Statistics:"))
print_info(brown(" Number of checked packages:\t\t")+maxcount)
@@ -1122,6 +1124,84 @@ def database(options):
print_info(red(" Number of not found packages:\t\t")+str(stats['not_found']))
dbconn.closeDB()
# FIXME: this function does not update metadata inside tbz2
# Please implement this !!!
# it would just have to repackage the bins
elif (options[0] == "contentregen"):
# first of all, sync and lock database
if not databaseRequestJustScan:
print_info(green(" * ")+red("Remember to flush all the pending uploads. It's always better having a fully synchronized system."))
time.sleep(5)
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = False, noUpload = True)
print_info(green(" * ")+red("Starting to regenerate package content metadata in repository"))
else:
dbconn = Entropy.databaseTools.openServerDatabase(readOnly = True, noUpload = True)
print_info(green(" * ")+red("Starting to scan and compare package content metadata in repository"))
idpackages = dbconn.listAllIdpackages()
maxcount = str(len(idpackages))
count = 0
stats = {}
stats['updated'] = 0
stats['not_found'] = 0
stats['bad_digest'] = 0
for idpackage in idpackages:
count += 1
atom = dbconn.retrieveAtom(idpackage)
branch = dbconn.retrieveBranch(idpackage)
download = dbconn.retrieveDownloadURL(idpackage)
checksum = dbconn.retrieveDigest(idpackage)
# start scanning
countstring = green(" * ")+red("(")+blue(str(count))+"/"+darkgreen(maxcount)+red(") ")+red("[")+blue(branch)+red("] ")
print_info(countstring+red("Scanning ")+brown(atom), back = True)
download = os.path.join(etpConst['entropyworkdir'],download)
download_upload = os.path.join(etpConst['packagessuploaddir'],branch)
download_upload = os.path.join(download_upload,os.path.basename(download))
if not os.path.isfile(download) and not os.path.isfile(download_upload):
print_warning(countstring+red(" Package Error:"))
print_warning(countstring+" "+blue(download)+" not found!")
print_warning(countstring+" "+blue(download_upload)+" not found!")
stats['not_found'] += 1
continue
if os.path.isfile(download_upload):
download = download_upload
# verify checksum before starting
if not databaseRequestNoChecksum:
print_info(countstring+red("Verifying checksum ")+brown(atom), back = True)
status = Entropy.entropyTools.compareMd5(download,checksum)
if not status:
print_warning(countstring+red(" Checksum Error:"))
print_warning(countstring+" "+blue(download)+" is corrupted!")
stats['bad_digest'] += 1
continue
# rescan package
metadata = Entropy.entropyTools.extractPkgData(download, silent = True)
db_content = dbconn.retrieveContent(idpackage)
found_content = set([x for x in metadata['content']])
del metadata
if db_content != found_content:
if databaseRequestJustScan:
# show difference
print_warning(countstring+red(" Content difference for ")+brown(atom)+(":"))
print_warning(countstring+" repository entries: "+str(len(db_content)))
print_warning(countstring+" scanned entries: "+str(len(found_content)))
else:
stats['updated'] += 1
print_info(countstring+red("Updating content metadata for ")+brown(atom))
dbconn.removeContent(idpackage)
dbconn.insertContent(idpackage, found_content)
# done !
if not databaseRequestJustScan:
dependsTableInitialize(dbconn, False)
print_info(blue(" * Statistics:"))
print_info(brown(" Number of checked packages:\t\t")+maxcount)
print_info(green(" Number of updated packages:\t\t")+str(stats['updated']))
print_info(red(" Number of broken packages:\t\t")+str(stats['bad_digest']))
print_info(red(" Number of not found packages:\t\t")+str(stats['not_found']))
dbconn.closeDB()
# query tools
elif (options[0] == "query"):
server/reagent
+3
View File
@@ -76,6 +76,9 @@ def print_help():
print_info(" \t\t"+green("depsregen")+"\t\t\t Recalculate and update package dependencies in repository")
print_info(" \t\t\t"+red("--justscan")+"\t Just show differences between results")
print_info(" \t\t\t"+red("--nochecksum")+"\t Disable checksum verification")
print_info(" \t\t"+green("contentregen")+"\t\t\t Recalculate and update package content metadata in repository")
print_info(" \t\t\t"+red("--justscan")+"\t Just show differences between results")
print_info(" \t\t\t"+red("--nochecksum")+"\t Disable checksum verification")
print_info(" \t"+green("spm")+brown("\t\t Source Package Manager tool manager"))
print_info(" \t\t"+green("compile")+"\t\t\t Start SPM and compile something")