sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
Files
f9dfb3fbe6799dacf71b8fa54c85db78adee9171
entropy/libraries/databaseTools.py
lxnay f9dfb3fbe6 Entropy/ServerInterface: 
- change server.conf specs adding http handlers URL to it
- added multirepository support to all server-side functions (not yet done)


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1681 cd1c1023-2f26-0410-ae45-c471fc1f0318
2008-04-10 23:14:50 +00:00

4153 lines
167 KiB
Python
Raw Blame History

#!/usr/bin/python
'''
# DESCRIPTION:
# Entropy Database Interface
Copyright (C) 2007-2008 Fabio Erculiani
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
'''
from entropyConstants import *
from outputTools import *
import exceptionTools
Text = TextInterface()
try: # try with sqlite3 from python 2.5 - default one
from sqlite3 import dbapi2
except ImportError: # fallback to embedded pysqlite
try:
from pysqlite2 import dbapi2
except ImportError, e:
raise exceptionTools.SystemError("Entropy needs a working sqlite+pysqlite or Python compiled with sqlite support. Error: %s" % (str(e),))
import dumpTools
class etpDatabase:
import entropyTools
def __init__(self, readOnly = False, noUpload = False, dbFile = None, clientDatabase = False, xcache = False, dbname = etpConst['serverdbid'], indexing = True, OutputInterface = Text, ServiceInterface = None):
if dbFile == None:
raise exceptionTools.IncorrectParameter("IncorrectParameter: valid database path needed")
# setup output interface
self.OutputInterface = OutputInterface
self.updateProgress = self.OutputInterface.updateProgress
self.askQuestion = self.OutputInterface.askQuestion
# setup service interface
self.ServiceInterface = ServiceInterface
self.readOnly = readOnly
self.noUpload = noUpload
self.clientDatabase = clientDatabase
self.xcache = xcache
self.dbname = dbname
self.indexing = indexing
if not self.entropyTools.is_user_in_entropy_group(): # forcing since we won't have write access to db
self.indexing = False
# live systems don't like wasting RAM
if self.entropyTools.islive():
self.indexing = False
self.dbFile = dbFile
self.dbclosed = False
self.server_repo = None
if not self.clientDatabase:
self.server_repo = self.dbname[len(etpConst['serverdbid']):]
self.create_dbstatus_data()
# no caching for non root and server connections
if (self.dbname.startswith(etpConst['serverdbid'])) or (not self.entropyTools.is_user_in_entropy_group()):
self.xcache = False
self.live_cache = {}
# create connection
self.connection = dbapi2.connect(dbFile,timeout=300.0)
self.cursor = self.connection.cursor()
if not self.clientDatabase and not self.readOnly:
# server side is calling
# lock mirror remotely and ensure to have latest database revision
self.doServerDatabaseSyncLock(self.noUpload)
if os.access(self.dbFile,os.W_OK) and self.doesTableExist('baseinfo') and self.doesTableExist('extrainfo'):
if self.entropyTools.islive():
# check where's the file
if etpConst['systemroot']:
self.databaseStructureUpdates()
else:
self.databaseStructureUpdates()
def __del__(self):
if not self.dbclosed:
self.closeDB()
def create_dbstatus_data(self):
taint_file = self.ServiceInterface.get_local_database_taint_file(self.server_repo)
if not etpDbStatus.has_key(self.dbFile):
etpDbStatus[self.dbFile] = {}
etpDbStatus[self.dbFile]['tainted'] = False
etpDbStatus[self.dbFile]['bumped'] = False
if os.path.isfile(taint_file):
etpDbStatus[self.dbFile]['tainted'] = True
etpDbStatus[self.dbFile]['bumped'] = True
def doServerDatabaseSyncLock(self, noUpload):
# check if the database is locked locally
# self.server_repo
lock_file = self.ServiceInterface.MirrorsService.get_database_lockfile(self.server_repo)
if os.path.isfile(lock_file):
self.updateProgress(
red("Entropy database is already locked by you :-)"),
importance = 1,
type = "info",
header = red(" * ")
)
else:
# check if the database is locked REMOTELY
self.updateProgress(
red("Locking and Syncing Entropy database..."),
importance = 1,
type = "info",
header = red(" * "),
back = True
)
for uri in self.ServiceInterface.get_remote_mirrors(self.server_repo):
given_up = self.ServiceInterface.MirrorsService.mirror_lock_check(uri, repo = self.server_repo)
if given_up:
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
self.updateProgress(
darkgreen("Mirrors status table:"),
importance = 1,
type = "info",
header = brown(" * ")
)
dbstatus = self.ServiceInterface.MirrorsService.get_mirrors_lock(repo = self.server_repo)
for db in dbstatus:
db[1] = green("Unlocked")
if (db[1]):
db[1] = red("Locked")
db[2] = green("Unlocked")
if (db[2]):
db[2] = red("Locked")
crippled_uri = self.entropyTools.extractFTPHostFromUri(db[0])
self.updateProgress(
bold("%s: ")+red("[")+brown("DATABASE: %s")+red("] [")+brown("DOWNLOAD: %s")+red("]") % (crippled_uri,db[1],db[2],),
importance = 1,
type = "info",
header = "\t"
)
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: cannot lock mirror %s" % (crippled_uri,))
# if we arrive here, it is because all the mirrors are unlocked
self.ServiceInterface.MirrorsService.lock_mirrors(True, repo = self.server_repo)
self.ServiceInterface.MirrorsService.sync_databases(noUpload, repo = repo)
def closeDB(self):
self.dbclosed = True
# if the class is opened readOnly, close and forget
if self.readOnly:
self.cursor.close()
self.connection.close()
return
if self.clientDatabase:
self.commitChanges()
self.cursor.close()
self.connection.close()
return
if not etpDbStatus[self.dbFile]['tainted']:
# we can unlock it, no changes were made
self.ServiceInterface.MirrorsService.lock_mirrors(False, repo = self.server_repo)
else:
self.updateProgress(
darkgreen("Mirrors have not been unlocked. Run activator."),
importance = 1,
type = "info",
header = brown(" * ")
)
self.commitChanges()
#self.vacuum()
self.cursor.close()
self.connection.close()
def vacuum(self):
self.cursor.execute("vacuum")
def commitChanges(self):
if self.readOnly:
return
try:
self.connection.commit()
except:
pass
if not self.clientDatabase:
self.taintDatabase()
if (etpDbStatus[self.dbFile]['tainted']) and \
(not etpDbStatus[self.dbFile]['bumped']):
# bump revision, setting DatabaseBump causes the session to just bump once
etpDbStatus[self.dbFile]['bumped'] = True
self.revisionBump()
def taintDatabase(self):
# if it's equo to open it, this should be avoided
if self.clientDatabase:
return
# taint the database status
taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
f = open(taint_file,"w")
f.write(etpConst['currentarch']+" database tainted\n")
f.flush()
f.close()
etpDbStatus[self.dbFile]['tainted'] = True
def untaintDatabase(self):
if (self.clientDatabase): # if it's equo to open it, this should be avoided
return
etpDbStatus[self.dbFile]['tainted'] = False
# untaint the database status
taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
if os.path.isfile(taint_file):
os.remove(taint_file)
def revisionBump(self):
revision_file = self.ServiceInterface.get_local_database_revision_file(repo = self.server_repo)
if not os.path.isfile(revision_file):
revision = 0
else:
f = open(revision_file,"r")
revision = int(f.readline().strip())
revision += 1
f.close()
f = open(revision_file,"w")
f.write(str(revision)+"\n")
f.flush()
f.close()
def isDatabaseTainted(self):
taint_file = self.ServiceInterface.get_local_database_taint_file(repo = self.server_repo)
if os.path.isfile(taint_file):
return True
return False
# never use this unless you know what you're doing
def initializeDatabase(self):
self.checkReadOnly()
self.cursor.executescript(etpConst['sql_destroy'])
self.cursor.executescript(etpConst['sql_init'])
self.databaseStructureUpdates()
self.commitChanges()
def checkReadOnly(self):
if (self.readOnly):
raise exceptionTools.OperationNotPermitted("OperationNotPermitted: can't do that on a readonly database.")
# check for /usr/portage/profiles/updates changes
def serverUpdatePackagesData(self):
etpConst['treeupdatescalled'] = True
repository = self.server_repo
repo_updates_file = self.ServiceInterface.get_local_database_treeupdates_file(self.server_repo)
doRescan = False
if repositoryUpdatesDigestCache_db.has_key(self.server_repo):
stored_digest = repositoryUpdatesDigestCache_db.get(self.server_repo)
else:
# check database digest
stored_digest = self.retrieveRepositoryUpdatesDigest(self.server_repo)
repositoryUpdatesDigestCache_db[self.server_repo] = stored_digest
if stored_digest == -1:
doRescan = True
# check portage files for changes if doRescan is still false
portage_dirs_digest = "0"
if not doRescan:
if repositoryUpdatesDigestCache_disk.has_key(self.server_repo):
portage_dirs_digest = repositoryUpdatesDigestCache_disk.get(self.server_repo)
else:
from entropy import SpmInterface
SpmIntf = SpmInterface(self.OutputInterface)
Spm = SpmIntf.intf
# grab portdir
updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates"
if os.path.isdir(updates_dir):
# get checksum
mdigest = self.entropyTools.md5sum_directory(updates_dir, get_obj = True)
# also checksum etpConst['etpdatabaseupdatefile']
if os.path.isfile(repo_updates_file):
f = open(repo_updates_file)
block = f.read(1024)
while block:
mdigest.update(block)
block = f.read(1024)
f.close()
portage_dirs_digest = mdigest.hexdigest()
repositoryUpdatesDigestCache_disk[self.server_repo] = portage_dirs_digest
del updates_dir
if doRescan or (str(stored_digest) != str(portage_dirs_digest)):
# force parameters
self.readOnly = False
self.noUpload = True
# reset database tables
self.clearTreeupdatesEntries(self.server_repo)
from entropy import SpmInterface
SpmIntf = SpmInterface(self.OutputInterface)
Spm = SpmIntf.intf
updates_dir = etpConst['systemroot']+Spm.get_spm_setting("PORTDIR")+"/profiles/updates"
update_files = self.entropyTools.sortUpdateFiles(os.listdir(updates_dir))
update_files = [os.path.join(updates_dir,x) for x in update_files]
# now load actions from files
update_actions = []
for update_file in update_files:
f = open(update_file,"r")
lines = [x.strip() for x in f.readlines() if x.strip()]
for line in lines:
update_actions.append(line)
f.close()
del lines
# add entropy packages.db.repo_updates content
if os.path.isfile(repo_updates_file):
f = open(repo_updates_file,"r")
lines = [x.strip() for x in f.readlines() if x.strip()]
for line in lines:
update_actions.append(line)
f.close()
# now filter the required actions
update_actions = self.filterTreeUpdatesActions(update_actions)
if update_actions:
self.updateProgress(
bold("ATTENTION: ")+red("forcing package updates. Syncing with %s") % (blue(updates_dir),),
importance = 1,
type = "info",
header = brown(" * ")
)
# lock database
self.doServerDatabaseSyncLock(self.noUpload)
# now run queue
self.runTreeUpdatesActions(update_actions)
# store new actions
self.addRepositoryUpdatesActions(self.server_repo,update_actions)
# store new digest into database
self.setRepositoryUpdatesDigest(self.server_repo, portage_dirs_digest)
# client side, no portage dependency
# lxnay: it is indeed very similar to serverUpdatePackagesData() but I prefer keeping both separate
# also, we reuse the same caching dictionaries of the server function
# repositoryUpdatesDigestCache_db -> repository cache
# repositoryUpdatesDigestCache_disk -> client database cache
# check for repository packages updates
# this will read database treeupdates* tables and do
# changes required if running as root.
def clientUpdatePackagesData(self, clientDbconn):
etpConst['treeupdatescalled'] = True
repository = self.dbname[len(etpConst['dbnamerepoprefix']):]
doRescan = False
if repositoryUpdatesDigestCache_db.has_key(repository):
stored_digest = repositoryUpdatesDigestCache_db.get(repository)
else:
# check database digest
stored_digest = self.retrieveRepositoryUpdatesDigest(repository)
repositoryUpdatesDigestCache_db[repository] = stored_digest
if stored_digest == -1:
doRescan = True
# check stored value in client database
client_digest = "0"
if not doRescan:
if repositoryUpdatesDigestCache_disk.has_key(etpConst['systemroot']):
client_digest = repositoryUpdatesDigestCache_disk.get(etpConst['systemroot'])
else:
client_digest = clientDbconn.retrieveRepositoryUpdatesDigest(repository)
if doRescan or (str(stored_digest) != str(client_digest)):
# reset database tables
clientDbconn.clearTreeupdatesEntries(repository)
# load updates
update_actions = self.retrieveTreeUpdatesActions(repository)
# now filter the required actions
update_actions = clientDbconn.filterTreeUpdatesActions(update_actions)
if update_actions:
self.updateProgress(
bold("ATTENTION: ")+red("forcing packages metadata update. Updating system database using repository id: %s") % (blue(repository),),
importance = 1,
type = "info",
header = darkred(" * ")
)
# run stuff
clientDbconn.runTreeUpdatesActions(update_actions)
# store new digest into database
clientDbconn.setRepositoryUpdatesDigest(repository, stored_digest)
# clear client cache
clientDbconn.clearCache(all = True)
# this functions will filter either data from /usr/portage/profiles/updates/*
# or repository database returning only the needed actions
def filterTreeUpdatesActions(self, actions):
new_actions = []
for action in actions:
doaction = action.split()
if doaction[0] == "slotmove":
# slot move
atom = doaction[1]
from_slot = doaction[2]
to_slot = doaction[3]
category = atom.split("/")[0]
matches = self.atomMatch(atom, multiMatch = True)
if matches[1] == 0:
# found atom, check slot and category
for idpackage in matches[0]:
myslot = str(self.retrieveSlot(idpackage))
mycategory = self.retrieveCategory(idpackage)
if mycategory == category:
if (myslot == from_slot) and (myslot != to_slot):
new_actions.append(action)
elif doaction[0] == "move":
atom = doaction[1]
category = atom.split("/")[0]
matches = self.atomMatch(atom, multiMatch = True)
if matches[1] == 0:
for idpackage in matches[0]:
mycategory = self.retrieveCategory(idpackage)
if mycategory == category:
new_actions.append(action)
return new_actions
# this is the place to add extra actions support
def runTreeUpdatesActions(self, actions):
# just run fixpackages if gentoo-compat is enabled
if etpConst['gentoo-compat']:
self.updateProgress(
bold("GENTOO: ")+red("Running fixpackages, could take a while."),
importance = 1,
type = "warning",
header = darkred(" * ")
)
if self.clientDatabase:
os.system("fixpackages &> /dev/null")
else:
os.system("fixpackages")
quickpkg_atoms = set()
for action in actions:
command = action.split()
self.updateProgress(
bold("ENTROPY: ")+red("action: %s") % (blue(action),),
importance = 1,
type = "warning",
header = darkred(" * ")
)
if command[0] == "move":
quickpkg_atoms |= self.runTreeUpdatesMoveAction(command[1:], quickpkg_atoms)
elif command[0] == "slotmove":
quickpkg_atoms |= self.runTreeUpdatesSlotmoveAction(command[1:])
if quickpkg_atoms and not self.clientDatabase:
# quickpkg package and packages owning it as a dependency
self.runTreeUpdatesQuickpkgAction(quickpkg_atoms)
self.commitChanges()
# discard cache
self.clearCache(all = True)
# -- move action:
# 1) move package key to the new name: category + name + atom
# 2) update all the dependencies in dependenciesreference to the new key
# 3) run fixpackages which will update /var/db/pkg files
# 4) automatically run quickpkg() to build the new binary and
# tainted binaries owning tainted iddependency and taint database
def runTreeUpdatesMoveAction(self, move_command, quickpkg_queue):
key_from = move_command[0]
key_to = move_command[1]
cat_to = key_to.split("/")[0]
name_to = key_to.split("/")[1]
matches = self.atomMatch(key_from, multiMatch = True)
iddependencies_idpackages = set()
for idpackage in matches[0]:
slot = self.retrieveSlot(idpackage)
old_atom = self.retrieveAtom(idpackage)
new_atom = old_atom.replace(key_from,key_to)
### UPDATE DATABASE
# update category
self.setCategory(idpackage, cat_to)
# update name
self.setName(idpackage, name_to)
# update atom
self.setAtom(idpackage, new_atom)
# look for packages we need to quickpkg again
# note: quickpkg_queue is simply ignored if self.clientDatabase
quickpkg_queue.add(key_to+":"+str(slot))
iddeps = self.searchDependency(key_from, like = True, multi = True)
for iddep in iddeps:
# update string
mydep = self.retrieveDependencyFromIddependency(iddep)
mydep_key = self.entropyTools.dep_getkey(mydep)
if mydep_key != key_from: # avoid changing wrong atoms -> dev-python/qscintilla-python would
continue # become x11-libs/qscintilla if we don't do this check
mydep = mydep.replace(key_from,key_to)
# now update
# dependstable on server is always re-generated
self.setDependency(iddep, mydep)
# we have to repackage also package owning this iddep
iddependencies_idpackages |= self.searchIdpackageFromIddependency(iddep)
if not self.clientDatabase:
# check for injection and warn the developer
injected = self.isInjected(idpackage)
if injected:
self.updateProgress(
bold("INJECT: ")+red("Package %s has been injected. You need to quickpkg it manually to update embedded database !!! Repository database will be updated anyway.") % (blue(new_atom),),
importance = 1,
type = "warning",
header = darkred(" * ")
)
self.commitChanges()
quickpkg_queue = list(quickpkg_queue)
for x in range(len(quickpkg_queue)):
myatom = quickpkg_queue[x]
myatom = myatom.replace(key_from,key_to)
quickpkg_queue[x] = myatom
quickpkg_queue = set(quickpkg_queue)
for idpackage_owner in iddependencies_idpackages:
myatom = self.retrieveAtom(idpackage_owner)
myatom = myatom.replace(key_from,key_to)
quickpkg_queue.add(myatom)
return quickpkg_queue
# -- slotmove action:
# 1) move package slot
# 2) update all the dependencies in dependenciesreference owning same matched atom + slot
# 3) run fixpackages which will update /var/db/pkg files
# 4) automatically run quickpkg() to build the new binary and tainted binaries owning tainted iddependency and taint database
def runTreeUpdatesSlotmoveAction(self, slotmove_command):
atom = slotmove_command[0]
atomkey = self.entropyTools.dep_getkey(atom)
slot_from = slotmove_command[1]
slot_to = slotmove_command[2]
matches = self.atomMatch(atom, multiMatch = True)
iddependencies_idpackages = set()
quickpkg_queue = set()
for idpackage in matches[0]:
### UPDATE DATABASE
# update slot
self.setSlot(idpackage, slot_to)
# look for packages we need to quickpkg again
# note: quickpkg_queue is simply ignored if self.clientDatabase
quickpkg_queue = [atom+":"+str(slot_to)]
iddeps = self.searchDependency(atomkey, like = True, multi = True)
for iddep in iddeps:
# update string
mydep = self.retrieveDependencyFromIddependency(iddep)
if mydep.find(":"+str(slot_from)) != -1: # probably slotted dep
mydep = mydep.replace(":"+str(slot_from),":"+str(slot_to))
else:
continue # it's fine
# now update
# dependstable on server is always re-generated
self.setDependency(iddep, mydep)
# we have to repackage also package owning this iddep
iddependencies_idpackages |= self.searchIdpackageFromIddependency(iddep)
if not self.clientDatabase:
# check for injection and warn the developer
injected = self.isInjected(idpackage)
if injected:
self.updateProgress(
bold("INJECT: ")+red("Package %s has been injected. You need to quickpkg it manually to update embedded database !!! Repository database will be updated anyway.") % (blue(atom),),
importance = 1,
type = "warning",
header = darkred(" * ")
)
self.commitChanges()
for idpackage_owner in iddependencies_idpackages:
myatom = self.retrieveAtom(idpackage_owner)
quickpkg_queue.add(myatom)
return quickpkg_queue
def runTreeUpdatesQuickpkgAction(self, atoms):
branch = etpConst['branch']
# ask branch question
rc = self.askQuestion("Would you like to continue with the default branch \"%s\" ?" % (branch,))
if rc == "No":
# ask which
while 1:
branch = readtext("Type your branch: ")
if branch not in self.listAllBranches():
self.updateProgress(
bold("ATTENTION: ")+red("Specified branch %s does not exist.") % (blue(branch),),
importance = 1,
type = "warning",
header = darkred(" * ")
)
continue
# ask to confirm
rc = self.askQuestion("Confirm %s ?" % (branch,))
if rc == "Yes":
break
self.commitChanges()
package_paths = set()
runatoms = set()
for myatom in atoms:
mymatch = self.atomMatch(myatom)
if mymatch[0] == -1:
continue
myatom = self.retrieveAtom(mymatch[0])
runatoms.add(myatom)
for myatom in runatoms:
self.updateProgress(
red("repackaging: ")+blue(myatom),
importance = 1,
type = "warning",
header = blue(" # ")
)
mypath = self.ServiceInterface.quickpkg(myatom,self.ServiceInterface.get_local_store_directory(self.server_repo))
package_paths.add(mypath)
packages_data = [(x,branch,False) for x in package_paths]
idpackages = self.ServiceInterface.add_packages_to_repository(packages_data, repo = self.server_repo)
if not idpackages:
self.updateProgress(
bold("ATTENTION: ")+red("reagent update did not run properly. Please update packages manually"),
importance = 1,
type = "warning",
header = darkred(" * ")
)
# this function manages the submitted package
# if it does not exist, it fires up addPackage
# otherwise it fires up updatePackage
def handlePackage(self, etpData, forcedRevision = -1):
self.checkReadOnly()
# build atom string
versiontag = ''
if etpData['versiontag']:
versiontag = '#'+etpData['versiontag']
foundid = self.isPackageAvailable(etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag)
if (foundid < 0): # same atom doesn't exist in any branch
return self.addPackage(etpData, revision = forcedRevision)
else:
return self.updatePackage(etpData, forcedRevision) # only when the same atom exists
def addPackage(self, etpData, revision = -1):
self.checkReadOnly()
self.live_cache.clear()
if revision == -1:
try:
revision = int(etpData['revision'])
except (KeyError, ValueError):
etpData['revision'] = 0 # revision not specified
revision = 0
# we need to find other packages with the same key and slot, and remove them
if (self.clientDatabase): # client database can't care about branch
searchsimilar = self.searchPackagesByNameAndCategory(name = etpData['name'], category = etpData['category'], sensitive = True)
else: # server supports multiple branches inside a db
searchsimilar = self.searchPackagesByNameAndCategory(name = etpData['name'], category = etpData['category'], sensitive = True, branch = etpData['branch'])
removelist = set()
if not etpData['injected']: # read: if package has been injected, we'll skip the removal of packages in the same slot, usually used server side btw
for oldpkg in searchsimilar:
# get the package slot
idpackage = oldpkg[1]
slot = self.retrieveSlot(idpackage)
isinjected = self.isInjected(idpackage)
if isinjected:
continue # we merely ignore packages with negative counters, since they're the injected ones
if (etpData['slot'] == slot):
# remove!
removelist.add(idpackage)
for pkg in removelist:
self.removePackage(pkg)
# create new category if it doesn't exist
catid = self.isCategoryAvailable(etpData['category'])
if (catid == -1):
# create category
catid = self.addCategory(etpData['category'])
# create new license if it doesn't exist
licid = self.isLicenseAvailable(etpData['license'])
if (licid == -1):
# create category
licid = self.addLicense(etpData['license'])
# insert license information
mylicenses = etpData['licensedata'].keys()
for mylicense in mylicenses:
found = self.isLicensedataKeyAvailable(mylicense)
if not found:
text = etpData['licensedata'][mylicense]
self.cursor.execute(
'INSERT into licensedata VALUES '
'(?,?,?)'
, ( mylicense,
buffer(text),
0,
))
# look for configured versiontag
versiontag = ""
if (etpData['versiontag']):
versiontag = "#"+etpData['versiontag']
trigger = 0
if etpData['trigger']:
trigger = 1
# baseinfo
pkgatom = etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag
# create new idflag if it doesn't exist
idflags = self.areCompileFlagsAvailable(etpData['chost'],etpData['cflags'],etpData['cxxflags'])
if (idflags == -1):
# create category
idflags = self.addCompileFlags(etpData['chost'],etpData['cflags'],etpData['cxxflags'])
self.cursor.execute(
'INSERT into baseinfo VALUES '
'(NULL,?,?,?,?,?,?,?,?,?,?,?)'
, ( pkgatom,
catid,
etpData['name'],
etpData['version'],
etpData['versiontag'],
revision,
etpData['branch'],
etpData['slot'],
licid,
etpData['etpapi'],
trigger,
)
)
idpackage = self.cursor.lastrowid
# extrainfo
self.cursor.execute(
'INSERT into extrainfo VALUES '
'(?,?,?,?,?,?,?,?)'
, ( idpackage,
etpData['description'],
etpData['homepage'],
etpData['download'],
etpData['size'],
idflags,
etpData['digest'],
etpData['datecreation'],
)
)
### other information iserted below are not as critical as these above
# content, a list
self.insertContent(idpackage,etpData['content'])
etpData['counter'] = int(etpData['counter']) # cast to integer
if etpData['counter'] != -1 and not (etpData['injected']):
if etpData['counter'] <= -2:
# special cases
etpData['counter'] = self.getNewNegativeCounter()
try:
self.cursor.execute(
'INSERT into counters VALUES '
'(?,?,?)'
, ( etpData['counter'],
idpackage,
etpData['branch'],
)
)
except dbapi2.IntegrityError: # we have a PRIMARY KEY we need to remove
self.migrateCountersTable()
self.cursor.execute(
'INSERT into counters VALUES '
'(?,?,?)'
, ( etpData['counter'],
idpackage,
etpData['branch'],
)
)
except:
if self.dbname == etpConst['clientdbid']: # force only for client database
if self.doesTableExist("counters"):
raise
self.cursor.execute(
'INSERT into counters VALUES '
'(?,?,?)'
, ( etpData['counter'],
idpackage,
etpData['branch'],
)
)
elif self.dbname.startswith(etpConst['serverdbid']):
raise
# on disk size
self.cursor.execute(
'INSERT into sizes VALUES '
'(?,?)'
, ( idpackage,
etpData['disksize'],
)
)
# trigger blob
self.cursor.execute(
'INSERT into triggers VALUES '
'(?,?)'
, ( idpackage,
buffer(etpData['trigger']),
))
# eclasses table
for var in etpData['eclasses']:
idclass = self.isEclassAvailable(var)
if (idclass == -1):
# create eclass
idclass = self.addEclass(var)
self.cursor.execute(
'INSERT into eclasses VALUES '
'(?,?)'
, ( idpackage,
idclass,
)
)
# needed table
for mydata in etpData['needed']:
needed = mydata[0]
elfclass = mydata[1]
idneeded = self.isNeededAvailable(needed)
if (idneeded == -1):
# create eclass
idneeded = self.addNeeded(needed)
self.cursor.execute(
'INSERT into needed VALUES '
'(?,?,?)'
, ( idpackage,
idneeded,
elfclass,
)
)
# dependencies, a list
self.insertDependencies(idpackage, etpData['dependencies'])
# provide
for atom in etpData['provide']:
self.cursor.execute(
'INSERT into provide VALUES '
'(?,?)'
, ( idpackage,
atom,
)
)
# injected?
if etpData['injected']:
self.setInjected(idpackage)
# compile messages
for message in etpData['messages']:
self.cursor.execute(
'INSERT into messages VALUES '
'(?,?)'
, ( idpackage,
message,
)
)
# is it a system package?
if etpData['systempackage']:
self.cursor.execute(
'INSERT into systempackages VALUES '
'(?)'
, ( idpackage,
)
)
# create new protect if it doesn't exist
idprotect = self.isProtectAvailable(etpData['config_protect'])
if (idprotect == -1):
# create category
idprotect = self.addProtect(etpData['config_protect'])
# fill configprotect
self.cursor.execute(
'INSERT into configprotect VALUES '
'(?,?)'
, ( idpackage,
idprotect,
)
)
idprotect = self.isProtectAvailable(etpData['config_protect_mask'])
if (idprotect == -1):
# create category
idprotect = self.addProtect(etpData['config_protect_mask'])
# fill configprotect
self.cursor.execute(
'INSERT into configprotectmask VALUES '
'(?,?)'
, ( idpackage,
idprotect,
)
)
# conflicts, a list
for conflict in etpData['conflicts']:
self.cursor.execute(
'INSERT into conflicts VALUES '
'(?,?)'
, ( idpackage,
conflict,
)
)
# mirrorlinks, always update the table
for mirrordata in etpData['mirrorlinks']:
mirrorname = mirrordata[0]
mirrorlist = mirrordata[1]
# remove old
self.removeMirrorEntries(mirrorname)
# add new
self.addMirrors(mirrorname,mirrorlist)
# sources, a list
for source in etpData['sources']:
if (not source) or (source == "") or (not self.entropyTools.is_valid_string(source)):
continue
idsource = self.isSourceAvailable(source)
if (idsource == -1):
# create category
idsource = self.addSource(source)
self.cursor.execute(
'INSERT into sources VALUES '
'(?,?)'
, ( idpackage,
idsource,
)
)
# useflags, a list
for flag in etpData['useflags']:
iduseflag = self.isUseflagAvailable(flag)
if (iduseflag == -1):
# create category
iduseflag = self.addUseflag(flag)
self.cursor.execute(
'INSERT into useflags VALUES '
'(?,?)'
, ( idpackage,
iduseflag,
)
)
# create new keyword if it doesn't exist
for key in etpData['keywords']:
idkeyword = self.isKeywordAvailable(key)
if (idkeyword == -1):
# create category
idkeyword = self.addKeyword(key)
self.cursor.execute(
'INSERT into keywords VALUES '
'(?,?)'
, ( idpackage,
idkeyword,
)
)
self.clearCache()
self.commitChanges()
### RSS Atom support
### dictionary will be elaborated by activator
if etpConst['rss-feed'] and not self.clientDatabase:
rssAtom = pkgatom+"~"+str(revision)
# store addPackage action
rssObj = dumpTools.loadobj(etpConst['rss-dump-name'])
if rssObj:
global etpRSSMessages
etpRSSMessages = rssObj.copy()
if rssAtom in etpRSSMessages['removed']:
del etpRSSMessages['removed'][rssAtom]
etpRSSMessages['added'][rssAtom] = {}
etpRSSMessages['added'][rssAtom]['description'] = etpData['description']
etpRSSMessages['added'][rssAtom]['homepage'] = etpData['homepage']
etpRSSMessages['light'][rssAtom] = {}
etpRSSMessages['light'][rssAtom]['description'] = etpData['description']
# save
dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages)
return idpackage, revision, etpData
# Update already available atom in db
# returns True,revision if the package has been updated
# returns False,revision if not
def updatePackage(self, etpData, forcedRevision = -1):
self.checkReadOnly()
# build atom string
versiontag = ''
if etpData['versiontag']:
versiontag = '#'+etpData['versiontag']
pkgatom = etpData['category'] + "/" + etpData['name'] + "-" + etpData['version']+versiontag
# for client database - the atom if present, must be overwritten with the new one regardless its branch
if (self.clientDatabase):
atomid = self.isPackageAvailable(pkgatom)
if atomid > -1:
self.removePackage(atomid)
return self.addPackage(etpData, revision = forcedRevision)
else:
# update package in etpData['branch']
# get its package revision
idpackage = self.getIDPackage(pkgatom,etpData['branch'])
if (forcedRevision == -1):
if (idpackage != -1):
curRevision = self.retrieveRevision(idpackage)
else:
curRevision = 0
else:
curRevision = forcedRevision
if (idpackage != -1): # remove old package in branch
self.removePackage(idpackage)
if (forcedRevision == -1):
curRevision += 1
# add the new one
return self.addPackage(etpData, revision = curRevision)
def removePackage(self,idpackage):
self.checkReadOnly()
self.live_cache.clear()
### RSS Atom support
### dictionary will be elaborated by activator
if etpConst['rss-feed'] and not self.clientDatabase:
# store addPackage action
rssObj = dumpTools.loadobj(etpConst['rss-dump-name'])
if rssObj:
global etpRSSMessages
etpRSSMessages = rssObj.copy()
rssAtom = self.retrieveAtom(idpackage)
rssRevision = self.retrieveRevision(idpackage)
rssAtom += "~"+str(rssRevision)
if rssAtom in etpRSSMessages['added']:
del etpRSSMessages['added'][rssAtom]
etpRSSMessages['removed'][rssAtom] = {}
try:
etpRSSMessages['removed'][rssAtom]['description'] = self.retrieveDescription(idpackage)
except:
etpRSSMessages['removed'][rssAtom]['description'] = "N/A"
try:
etpRSSMessages['removed'][rssAtom]['homepage'] = self.retrieveHomepage(idpackage)
except:
etpRSSMessages['removed'][rssAtom]['homepage'] = ""
# save
dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages)
idpackage = str(idpackage)
# baseinfo
self.cursor.execute('DELETE FROM baseinfo WHERE idpackage = '+idpackage)
# extrainfo
self.cursor.execute('DELETE FROM extrainfo WHERE idpackage = '+idpackage)
# content
self.cursor.execute('DELETE FROM content WHERE idpackage = '+idpackage)
# dependencies
self.cursor.execute('DELETE FROM dependencies WHERE idpackage = '+idpackage)
# provide
self.cursor.execute('DELETE FROM provide WHERE idpackage = '+idpackage)
# conflicts
self.cursor.execute('DELETE FROM conflicts WHERE idpackage = '+idpackage)
# protect
self.cursor.execute('DELETE FROM configprotect WHERE idpackage = '+idpackage)
# protect_mask
self.cursor.execute('DELETE FROM configprotectmask WHERE idpackage = '+idpackage)
# sources
self.cursor.execute('DELETE FROM sources WHERE idpackage = '+idpackage)
# useflags
self.cursor.execute('DELETE FROM useflags WHERE idpackage = '+idpackage)
# keywords
self.cursor.execute('DELETE FROM keywords WHERE idpackage = '+idpackage)
#
# WARNING: exception won't be handled anymore with 1.0
#
try:
# messages
self.cursor.execute('DELETE FROM messages WHERE idpackage = '+idpackage)
except:
pass
try:
# systempackage
self.cursor.execute('DELETE FROM systempackages WHERE idpackage = '+idpackage)
except:
pass
try:
# counter
self.cursor.execute('DELETE FROM counters WHERE idpackage = '+idpackage)
except:
if (self.dbname == etpConst['clientdbid']) or self.dbname.startswith(etpConst['serverdbid']):
raise
try:
# on disk sizes
self.cursor.execute('DELETE FROM sizes WHERE idpackage = '+idpackage)
except:
pass
try:
# eclasses
self.cursor.execute('DELETE FROM eclasses WHERE idpackage = '+idpackage)
except:
pass
try:
# needed
self.cursor.execute('DELETE FROM needed WHERE idpackage = '+idpackage)
except:
pass
try:
# triggers
self.cursor.execute('DELETE FROM triggers WHERE idpackage = '+idpackage)
except:
pass
try:
# inject table
self.cursor.execute('DELETE FROM injected WHERE idpackage = '+idpackage)
except:
pass
# Remove from installedtable if exists
self.removePackageFromInstalledTable(idpackage)
# Remove from dependstable if exists
self.removePackageFromDependsTable(idpackage)
# Cleanups if at least one package has been removed
self.cleanupUseflags()
self.cleanupSources()
self.cleanupEclasses()
self.cleanupNeeded()
self.cleanupDependencies()
# clear caches
self.clearCache()
self.commitChanges()
def removeMirrorEntries(self,mirrorname):
self.cursor.execute('DELETE FROM mirrorlinks WHERE mirrorname = "'+mirrorname+'"')
def addMirrors(self,mirrorname,mirrorlist):
for x in mirrorlist:
self.cursor.execute(
'INSERT into mirrorlinks VALUES '
'(?,?)', (mirrorname,x,)
)
def addCategory(self,category):
self.cursor.execute(
'INSERT into categories VALUES '
'(NULL,?)', (category,)
)
# get info about inserted value and return
cat = self.isCategoryAvailable(category)
if cat != -1:
return cat
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a category but then, fetching it returned -1. There's something broken.")
def addProtect(self,protect):
self.cursor.execute(
'INSERT into configprotectreference VALUES '
'(NULL,?)', (protect,)
)
# get info about inserted value and return
prt = self.isProtectAvailable(protect)
if prt != -1:
return prt
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a protect but then, fetching it returned -1. There's something broken.")
def addSource(self,source):
self.cursor.execute(
'INSERT into sourcesreference VALUES '
'(NULL,?)', (source,)
)
# get info about inserted value and return
src = self.isSourceAvailable(source)
if src != -1:
return src
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a source but then, fetching it returned -1. There's something broken.")
def addDependency(self,dependency):
self.cursor.execute(
'INSERT into dependenciesreference VALUES '
'(NULL,?)', (dependency,)
)
# get info about inserted value and return
dep = self.isDependencyAvailable(dependency)
if dep != -1:
return dep
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a dependency but then, fetching it returned -1. There's something broken.")
def addKeyword(self,keyword):
self.cursor.execute(
'INSERT into keywordsreference VALUES '
'(NULL,?)', (keyword,)
)
# get info about inserted value and return
key = self.isKeywordAvailable(keyword)
if key != -1:
return key
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a keyword but then, fetching it returned -1. There's something broken.")
def addUseflag(self,useflag):
self.cursor.execute(
'INSERT into useflagsreference VALUES '
'(NULL,?)', (useflag,)
)
# get info about inserted value and return
use = self.isUseflagAvailable(useflag)
if use != -1:
return use
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a use flag but then, fetching it returned -1. There's something broken.")
def addEclass(self,eclass):
self.cursor.execute(
'INSERT into eclassesreference VALUES '
'(NULL,?)', (eclass,)
)
# get info about inserted value and return
myclass = self.isEclassAvailable(eclass)
if myclass != -1:
return myclass
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert an eclass but then, fetching it returned -1. There's something broken.")
def addNeeded(self,needed):
self.cursor.execute(
'INSERT into neededreference VALUES '
'(NULL,?)', (needed,)
)
# get info about inserted value and return
myneeded = self.isNeededAvailable(needed)
if myneeded != -1:
return myneeded
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a needed library but then, fetching it returned -1. There's something broken.")
def addLicense(self,pkglicense):
if not self.entropyTools.is_valid_string(pkglicense):
pkglicense = ' ' # workaround for broken license entries
self.cursor.execute(
'INSERT into licenses VALUES '
'(NULL,?)', (pkglicense,)
)
# get info about inserted value and return
lic = self.isLicenseAvailable(pkglicense)
if lic != -1:
return lic
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert a license but then, fetching it returned -1. There's something broken.")
def addCompileFlags(self,chost,cflags,cxxflags):
self.cursor.execute(
'INSERT into flags VALUES '
'(NULL,?,?,?)', (chost,cflags,cxxflags,)
)
# get info about inserted value and return
idflag = self.areCompileFlagsAvailable(chost,cflags,cxxflags)
if idflag != -1:
return idflag
raise exceptionTools.CorruptionError("CorruptionError: I tried to insert compile flags but then, fetching it returned -1. There's something broken.")
def setInjected(self, idpackage):
self.checkReadOnly()
if not self.isInjected(idpackage):
self.cursor.execute(
'INSERT into injected VALUES '
'(?)'
, ( idpackage, )
)
self.commitChanges()
# date expressed the unix way
def setDateCreation(self, idpackage, date):
self.checkReadOnly()
self.cursor.execute('UPDATE extrainfo SET datecreation = (?) WHERE idpackage = (?)', (str(date),idpackage,))
self.commitChanges()
def setDigest(self, idpackage, digest):
self.checkReadOnly()
self.cursor.execute('UPDATE extrainfo SET digest = (?) WHERE idpackage = (?)', (digest,idpackage,))
self.commitChanges()
def setDownloadURL(self, idpackage, url):
self.checkReadOnly()
self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (url,idpackage,))
self.commitChanges()
def setCategory(self, idpackage, category):
self.checkReadOnly()
# create new category if it doesn't exist
catid = self.isCategoryAvailable(category)
if (catid == -1):
# create category
catid = self.addCategory(category)
self.cursor.execute('UPDATE baseinfo SET idcategory = (?) WHERE idpackage = (?)', (catid,idpackage,))
self.commitChanges()
def setName(self, idpackage, name):
self.checkReadOnly()
self.cursor.execute('UPDATE baseinfo SET name = (?) WHERE idpackage = (?)', (name,idpackage,))
self.commitChanges()
def setDependency(self, iddependency, dependency):
self.checkReadOnly()
self.cursor.execute('UPDATE dependenciesreference SET dependency = (?) WHERE iddependency = (?)', (dependency,iddependency,))
self.commitChanges()
def setAtom(self, idpackage, atom):
self.checkReadOnly()
self.cursor.execute('UPDATE baseinfo SET atom = (?) WHERE idpackage = (?)', (atom,idpackage,))
self.commitChanges()
def setSlot(self, idpackage, slot):
self.checkReadOnly()
self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot,idpackage,))
self.commitChanges()
def removeLicensedata(self, license_name):
if not self.doesTableExist("licensedata"):
return
self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,))
def removeDependencies(self, idpackage):
self.checkReadOnly()
self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,))
self.commitChanges()
def insertDependencies(self, idpackage, deplist):
for dep in deplist:
iddep = self.isDependencyAvailable(dep)
if (iddep == -1):
# create category
iddep = self.addDependency(dep)
self.cursor.execute(
'INSERT into dependencies VALUES '
'(?,?)'
, ( idpackage,
iddep,
)
)
def removeContent(self, idpackage):
self.checkReadOnly()
self.cursor.execute("DELETE FROM content WHERE idpackage = (?)", (idpackage,))
self.commitChanges()
def insertContent(self, idpackage, content):
def myiter():
for xfile in content:
contenttype = content[xfile]
if type(xfile) is unicode:
xfile = xfile.encode('raw_unicode_escape')
yield (idpackage,xfile,contenttype,)
self.cursor.executemany('INSERT INTO content VALUES (?,?,?)',myiter())
def insertCounter(self, idpackage, counter, branch = None):
self.checkReadOnly()
if not branch:
branch = etpConst['branch']
self.cursor.execute('DELETE FROM counters WHERE counter = (?) OR idpackage = (?)', (counter,idpackage,))
self.cursor.execute('INSERT INTO counters VALUES (?,?,?)', (counter,idpackage,branch,))
self.commitChanges()
def setCounter(self, idpackage, counter, branch = None):
self.checkReadOnly()
branchstring = ''
insertdata = [counter,idpackage]
if branch:
branchstring = ', branch = (?)'
insertdata.insert(1,branch)
else:
branch = etpConst['branch']
try:
self.cursor.execute('UPDATE counters SET counter = (?) '+branchstring+' WHERE idpackage = (?)', insertdata)
except:
if self.dbname == etpConst['clientdbid']:
raise
self.commitChanges()
def contentDiff(self, idpackage, dbconn, dbconn_idpackage):
self.checkReadOnly()
self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
# create a random table and fill
randomtable = "cdiff"+str(self.entropyTools.getRandomNumber())
self.cursor.execute('DROP TABLE IF EXISTS '+randomtable)
self.cursor.execute('CREATE TEMPORARY TABLE '+randomtable+' ( file VARCHAR )')
dbconn.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
dbconn.cursor.execute('select file from content where idpackage = (?)', (dbconn_idpackage,))
xfile = dbconn.cursor.fetchone()
while xfile:
self.cursor.execute('INSERT INTO '+randomtable+' VALUES (?)', (xfile[0],))
xfile = dbconn.cursor.fetchone()
# now compare
self.cursor.execute('SELECT file FROM content WHERE content.idpackage = (?) AND content.file NOT IN (SELECT file from '+randomtable+') ', (idpackage,))
diff = self.fetchall2set(self.cursor.fetchall())
self.cursor.execute('DROP TABLE IF EXISTS '+randomtable)
return diff
def cleanupUseflags(self):
self.checkReadOnly()
self.cursor.execute('delete from useflagsreference where idflag IN (select idflag from useflagsreference where idflag NOT in (select idflag from useflags))')
self.commitChanges()
def cleanupSources(self):
self.checkReadOnly()
self.cursor.execute('delete from sourcesreference where idsource IN (select idsource from sourcesreference where idsource NOT in (select idsource from sources))')
self.commitChanges()
def cleanupEclasses(self):
self.checkReadOnly()
self.cursor.execute('delete from eclassesreference where idclass IN (select idclass from eclassesreference where idclass NOT in (select idclass from eclasses))')
self.commitChanges()
def cleanupNeeded(self):
self.checkReadOnly()
self.cursor.execute('delete from neededreference where idneeded IN (select idneeded from neededreference where idneeded NOT in (select idneeded from needed))')
self.commitChanges()
def cleanupDependencies(self):
self.checkReadOnly()
self.cursor.execute('delete from dependenciesreference where iddependency IN (select iddependency from dependenciesreference where iddependency NOT in (select iddependency from dependencies))')
self.commitChanges()
def getNewNegativeCounter(self):
counter = -2
try:
mycounters = list(self.listAllCounters(onlycounters = True))
mycounter = min(mycounters)
if mycounter >= -1:
counter = -2
else:
counter = mycounter-1
except:
pass
return counter
def getApi(self):
self.cursor.execute('SELECT max(etpapi) FROM baseinfo')
return self.cursor.fetchone()[0]
def getIDPackage(self, atom, branch = etpConst['branch']):
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = "'+atom+'" AND branch = "'+branch+'"')
idpackage = -1
idpackage = self.cursor.fetchone()
if idpackage:
idpackage = idpackage[0]
else:
idpackage = -1
return idpackage
def getIDPackageFromDownload(self, file, branch = etpConst['branch']):
self.cursor.execute('SELECT baseinfo.idpackage FROM content,baseinfo WHERE content.file = (?) and baseinfo.branch = (?)', (file,branch,))
idpackage = self.cursor.fetchone()
if idpackage:
idpackage = idpackage[0]
else:
idpackage = -1
return idpackage
def getIDPackagesFromFile(self, file):
self.cursor.execute('SELECT idpackage FROM content WHERE file = "'+file+'"')
idpackages = []
for row in self.cursor:
idpackages.append(row[0])
return idpackages
def getIDCategory(self, category):
self.cursor.execute('SELECT "idcategory" FROM categories WHERE category = "'+str(category)+'"')
idcat = -1
for row in self.cursor:
idcat = int(row[0])
break
return idcat
def getScopeData(self, idpackage):
self.cursor.execute("""
SELECT
baseinfo.atom,
categories.category,
baseinfo.name,
baseinfo.version,
baseinfo.slot,
baseinfo.versiontag,
baseinfo.revision,
baseinfo.branch
FROM
baseinfo,
categories
WHERE
baseinfo.idpackage = (?)
and baseinfo.idcategory = categories.idcategory
""", (idpackage,))
return self.cursor.fetchone()
def getBaseData(self,idpackage):
sql = """
SELECT
baseinfo.atom,
baseinfo.name,
baseinfo.version,
baseinfo.versiontag,
extrainfo.description,
categories.category,
flags.chost,
flags.cflags,
flags.cxxflags,
extrainfo.homepage,
licenses.license,
baseinfo.branch,
extrainfo.download,
extrainfo.digest,
baseinfo.slot,
baseinfo.etpapi,
extrainfo.datecreation,
extrainfo.size,
baseinfo.revision
FROM
baseinfo,
extrainfo,
categories,
flags,
licenses
WHERE
baseinfo.idpackage = '"""+str(idpackage)+"""'
and baseinfo.idpackage = extrainfo.idpackage
and baseinfo.idcategory = categories.idcategory
and extrainfo.idflags = flags.idflags
and baseinfo.idlicense = licenses.idlicense
"""
self.cursor.execute(sql)
return self.cursor.fetchone()
def getTriggerInfo(self, idpackage):
data = {}
mydata = self.getScopeData(idpackage)
data['atom'] = mydata[0]
data['category'] = mydata[1]
data['name'] = mydata[2]
data['version'] = mydata[3]
data['versiontag'] = mydata[5]
flags = self.retrieveCompileFlags(idpackage)
data['chost'] = flags[0]
data['cflags'] = flags[1]
data['cxxflags'] = flags[2]
data['trigger'] = self.retrieveTrigger(idpackage)
data['eclasses'] = self.retrieveEclasses(idpackage)
data['content'] = self.retrieveContent(idpackage)
return data
def getPackageData(self, idpackage, get_content = True):
data = {}
mydata = self.getBaseData(idpackage)
data['name'] = mydata[1]
data['version'] = mydata[2]
data['versiontag'] = mydata[3]
data['description'] = mydata[4]
data['category'] = mydata[5]
data['chost'] = mydata[6]
data['cflags'] = mydata[7]
data['cxxflags'] = mydata[8]
data['homepage'] = mydata[9]
data['useflags'] = self.retrieveUseflags(idpackage)
data['license'] = mydata[10]
data['keywords'] = self.retrieveKeywords(idpackage)
data['branch'] = mydata[11]
data['download'] = mydata[12]
data['digest'] = mydata[13]
data['sources'] = self.retrieveSources(idpackage)
data['counter'] = self.retrieveCounter(idpackage) # cannot insert into the sql above
data['messages'] = self.retrieveMessages(idpackage)
data['trigger'] = self.retrieveTrigger(idpackage) #FIXME: needed for now because of new column
if (self.isSystemPackage(idpackage)):
data['systempackage'] = 'xxx'
else:
data['systempackage'] = ''
# FIXME: this will be removed when 1.0 will be out
data['config_protect'] = self.retrieveProtect(idpackage)
data['config_protect_mask'] = self.retrieveProtectMask(idpackage)
data['eclasses'] = self.retrieveEclasses(idpackage)
data['needed'] = self.retrieveNeeded(idpackage, extended = True)
mirrornames = set()
for x in data['sources']:
if x.startswith("mirror://"):
mirrorname = x.split("/")[2]
mirrornames.add(mirrorname)
data['mirrorlinks'] = []
for mirror in mirrornames:
mirrorlinks = self.retrieveMirrorInfo(mirror)
data['mirrorlinks'].append([mirror,mirrorlinks])
data['slot'] = mydata[14]
data['injected'] = self.isInjected(idpackage)
data['content'] = {}
if get_content:
mycontent = self.retrieveContent(idpackage, extended = True)
for cdata in mycontent:
data['content'][cdata[0]] = cdata[1]
data['dependencies'] = self.retrieveDependencies(idpackage)
data['provide'] = self.retrieveProvide(idpackage)
data['conflicts'] = self.retrieveConflicts(idpackage)
data['etpapi'] = mydata[15]
data['datecreation'] = mydata[16]
data['size'] = mydata[17]
data['revision'] = mydata[18]
data['disksize'] = self.retrieveOnDiskSize(idpackage) # cannot do this too, for backward compat
data['licensedata'] = self.retrieveLicensedata(idpackage)
return data
def fetchall2set(self, item):
mycontent = set()
for x in item:
mycontent |= set(x)
return mycontent
def fetchall2list(self, item):
content = []
for x in item:
content += list(x)
return content
def fetchone2list(self, item):
return list(item)
def fetchone2set(self, item):
return set(item)
def clearCache(self, all = False):
self.live_cache.clear()
def do_clear(name):
dump_path = os.path.join(etpConst['dumpstoragedir'],name)
dump_dir = os.path.dirname(dump_path)
if os.path.isdir(dump_dir):
for item in os.listdir(dump_dir):
item = os.path.join(dump_dir,item)
if os.path.isfile(item):
os.remove(item)
if all:
do_clear(etpCache['dbInfo']+"/"+self.dbname+"/")
do_clear(etpCache['dbMatch']+"/"+self.dbname+"/")
do_clear(etpCache['dbSearch']+"/"+self.dbname+"/")
def fetchInfoCache(self, idpackage, function, extra_hash = 0):
if self.xcache:
c_hash = str(hash(function)) + str(extra_hash)
c_match = str(idpackage)
try:
cached = dumpTools.loadobj(etpCache['dbInfo']+"/"+self.dbname+"/"+c_match+"/"+c_hash)
if cached != None:
return cached
except EOFError:
pass
def storeInfoCache(self, idpackage, function, info_cache_data, extra_hash = 0):
if self.xcache:
c_hash = str(hash(function)) + str(extra_hash)
c_match = str(idpackage)
try:
sperms = False
if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbInfo']+"/"+self.dbname)):
sperms = True
dumpTools.dumpobj(etpCache['dbInfo']+"/"+self.dbname+"/"+c_match+"/"+c_hash,info_cache_data)
if sperms:
const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid'])
except IOError:
pass
def fetchSearchCache(self, key, function, extra_hash = 0):
if self.xcache:
c_hash = str(hash(function)) + str(extra_hash)
c_match = str(key)
try:
cached = dumpTools.loadobj(etpCache['dbSearch']+"/"+self.dbname+"/"+c_match+"/"+c_hash)
if cached != None:
return cached
except EOFError:
pass
def storeSearchCache(self, key, function, search_cache_data, extra_hash = 0):
if self.xcache:
c_hash = str(hash(function)) + str(extra_hash)
c_match = str(key)
try:
sperms = False
if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbSearch']+"/"+self.dbname)):
sperms = True
dumpTools.dumpobj(etpCache['dbSearch']+"/"+self.dbname+"/"+c_match+"/"+c_hash,search_cache_data)
if sperms:
const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid'])
except IOError:
pass
def retrieveRepositoryUpdatesDigest(self, repository):
self.cursor.execute('SELECT digest FROM treeupdates WHERE repository = (?)', (repository,))
mydigest = self.cursor.fetchone()
if mydigest:
return mydigest[0]
else:
return -1
def listAllTreeUpdatesActions(self):
self.cursor.execute('SELECT * FROM treeupdatesactions')
return self.cursor.fetchall()
def retrieveTreeUpdatesActions(self, repository, forbranch = etpConst['branch']):
if not self.doesTableExist("treeupdatesactions"):
return set()
self.cursor.execute('SELECT command FROM treeupdatesactions where repository = (?) and branch = (?)', (repository,forbranch))
return self.fetchall2set(self.cursor.fetchall())
# mainly used to restore a previous table, used by reagent in --initialize
def addTreeUpdatesActions(self, updates):
for update in updates:
idupdate = update[0]
repository = update[1]
command = update[2]
branch = etpConst['branch']
self.cursor.execute('INSERT INTO treeupdatesactions VALUES (?,?,?,?)', (idupdate,repository,command,branch,))
def setRepositoryUpdatesDigest(self, repository, digest):
self.cursor.execute('DELETE FROM treeupdates where repository = (?)', (repository,)) # doing it for safety
self.cursor.execute('INSERT INTO treeupdates VALUES (?,?)', (repository,digest,))
self.commitChanges()
def addRepositoryUpdatesActions(self, repository, actions, forbranch = etpConst['branch']):
for command in actions:
self.cursor.execute('INSERT INTO treeupdatesactions VALUES (NULL,?,?,?)', (repository,command,forbranch,))
def retrieveAtom(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveAtom')
#if cache != None: return cache
self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,))
atom = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveAtom',atom)
return atom
def retrieveBranch(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveBranch')
if cache != None: return cache
self.cursor.execute('SELECT branch FROM baseinfo WHERE idpackage = (?)', (idpackage,))
br = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveBranch',br)
return br
def retrieveTrigger(self, idpackage):
self.cursor.execute('SELECT data FROM triggers WHERE idpackage = (?)', (idpackage,))
trigger = self.cursor.fetchone()
if trigger:
trigger = trigger[0]
else:
trigger = ''
return trigger
def retrieveDownloadURL(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveDownloadURL')
if cache != None: return cache
self.cursor.execute('SELECT download FROM extrainfo WHERE idpackage = (?)', (idpackage,))
download = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveDownloadURL',download)
return download
def retrieveDescription(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveDescription')
if cache != None: return cache
self.cursor.execute('SELECT description FROM extrainfo WHERE idpackage = (?)', (idpackage,))
description = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveDescription',description)
return description
def retrieveHomepage(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveHomepage')
if cache != None: return cache
self.cursor.execute('SELECT homepage FROM extrainfo WHERE idpackage = (?)', (idpackage,))
home = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveHomepage',home)
return home
def retrieveCounter(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveCounter')
#if cache != None: return cache
counter = -1
self.cursor.execute('SELECT counter FROM counters WHERE idpackage = (?)', (idpackage,))
mycounter = self.cursor.fetchone()
if mycounter:
counter = mycounter[0]
#self.storeInfoCache(idpackage,'retrieveCounter',int(counter))
return int(counter)
def retrieveMessages(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveMessages')
if cache != None: return cache
messages = []
try:
self.cursor.execute('SELECT message FROM messages WHERE idpackage = (?)', (idpackage,))
messages = self.fetchall2list(self.cursor.fetchall())
except:
pass
self.storeInfoCache(idpackage,'retrieveMessages',messages)
return messages
# in bytes
def retrieveSize(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveSize')
#if cache != None: return cache
self.cursor.execute('SELECT size FROM extrainfo WHERE idpackage = (?)', (idpackage,))
size = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveSize',size)
return size
# in bytes
def retrieveOnDiskSize(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveOnDiskSize')
#if cache != None: return cache
self.cursor.execute('SELECT size FROM sizes WHERE idpackage = (?)', (idpackage,))
size = self.cursor.fetchone() # do not use [0]!
if not size:
size = 0
else:
size = size[0]
#self.storeInfoCache(idpackage,'retrieveOnDiskSize',size)
return size
def retrieveDigest(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveDigest')
#if cache != None: return cache
self.cursor.execute('SELECT digest FROM extrainfo WHERE idpackage = (?)', (idpackage,))
digest = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveDigest',digest)
return digest
def retrieveName(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveName')
#if cache != None: return cache
self.cursor.execute('SELECT name FROM baseinfo WHERE idpackage = (?)', (idpackage,))
name = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveName',name)
return name
def retrieveKeySlot(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveKey')
if cache != None: return cache
self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
data = self.cursor.fetchone()
self.storeInfoCache(idpackage,'retrieveKey',data)
return data
def retrieveVersion(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveVersion')
#if cache != None: return cache
self.cursor.execute('SELECT version FROM baseinfo WHERE idpackage = (?)', (idpackage,))
ver = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveVersion',ver)
return ver
def retrieveRevision(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveRevision')
#if cache != None: return cache
self.cursor.execute('SELECT revision FROM baseinfo WHERE idpackage = (?)', (idpackage,))
rev = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveRevision',rev)
return rev
def retrieveDateCreation(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveDateCreation')
#if cache != None: return cache
self.cursor.execute('SELECT "datecreation" FROM extrainfo WHERE idpackage = (?)', (idpackage,))
date = self.cursor.fetchone()[0]
if not date:
date = "N/A" #FIXME: to be removed?
#self.storeInfoCache(idpackage,'retrieveDateCreation',date)
return date
def retrieveApi(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveApi')
#if cache != None: return cache
self.cursor.execute('SELECT "etpapi" FROM baseinfo WHERE idpackage = (?)', (idpackage,))
api = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveApi',api)
return api
def retrieveUseflags(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveUseflags')
if cache != None: return cache
self.cursor.execute('SELECT flagname FROM useflags,useflagsreference WHERE useflags.idpackage = (?) and useflags.idflag = useflagsreference.idflag', (idpackage,))
flags = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveUseflags',flags)
return flags
def retrieveEclasses(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveEclasses')
if cache != None: return cache
self.cursor.execute('SELECT classname FROM eclasses,eclassesreference WHERE eclasses.idpackage = (?) and eclasses.idclass = eclassesreference.idclass', (idpackage,))
classes = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveEclasses',classes)
return classes
def retrieveNeeded(self, idpackage, extended = False):
#cache = self.fetchInfoCache(idpackage,'retrieveNeeded')
#if cache != None: return cache
if extended:
self.cursor.execute('SELECT library,elfclass FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,))
needed = self.cursor.fetchall()
else:
self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = (?) and needed.idneeded = neededreference.idneeded', (idpackage,))
needed = self.fetchall2set(self.cursor.fetchall())
#self.storeInfoCache(idpackage,'retrieveNeeded',needed)
return needed
def retrieveConflicts(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveConflicts')
if cache != None: return cache
self.cursor.execute('SELECT "conflict" FROM conflicts WHERE idpackage = (?)', (idpackage,))
confl = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveConflicts',confl)
return confl
def retrieveProvide(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveProvide')
#if cache != None: return cache
self.cursor.execute('SELECT atom FROM provide WHERE idpackage = (?)', (idpackage,))
provide = self.fetchall2set(self.cursor.fetchall())
#self.storeInfoCache(idpackage,'retrieveProvide',provide)
return provide
def retrieveDependenciesList(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveDependenciesList')
if cache != None: return cache
deps = self.retrieveDependencies(idpackage)
conflicts = self.retrieveConflicts(idpackage)
for x in conflicts:
if x[0] != "!":
x = "!"+x
deps.add(x)
del conflicts
self.storeInfoCache(idpackage,'retrieveDependenciesList',deps)
return deps
def retrieveDependencies(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveDependencies')
if cache != None: return cache
self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE dependencies.idpackage = (?) and dependencies.iddependency = dependenciesreference.iddependency', (idpackage,))
deps = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveDependencies',deps)
return deps
def retrieveIdDependencies(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveIdDependencies')
if cache != None: return cache
self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = (?)', (idpackage,))
iddeps = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveIdDependencies',iddeps)
return iddeps
def retrieveDependencyFromIddependency(self, iddependency):
self.cursor.execute('SELECT dependency FROM dependenciesreference WHERE iddependency = (?)', (iddependency,))
return self.cursor.fetchone()[0]
def retrieveKeywords(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveKeywords')
if cache != None: return cache
self.cursor.execute('SELECT keywordname FROM keywords,keywordsreference WHERE keywords.idpackage = (?) and keywords.idkeyword = keywordsreference.idkeyword', (idpackage,))
kw = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveKeywords',kw)
return kw
def retrieveProtect(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveProtect')
if cache != None: return cache
self.cursor.execute('SELECT protect FROM configprotect,configprotectreference WHERE configprotect.idpackage = (?) and configprotect.idprotect = configprotectreference.idprotect', (idpackage,))
protect = self.cursor.fetchone()
if not protect:
protect = ''
else:
protect = protect[0]
self.storeInfoCache(idpackage,'retrieveProtect',protect)
return protect
def retrieveProtectMask(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveProtectMask')
if cache != None: return cache
self.cursor.execute('SELECT protect FROM configprotectmask,configprotectreference WHERE idpackage = (?) and configprotectmask.idprotect= configprotectreference.idprotect', (idpackage,))
protect = self.cursor.fetchone()
if not protect:
protect = ''
else:
protect = protect[0]
self.storeInfoCache(idpackage,'retrieveProtectMask',protect)
return protect
def retrieveSources(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveSources')
if cache != None: return cache
self.cursor.execute('SELECT sourcesreference.source FROM sources,sourcesreference WHERE idpackage = (?) and sources.idsource = sourcesreference.idsource', (idpackage,))
sources = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveSources',sources)
return sources
def retrieveContent(self, idpackage, extended = False, contentType = None):
c_hash = -4
if extended:
c_hash += 2
c_hash += hash(contentType)
cache = self.fetchInfoCache(idpackage,'retrieveContent', extra_hash = c_hash)
if cache != None: return cache
# like portage does
self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
extstring = ''
if extended:
extstring = ",type"
searchkeywords = [idpackage]
contentstring = ''
if contentType:
searchkeywords.append(contentType)
contentstring = ' and type = (?)'
self.cursor.execute('SELECT file'+extstring+' FROM content WHERE idpackage = (?) '+contentstring, searchkeywords)
if extended:
fl = self.cursor.fetchall()
else:
fl = self.fetchall2set(self.cursor.fetchall())
self.storeInfoCache(idpackage,'retrieveContent',fl, extra_hash = c_hash)
return fl
def retrieveSlot(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveSlot')
#if cache != None: return cache
self.cursor.execute('SELECT slot FROM baseinfo WHERE idpackage = (?)', (idpackage,))
ver = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveSlot',ver)
return ver
def retrieveVersionTag(self, idpackage):
#cache = self.fetchInfoCache(idpackage,'retrieveVersionTag')
#if cache != None: return cache
self.cursor.execute('SELECT versiontag FROM baseinfo WHERE idpackage = (?)', (idpackage,))
vtag = self.cursor.fetchone()[0]
#self.storeInfoCache(idpackage,'retrieveVersionTag',ver)
return vtag
def retrieveMirrorInfo(self, mirrorname):
self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,))
mirrorlist = self.fetchall2set(self.cursor.fetchall())
return mirrorlist
def retrieveCategory(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveCategory')
if cache != None: return cache
self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
cat = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveCategory',cat)
return cat
def retrieveLicensedata(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveLicensedata')
if cache != None: return cache
# insert license information
if not self.doesTableExist("licensedata"):
return {}
licenses = self.retrieveLicense(idpackage)
licenses = licenses.split()
licdata = {}
for licname in licenses:
licname = licname.strip()
if not self.entropyTools.is_valid_string(licname):
continue
self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,))
lictext = self.cursor.fetchone()
if lictext != None:
#licdata[licname] = unicode(lictext[0],'raw_unicode_escape','replace')
licdata[licname] = str(lictext[0])
self.storeInfoCache(idpackage,'retrieveLicensedata',licdata)
return licdata
def retrieveLicensedataKeys(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveLicensedataKeys')
if cache != None: return cache
if not self.doesTableExist("licensedata"):
return set()
licenses = self.retrieveLicense(idpackage)
licenses = licenses.split()
licdata = set()
for licname in licenses:
licname = licname.strip()
if not self.entropyTools.is_valid_string(licname):
continue
self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,))
licidentifier = self.cursor.fetchone()
if licidentifier:
licdata.add(licidentifier[0])
self.storeInfoCache(idpackage,'retrieveLicensedataKeys',licdata)
return licdata
def retrieveLicenseText(self, license_name):
if not self.doesTableExist("licensedata"):
return None
self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (license_name,))
text = self.cursor.fetchone()
if not text:
return None
#return unicode(text[0],'raw_unicode_escape','replace')
return str(text[0])
def retrieveLicense(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveLicense')
if cache != None: return cache
self.cursor.execute('SELECT license FROM baseinfo,licenses WHERE baseinfo.idpackage = (?) and baseinfo.idlicense = licenses.idlicense', (idpackage,))
licname = self.cursor.fetchone()[0]
self.storeInfoCache(idpackage,'retrieveLicense',licname)
return licname
def retrieveCompileFlags(self, idpackage):
cache = self.fetchInfoCache(idpackage,'retrieveCompileFlags')
if cache != None: return cache
self.cursor.execute('SELECT "idflags" FROM extrainfo WHERE idpackage = (?)', (idpackage,))
idflag = self.cursor.fetchone()[0]
# now get the flags
self.cursor.execute('SELECT chost,cflags,cxxflags FROM flags WHERE idflags = (?)', (idflag,))
flags = self.cursor.fetchone()
if not flags:
flags = ("N/A","N/A","N/A")
self.storeInfoCache(idpackage,'retrieveCompileFlags',flags)
return flags
def retrieveDepends(self, idpackage, atoms = False, key_slot = False):
# sanity check on the table
if not self.isDependsTableSane(): # is empty, need generation
self.regenerateDependsTable(output = False)
if atoms:
self.cursor.execute('SELECT baseinfo.atom FROM dependstable,dependencies,baseinfo WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency and baseinfo.idpackage = dependencies.idpackage', (idpackage,))
result = self.fetchall2set(self.cursor.fetchall())
elif key_slot:
self.cursor.execute('SELECT categories.category || "/" || baseinfo.name,baseinfo.slot FROM baseinfo,categories,dependstable,dependencies WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency and baseinfo.idpackage = dependencies.idpackage and categories.idcategory = baseinfo.idcategory', (idpackage,))
result = self.cursor.fetchall()
else:
self.cursor.execute('SELECT dependencies.idpackage FROM dependstable,dependencies WHERE dependstable.idpackage = (?) and dependstable.iddependency = dependencies.iddependency', (idpackage,))
result = self.fetchall2set(self.cursor.fetchall())
return result
# You must provide the full atom to this function
# WARNING: this function does not support branches
# NOTE: server side uses this regardless branch specification because it already handles it in updatePackage()
def isPackageAvailable(self,pkgatom):
pkgatom = self.entropyTools.removePackageOperators(pkgatom)
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?)', (pkgatom,))
result = self.cursor.fetchone()
if result:
return result[0]
return -1
def isIDPackageAvailable(self,idpackage):
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = (?)', (idpackage,))
result = self.cursor.fetchone()
if not result:
return False
return True
# This version is more specific and supports branches
def isSpecificPackageAvailable(self, pkgkey, branch, branch_operator = "="):
pkgkey = self.entropyTools.removePackageOperators(pkgkey)
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = (?) AND branch '+branch_operator+' (?)', (pkgkey,branch,))
result = self.cursor.fetchone()
if not result:
return False
return True
def isCategoryAvailable(self,category):
self.cursor.execute('SELECT idcategory FROM categories WHERE category = (?)', (category,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isProtectAvailable(self,protect):
self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = (?)', (protect,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isFileAvailable(self, myfile):
self.cursor.execute('SELECT idpackage FROM content WHERE file = (?)', (myfile,))
result = self.cursor.fetchone()
rc = False
if result:
rc = True
return rc
def resolveNeeded(self, needed, elfclass = -1):
cache = self.fetchSearchCache(needed,'resolveNeeded')
if cache != None: return cache
ldpaths = self.entropyTools.collectLinkerPaths()
mypaths = [os.path.join(x,needed) for x in ldpaths]
query = """
SELECT
idpackage,file
FROM
content
WHERE
content.file IN (%s)
""" % ( ('?,'*len(mypaths))[:-1], )
self.cursor.execute(query,mypaths)
results = self.cursor.fetchall()
if elfclass == -1:
mydata = set(results)
else:
mydata = set()
for data in results:
if not os.access(data[1],os.R_OK):
continue
myclass = self.entropyTools.read_elf_class(data[1])
if myclass == elfclass:
mydata.add(data)
self.storeSearchCache(needed,'resolveNeeded',mydata)
return mydata
def isSourceAvailable(self,source):
self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = "'+source+'"')
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isDependencyAvailable(self,dependency):
self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = (?)', (dependency,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isKeywordAvailable(self,keyword):
self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = (?)', (keyword,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isUseflagAvailable(self,useflag):
self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = (?)', (useflag,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isEclassAvailable(self,eclass):
self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = (?)', (eclass,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isNeededAvailable(self,needed):
self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = (?)', (needed,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isCounterAvailable(self,counter, branch = None, branch_operator = "="):
result = False
if not branch:
branch = etpConst['branch']
self.cursor.execute('SELECT counter FROM counters WHERE counter = (?) and branch '+branch_operator+' (?)', (counter,branch,))
result = self.cursor.fetchone()
if result:
result = True
return result
def isLicensedataKeyAvailable(self, license_name):
if not self.doesTableExist("licensedata"):
return True
self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,))
result = self.cursor.fetchone()
if not result:
return False
return True
def isLicenseAccepted(self, license_name):
self.cursor.execute('SELECT licensename FROM licenses_accepted WHERE licensename = (?)', (license_name,))
result = self.cursor.fetchone()
if not result:
return False
return True
def acceptLicense(self, license_name):
if self.readOnly or (not self.entropyTools.is_user_in_entropy_group()):
return
if self.isLicenseAccepted(license_name):
return
self.cursor.execute('INSERT INTO licenses_accepted VALUES (?)', (license_name,))
self.commitChanges()
def isLicenseAvailable(self,pkglicense):
if not self.entropyTools.is_valid_string(pkglicense):
pkglicense = ' '
self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def isSystemPackage(self,idpackage):
self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = (?)', (idpackage,))
result = self.cursor.fetchone()
if result:
return True
return False
def isInjected(self,idpackage):
cache = self.fetchInfoCache(idpackage,'isInjected')
if cache != None: return cache
try:
self.cursor.execute('SELECT idpackage FROM injected WHERE idpackage = (?)', (idpackage,))
except:
# readonly database?
return False
result = self.cursor.fetchone()
rslt = False
if result:
rslt = True
self.storeInfoCache(idpackage,'isInjected',rslt)
return rslt
def areCompileFlagsAvailable(self,chost,cflags,cxxflags):
self.cursor.execute('SELECT idflags FROM flags WHERE chost = (?) AND cflags = (?) AND cxxflags = (?)', (chost,cflags,cxxflags,))
result = self.cursor.fetchone()
if not result:
return -1
return result[0]
def searchBelongs(self, file, like = False, branch = None, branch_operator = "="):
branchstring = ''
searchkeywords = [file]
if branch:
searchkeywords.append(branch)
branchstring = ' and baseinfo.branch '+branch_operator+' (?)'
if (like):
self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file LIKE (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords)
else:
self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file = (?) and content.idpackage = baseinfo.idpackage '+branchstring, searchkeywords)
return self.fetchall2set(self.cursor.fetchall())
''' search packages that uses the eclass provided '''
def searchEclassedPackages(self, eclass, atoms = False): # atoms = return atoms directly
if atoms:
self.cursor.execute('SELECT baseinfo.atom,eclasses.idpackage FROM baseinfo,eclasses,eclassesreference WHERE eclassesreference.classname = (?) and eclassesreference.idclass = eclasses.idclass and eclasses.idpackage = baseinfo.idpackage', (eclass,))
return self.cursor.fetchall()
else:
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (eclass,))
return self.fetchall2set(self.cursor.fetchall())
''' search packages whose versiontag matches the one provided '''
def searchTaggedPackages(self, tag, atoms = False): # atoms = return atoms directly
if atoms:
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE versiontag = (?)', (tag,))
return self.cursor.fetchall()
else:
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE versiontag = (?)', (tag,))
return self.fetchall2set(self.cursor.fetchall())
def searchLicenses(self, mylicense, caseSensitive = False, atoms = False):
if not self.entropyTools.is_valid_string(mylicense):
return []
request = "baseinfo.idpackage"
if atoms:
request = "baseinfo.atom,baseinfo.idpackage"
if caseSensitive:
self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE licenses.license LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%",))
else:
self.cursor.execute('SELECT '+request+' FROM baseinfo,licenses WHERE LOWER(licenses.license) LIKE (?) and licenses.idlicense = baseinfo.idlicense', ("%"+mylicense+"%".lower(),))
if atoms:
return self.cursor.fetchall()
return self.fetchall2set(self.cursor.fetchall())
''' search packages whose slot matches the one provided '''
def searchSlottedPackages(self, slot, atoms = False): # atoms = return atoms directly
if atoms:
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE slot = (?)', (slot,))
return self.cursor.fetchall()
else:
self.cursor.execute('SELECT idpackage FROM baseinfo WHERE slot = (?)', (slot,))
return self.fetchall2set(self.cursor.fetchall())
def searchKeySlot(self, key, slot, branch = None):
branchstring = ''
params = [key,slot]
if branch:
params.append(branch)
branchstring = ' and baseinfo.branch = (?)'
self.cursor.execute('SELECT idpackage FROM baseinfo,categories WHERE categories.category || "/" || baseinfo.name = (?) and baseinfo.slot = (?) and baseinfo.idcategory = categories.idcategory'+branchstring, params)
data = self.cursor.fetchall()
return data
''' search packages that need the specified library (in neededreference table) specified by keyword '''
def searchNeeded(self, keyword, like = False):
if like:
self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library LIKE (?) and needed.idneeded = neededreference.idneeded', (keyword,))
else:
self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = (?) and needed.idneeded = neededreference.idneeded', (keyword,))
return self.fetchall2set(self.cursor.fetchall())
# FIXME: deprecate and add functionalities to the function above
''' same as above but with branch support '''
def searchNeededInBranch(self, keyword, branch):
self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference,baseinfo WHERE library = (?) and needed.idneeded = neededreference.idneeded and baseinfo.branch = (?)', (keyword,branch,))
return self.fetchall2set(self.cursor.fetchall())
''' search dependency string inside dependenciesreference table and retrieve iddependency '''
def searchDependency(self, dep, like = False, multi = False):
sign = "="
if like:
sign = "LIKE"
dep = "%"+dep+"%"
self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency '+sign+' (?)', (dep,))
if multi:
return self.fetchall2set(self.cursor.fetchall())
else:
iddep = self.cursor.fetchone()
if iddep:
iddep = iddep[0]
else:
iddep = -1
return iddep
''' search iddependency inside dependencies table and retrieve idpackages '''
def searchIdpackageFromIddependency(self, iddep):
self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,))
return self.fetchall2set(self.cursor.fetchall())
def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None):
searchkeywords = ["%"+keyword+"%"]
slotstring = ''
if slot:
searchkeywords.append(slot)
slotstring = ' and slot = (?)'
tagstring = ''
if tag:
searchkeywords.append(tag)
tagstring = ' and versiontag = (?)'
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and branch = (?)'
if (sensitive):
self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE (?)'+slotstring+tagstring+branchstring, searchkeywords)
else:
self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE (?)'+slotstring+tagstring+branchstring, searchkeywords)
return self.cursor.fetchall()
def searchProvide(self, keyword, slot = None, tag = None, branch = None):
slotstring = ''
searchkeywords = [keyword]
if slot:
searchkeywords.append(slot)
slotstring = ' and baseinfo.slot = (?)'
tagstring = ''
if tag:
searchkeywords.append(tag)
tagstring = ' and baseinfo.versiontag = (?)'
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and baseinfo.branch = (?)'
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,provide WHERE provide.atom = (?) and provide.idpackage = baseinfo.idpackage'+slotstring+tagstring+branchstring, searchkeywords)
results = self.cursor.fetchall()
return results
def searchPackagesByDescription(self, keyword):
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM extrainfo,baseinfo WHERE LOWER(extrainfo.description) LIKE (?) and baseinfo.idpackage = extrainfo.idpackage', ("%"+keyword.lower()+"%",))
return self.cursor.fetchall()
def searchPackagesByName(self, keyword, sensitive = False, branch = None):
if sensitive:
searchkeywords = [keyword]
else:
searchkeywords = [keyword.lower()]
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and branch = (?)'
if sensitive:
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE name = (?)'+branchstring, searchkeywords)
else:
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE LOWER(name) = (?)'+branchstring, searchkeywords)
results = self.cursor.fetchall()
return results
def searchPackagesByCategory(self, keyword, like = False, branch = None):
searchkeywords = [keyword]
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and branch = (?)'
if like:
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category LIKE (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)
else:
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)
results = self.cursor.fetchall()
return results
def searchPackagesByNameAndCategory(self, name, category, sensitive = False, branch = None):
namestring = 'baseinfo.name'
catstring = 'categories.category'
myname = name
mycat = category
if not sensitive:
namestring = 'LOWER(baseinfo.name)'
catstring = 'LOWER(categories.category)'
myname = name.lower()
mycat = category.lower()
searchkeywords = [myname,mycat]
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and baseinfo.branch = (?)'
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE '+namestring+' = (?) AND '+catstring+' = (?) and baseinfo.idcategory = categories.idcategory'+branchstring, searchkeywords)
results = self.cursor.fetchall()
return results
def searchPackagesKeyVersion(self, key, version, branch = None, sensitive = False):
searchkeywords = []
if sensitive:
searchkeywords.append(key)
else:
searchkeywords.append(key.lower())
searchkeywords.append(version)
branchstring = ''
if branch:
searchkeywords.append(branch)
branchstring = ' and branch = (?)'
if (sensitive):
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE categories.category || "/" || baseinfo.name = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)
else:
self.cursor.execute('SELECT baseinfo.atom,baseinfo.idpackage FROM baseinfo,categories WHERE LOWER(categories.category) || "/" || LOWER(baseinfo.name) = (?) and version = (?) and baseinfo.idcategory = categories.idcategory '+branchstring, searchkeywords)
results = self.cursor.fetchall()
return results
def listAllPackages(self):
self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo')
return self.cursor.fetchall()
def listAllInjectedPackages(self, justFiles = False):
self.cursor.execute('SELECT idpackage FROM injected')
injecteds = self.fetchall2set(self.cursor.fetchall())
results = set()
# get download
for injected in injecteds:
download = self.retrieveDownloadURL(injected)
if justFiles:
results.add(download)
else:
results.add((download,injected))
return results
def listAllCounters(self, onlycounters = False, branch = None, branch_operator = "="):
branchstring = ''
if branch:
branchstring = ' WHERE branch '+branch_operator+' "'+str(branch)+'"'
if onlycounters:
self.cursor.execute('SELECT counter FROM counters'+branchstring)
return self.fetchall2set(self.cursor.fetchall())
else:
self.cursor.execute('SELECT counter,idpackage FROM counters'+branchstring)
return self.cursor.fetchall()
def listAllIdpackages(self, branch = None, branch_operator = "="):
# XXX if you will decide to re-enable this, consider that you have to clean this cache
# each time a package is updated/installed/removed
#cache = self.fetchInfoCache(hash(branch),'listAllIdpackages')
#if cache != None: return cache
branchstring = ''
searchkeywords = []
if branch:
searchkeywords.append(branch)
branchstring = ' where branch %s (?)' % (str(branch_operator),)
self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring, searchkeywords)
results = self.fetchall2set(self.cursor.fetchall())
#self.storeInfoCache(hash(branch),'listAllIdpackages',results)
return results
def listAllDependencies(self):
self.cursor.execute('SELECT * FROM dependenciesreference')
return self.cursor.fetchall()
def listAllBranches(self):
cache = self.live_cache.get('listAllBranches')
if cache != None:
return cache
self.cursor.execute('SELECT distinct branch FROM baseinfo')
results = self.fetchall2set(self.cursor.fetchall())
self.live_cache['listAllBranches'] = results.copy()
return results
def listIdPackagesInIdcategory(self,idcategory):
self.cursor.execute('SELECT idpackage FROM baseinfo where idcategory = (?)', (idcategory,))
return self.fetchall2set(self.cursor.fetchall())
def listIdpackageDependencies(self, idpackage):
self.cursor.execute('SELECT dependenciesreference.iddependency,dependenciesreference.dependency FROM dependenciesreference,dependencies WHERE dependencies.idpackage = (?) AND dependenciesreference.iddependency = dependencies.iddependency', (idpackage,))
return set(self.cursor.fetchall())
def listBranchPackagesTbz2(self, branch, do_sort = True):
self.cursor.execute('SELECT extrainfo.download FROM baseinfo,extrainfo WHERE baseinfo.branch = (?) AND baseinfo.idpackage = extrainfo.idpackage', (branch,))
result = self.fetchall2set(self.cursor.fetchall())
sorted_result = []
for package in result:
if package:
sorted_result.append(os.path.basename(package))
if do_sort:
sorted_result.sort()
return sorted_result
def listBranchPackages(self, branch):
self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE branch = (?)', (branch,))
return self.cursor.fetchall()
def listAllFiles(self, clean = False):
self.cursor.execute('SELECT file FROM content')
if clean:
return self.fetchall2set(self.cursor.fetchall())
else:
return self.fetchall2list(self.cursor.fetchall())
def listAllCategories(self):
self.cursor.execute('SELECT idcategory,category FROM categories')
return self.cursor.fetchall()
def listConfigProtectDirectories(self, mask = False):
query = 'SELECT max(idprotect) FROM configprotect'
if mask:
query += 'mask'
self.cursor.execute(query)
r = self.cursor.fetchone()
if not r:
return []
mymax = r[0]
self.cursor.execute('SELECT protect FROM configprotectreference where idprotect >= (?) and idprotect <= (?) order by protect', (1,mymax,))
results = self.cursor.fetchall()
dirs = []
for row in results:
mydirs = row[0].split()
for x in mydirs:
if x not in dirs:
dirs.append(x)
return dirs
def switchBranch(self, idpackage, tobranch):
self.checkReadOnly()
mycat = self.retrieveCategory(idpackage)
myname = self.retrieveName(idpackage)
myslot = self.retrieveSlot(idpackage)
mybranch = self.retrieveBranch(idpackage)
mydownload = self.retrieveDownloadURL(idpackage)
import re
out = re.subn('/'+mybranch+'/','/'+tobranch+'/',mydownload)
newdownload = out[0]
# remove package with the same key+slot and tobranch if exists
match = self.atomMatch(mycat+"/"+myname, matchSlot = myslot, matchBranches = (tobranch,))
if match[0] != -1:
self.removePackage(match[0])
# now switch selected idpackage to the new branch
self.cursor.execute('UPDATE baseinfo SET branch = (?) WHERE idpackage = (?)', (tobranch,idpackage,))
self.cursor.execute('UPDATE extrainfo SET download = (?) WHERE idpackage = (?)', (newdownload,idpackage,))
self.commitChanges()
def databaseStructureUpdates(self):
if not self.doesTableExist("licensedata"):
self.createLicensedataTable()
if not self.doesTableExist("licenses_accepted") and (self.dbname == etpConst['clientdbid']):
self.createLicensesAcceptedTable()
if not self.doesColumnInTableExist("baseinfo","trigger"):
self.createTriggerColumn()
if not self.doesTableExist("counters"):
self.createCountersTable()
elif not self.doesColumnInTableExist("counters","branch"):
self.createCountersBranchColumn()
if not self.doesTableExist("sizes"):
self.createSizesTable()
if not self.doesTableExist("triggers"):
self.createTriggerTable()
if not self.doesTableExist("messages"):
self.createMessagesTable()
if not self.doesTableExist("injected"):
self.createInjectedTable()
if not self.doesTableExist("systempackages"):
self.createSystemPackagesTable()
if (not self.doesTableExist("configprotect")) or (not self.doesTableExist("configprotectreference")):
self.createProtectTable()
if not self.doesColumnInTableExist("content","type"):
self.createContentTypeColumn()
if not self.doesTableExist("eclasses"):
self.createEclassesTable()
if not self.doesTableExist("treeupdates"):
self.createTreeupdatesTable()
if not self.doesTableExist("treeupdatesactions"):
self.createTreeupdatesactionsTable()
elif not self.doesColumnInTableExist("treeupdatesactions","branch"):
self.createTreeupdatesactionsBranchColumn()
if not self.doesTableExist("needed"):
self.createNeededTable()
elif not self.doesColumnInTableExist("needed","elfclass"):
self.createNeededElfclassColumn()
if not self.doesTableExist("installedtable") and (self.dbname == etpConst['clientdbid']):
self.createInstalledTable()
# these are the tables moved to INTEGER PRIMARY KEY AUTOINCREMENT
autoincrement_tables = [
'treeupdatesactions',
'neededreference',
'eclassesreference',
'configprotectreference',
'flags',
'licenses',
'categories',
'keywordsreference',
'useflagsreference',
'sourcesreference',
'dependenciesreference',
'baseinfo'
]
autoinc = False
for table in autoincrement_tables:
x = self.migrateTableToAutoincrement(table)
if x: autoinc = True
if autoinc:
self.updateProgress(
red("Entropy database: regenerating indexes after migration."),
importance = 1,
type = "warning",
header = blue(" !!! ")
)
self.createAllIndexes()
self.connection.commit()
def migrateTableToAutoincrement(self, table):
self.cursor.execute('select sql from sqlite_master where type = (?) and name = (?);', ("table",table))
schema = self.cursor.fetchone()
if not schema:
return False
schema = schema[0]
if schema.find("AUTOINCREMENT") != -1:
return False
schema = schema.replace('PRIMARY KEY','PRIMARY KEY AUTOINCREMENT')
new_schema = schema
totable = table+"_autoincrement"
schema = schema.replace('CREATE TABLE '+table,'CREATE TEMPORARY TABLE '+totable)
self.updateProgress(
red("Entropy database: migrating table ")+blue(table),
importance = 1,
type = "warning",
header = blue(" !!! ")
)
# create table
self.cursor.execute('DROP TABLE IF EXISTS '+totable)
self.cursor.execute(schema)
columns = ','.join(self.getColumnsInTable(table))
temp_query = 'INSERT INTO '+totable+' SELECT '+columns+' FROM '+table
self.cursor.execute(temp_query)
self.cursor.execute('DROP TABLE '+table)
self.cursor.execute(new_schema)
temp_query = 'INSERT INTO '+table+' SELECT '+columns+' FROM '+totable
self.cursor.execute(temp_query)
self.cursor.execute('DROP TABLE '+totable)
self.commitChanges()
return True
def validateDatabase(self):
self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","baseinfo"))
rslt = self.cursor.fetchone()
if rslt == None:
raise exceptionTools.SystemDatabaseError("SystemDatabaseError: table baseinfo not found. Either does not exist or corrupted.")
self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table","extrainfo"))
rslt = self.cursor.fetchone()
if rslt == None:
raise exceptionTools.SystemDatabaseError("SystemDatabaseError: table extrainfo not found. Either does not exist or corrupted.")
def checkDatabaseApi(self):
dbapi = self.getApi()
if dbapi > etpConst['etpapi']:
self.updateProgress(
red("Repository EAPI > Entropy EAPI. Please update Equo/Entropy as soon as possible !"),
importance = 1,
type = "warning",
header = " * ! * ! * ! * "
)
def doDatabaseImport(self, dumpfile, dbfile):
import subprocess
sqlite3_exec = "/usr/bin/sqlite3 %s < %s" % (dbfile,dumpfile,)
retcode = subprocess.call(sqlite3_exec, shell = True)
return retcode
def doDatabaseExport(self, dumpfile):
dumpfile.write("BEGIN TRANSACTION;\n")
self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type=='table'")
for name, x, sql in self.cursor.fetchall():
self.updateProgress(
red("Exporting database table ")+"["+blue(str(name))+"]",
importance = 0,
type = "info",
back = True,
header = " "
)
if name == "sqlite_sequence":
dumpfile.write("DELETE FROM sqlite_sequence;\n")
elif name == "sqlite_stat1":
dumpfile.write("ANALYZE sqlite_master;\n")
elif name.startswith("sqlite_"):
continue
else:
dumpfile.write("%s;\n" % sql)
self.cursor.execute("PRAGMA table_info('%s')" % name)
cols = [str(r[1]) for r in self.cursor.fetchall()]
q = "SELECT 'INSERT INTO \"%(tbl_name)s\" VALUES("
q += ", ".join(["'||quote(" + x + ")||'" for x in cols])
q += ")' FROM '%(tbl_name)s'"
self.cursor.execute(q % {'tbl_name': name})
self.connection.text_factory = lambda x: unicode(x, "raw_unicode_escape")
for row in self.cursor:
dumpfile.write("%s;\n" % str(row[0].encode('raw_unicode_escape')))
self.cursor.execute("SELECT name, type, sql FROM sqlite_master WHERE sql NOT NULL AND type!='table' AND type!='meta'")
for name, x, sql in self.cursor.fetchall():
dumpfile.write("%s;\n" % sql)
dumpfile.write("COMMIT;\n")
try:
dumpfile.flush()
except:
pass
self.updateProgress(
red("Database Export completed."),
importance = 0,
type = "info",
header = " "
)
# remember to close the file
# FIXME: this is only compatible with SQLITE
def doesTableExist(self, table):
self.cursor.execute('select name from SQLITE_MASTER where type = (?) and name = (?)', ("table",table))
rslt = self.cursor.fetchone()
if rslt == None:
return False
return True
# FIXME: this is only compatible with SQLITE
def doesColumnInTableExist(self, table, column):
self.cursor.execute('PRAGMA table_info( '+table+' )')
rslt = self.cursor.fetchall()
if not rslt:
return False
found = False
for row in rslt:
if row[1] == column:
found = True
break
return found
# FIXME: this is only compatible with SQLITE
def getColumnsInTable(self, table):
self.cursor.execute('PRAGMA table_info( '+table+' )')
rslt = self.cursor.fetchall()
columns = []
for row in rslt:
columns.append(row[1])
return columns
def database_checksum(self):
# primary keys are now autoincrement
self.cursor.execute('select idpackage from baseinfo')
c_hash = hash(tuple(self.cursor.fetchall()))
return str(c_hash)
########################################################
####
## Client Database API / but also used by server part
#
def addPackageToInstalledTable(self, idpackage, repositoryName):
self.checkReadOnly()
self.cursor.execute(
'INSERT into installedtable VALUES '
'(?,?)'
, ( idpackage,
repositoryName,
)
)
self.commitChanges()
def retrievePackageFromInstalledTable(self, idpackage):
self.checkReadOnly()
result = 'Not available'
try:
self.cursor.execute('SELECT repositoryname FROM installedtable WHERE idpackage = (?)', (idpackage,))
return self.cursor.fetchone()[0] # it's ok because it's inside try/except
except:
pass
return result
def removePackageFromInstalledTable(self, idpackage):
self.cursor.execute('DELETE FROM installedtable WHERE idpackage = (?)', (idpackage,))
def removePackageFromDependsTable(self, idpackage):
try:
self.cursor.execute('DELETE FROM dependstable WHERE idpackage = (?)', (idpackage,))
return 0
except:
return 1 # need reinit
def removeDependencyFromDependsTable(self, iddependency):
self.checkReadOnly()
try:
self.cursor.execute('DELETE FROM dependstable WHERE iddependency = (?)',(iddependency,))
self.commitChanges()
return 0
except:
return 1 # need reinit
# temporary/compat functions
def createDependsTable(self):
self.checkReadOnly()
self.cursor.execute('DROP TABLE IF EXISTS dependstable;')
self.cursor.execute('CREATE TABLE dependstable ( iddependency INTEGER PRIMARY KEY, idpackage INTEGER );')
# this will be removed when dependstable is refilled properly
self.cursor.execute(
'INSERT into dependstable VALUES '
'(?,?)'
, ( -1,
-1,
)
)
self.commitChanges()
def sanitizeDependsTable(self):
self.cursor.execute('DELETE FROM dependstable where iddependency = -1')
self.commitChanges()
def isDependsTableSane(self):
try:
self.cursor.execute('SELECT iddependency FROM dependstable WHERE iddependency = -1')
except:
return False # table does not exist, please regenerate and re-run
status = self.cursor.fetchone()
if status:
return False
self.cursor.execute('select count(*) from dependstable')
dependstable_count = self.cursor.fetchone()
if dependstable_count == 0:
return False
return True
def createXpakTable(self):
self.checkReadOnly()
self.cursor.execute('CREATE TABLE xpakdata ( idpackage INTEGER PRIMARY KEY, data BLOB );')
self.commitChanges()
def storeXpakMetadata(self, idpackage, blob):
self.cursor.execute(
'INSERT into xpakdata VALUES '
'(?,?)', ( int(idpackage), buffer(blob), )
)
self.commitChanges()
def retrieveXpakMetadata(self, idpackage):
try:
self.cursor.execute('SELECT data from xpakdata where idpackage = (?)', (idpackage,))
mydata = self.cursor.fetchone()
if not mydata:
return ""
else:
return mydata[0]
except:
return ""
pass
def createCountersTable(self):
self.cursor.execute('DROP TABLE IF EXISTS counters;')
self.cursor.execute('CREATE TABLE counters ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );')
def CreatePackedDataTable(self):
self.cursor.execute('CREATE TABLE packed_data ( idpack INTEGER PRIMARY KEY, data BLOB );')
def dropAllIndexes(self):
self.cursor.execute('SELECT name FROM SQLITE_MASTER WHERE type = "index"')
indexes = self.fetchall2set(self.cursor.fetchall())
for index in indexes:
if not index.startswith("sqlite"):
self.cursor.execute('DROP INDEX IF EXISTS %s' % (index,))
def createAllIndexes(self):
self.createContentIndex()
self.createBaseinfoIndex()
self.createKeywordsIndex()
self.createDependenciesIndex()
self.createExtrainfoIndex()
self.createNeededIndex()
self.createUseflagsIndex()
self.createLicensedataIndex()
self.createLicensesIndex()
self.createConfigProtectReferenceIndex()
def createNeededIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded )')
self.commitChanges()
def createUseflagsIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS useflagsindex ON useflagsreference ( flagname )')
self.commitChanges()
def createContentIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex_couple ON content ( idpackage )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS contentindex_file ON content ( file )')
self.commitChanges()
def createConfigProtectReferenceIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS configprotectreferenceindex ON configprotectreference ( protect )')
self.commitChanges()
def createBaseinfoIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_atom ON baseinfo ( atom )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_branch_name ON baseinfo ( name,branch )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_branch_name_idcategory ON baseinfo ( name,idcategory,branch )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex_idcategory ON baseinfo ( idcategory )')
self.commitChanges()
def createLicensedataIndex(self):
if self.indexing:
if not self.doesTableExist("licensedata"):
return
self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )')
self.commitChanges()
def createLicensesIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS licensesindex ON licenses ( license )')
self.commitChanges()
def createKeywordsIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsreferenceindex ON keywordsreference ( keywordname )')
self.commitChanges()
def createDependenciesIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesindex_idpackage_iddependency ON dependencies ( idpackage, iddependency )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS dependenciesreferenceindex_dependency ON dependenciesreference ( dependency )')
self.commitChanges()
def createExtrainfoIndex(self):
if self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS extrainfoindex ON extrainfo ( description )')
self.commitChanges()
def regenerateCountersTable(self, vdb_path, output = False):
self.checkReadOnly()
self.createCountersTable()
# assign a counter to an idpackage
myids = self.listAllIdpackages()
for myid in myids:
# get atom
myatom = self.retrieveAtom(myid)
mybranch = self.retrieveBranch(myid)
myatom = self.entropyTools.remove_tag(myatom)
myatomcounterpath = vdb_path+myatom+"/"+etpConst['spm']['xpak_entries']['counter']
if os.path.isfile(myatomcounterpath):
try:
f = open(myatomcounterpath,"r")
counter = int(f.readline().strip())
f.close()
except:
if output: self.updateProgress(red("ATTENTION: cannot open Gentoo counter file for: %s") % (bold(myatom),), importance = 1, type = "warning")
continue
# insert id+counter
try:
self.cursor.execute(
'INSERT into counters VALUES '
'(?,?,?)', ( counter, myid, mybranch )
)
except:
if output: self.updateProgress(red("ATTENTION: counter for atom %s")+red(" is duplicated. Ignoring.") % (bold(myatom),), importance = 1, type = "warning")
continue # don't trust counters, they might not be unique
self.commitChanges()
def clearTreeupdatesEntries(self, repository):
self.checkReadOnly()
# treeupdates
self.cursor.execute("DELETE FROM treeupdates WHERE repository = (?)", (repository,))
def resetTreeupdatesDigests(self):
self.checkReadOnly()
self.cursor.execute('UPDATE treeupdates SET digest = "-1"')
#
# FIXME: remove these when 1.0 will be out
#
def migrateCountersTable(self):
self.cursor.execute('DROP TABLE IF EXISTS counterstemp;')
self.cursor.execute('CREATE TABLE counterstemp ( counter INTEGER, idpackage INTEGER PRIMARY KEY, branch VARCHAR );')
self.cursor.execute('select * from counters')
countersdata = self.cursor.fetchall()
if countersdata:
for row in countersdata:
self.cursor.execute('INSERT INTO counterstemp VALUES = (?,?,?)',row)
self.cursor.execute('DROP TABLE counters')
self.cursor.execute('ALTER TABLE counterstemp RENAME TO counters')
self.commitChanges()
def createTreeupdatesTable(self):
self.cursor.execute('DROP TABLE IF EXISTS treeupdates;')
self.cursor.execute('CREATE TABLE treeupdates ( repository VARCHAR PRIMARY KEY, digest VARCHAR );')
def createTreeupdatesactionsTable(self):
self.cursor.execute('DROP TABLE IF EXISTS treeupdatesactions;')
self.cursor.execute('CREATE TABLE treeupdatesactions ( idupdate INTEGER PRIMARY KEY AUTOINCREMENT, repository VARCHAR, command VARCHAR, branch VARCHAR );')
def createSizesTable(self):
self.cursor.execute('DROP TABLE IF EXISTS sizes;')
self.cursor.execute('CREATE TABLE sizes ( idpackage INTEGER, size INTEGER );')
def createCountersBranchColumn(self):
self.cursor.execute('ALTER TABLE counters ADD COLUMN branch VARCHAR;')
idpackages = self.listAllIdpackages()
for idpackage in idpackages:
branch = self.retrieveBranch(idpackage)
self.cursor.execute('UPDATE counters SET branch = (?) WHERE idpackage = (?)', (branch,idpackage,))
def createTreeupdatesactionsBranchColumn(self):
try: # if database disk image is malformed, won't raise exception here
self.cursor.execute('ALTER TABLE treeupdatesactions ADD COLUMN branch VARCHAR;')
self.cursor.execute('UPDATE treeupdatesactions SET branch = (?)', (str(etpConst['branch']),))
except:
pass
def createNeededElfclassColumn(self):
try: # if database disk image is malformed, won't raise exception here
self.cursor.execute('ALTER TABLE needed ADD COLUMN elfclass INTEGER;')
self.cursor.execute('UPDATE needed SET elfclass = (?)', (-1,))
except:
pass
def createContentTypeColumn(self):
try: # if database disk image is malformed, won't raise exception here
self.cursor.execute('ALTER TABLE content ADD COLUMN type VARCHAR;')
self.cursor.execute('UPDATE content SET type = "0"')
except:
pass
def createLicensedataTable(self):
self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );')
def createLicensesAcceptedTable(self):
self.cursor.execute('CREATE TABLE licenses_accepted ( licensename VARCHAR UNIQUE );')
def createTriggerTable(self):
self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );')
def createTriggerColumn(self):
self.checkReadOnly()
self.cursor.execute('ALTER TABLE baseinfo ADD COLUMN trigger INTEGER;')
self.cursor.execute('UPDATE baseinfo SET trigger = 0')
def createMessagesTable(self):
self.cursor.execute("CREATE TABLE messages ( idpackage INTEGER, message VARCHAR);")
def createEclassesTable(self):
self.cursor.execute('DROP TABLE IF EXISTS eclasses;')
self.cursor.execute('DROP TABLE IF EXISTS eclassesreference;')
self.cursor.execute('CREATE TABLE eclasses ( idpackage INTEGER, idclass INTEGER );')
self.cursor.execute('CREATE TABLE eclassesreference ( idclass INTEGER PRIMARY KEY AUTOINCREMENT, classname VARCHAR );')
def createNeededTable(self):
self.cursor.execute('CREATE TABLE needed ( idpackage INTEGER, idneeded INTEGER, elfclass INTEGER );')
self.cursor.execute('CREATE TABLE neededreference ( idneeded INTEGER PRIMARY KEY AUTOINCREMENT, library VARCHAR );')
def createSystemPackagesTable(self):
self.cursor.execute('CREATE TABLE systempackages ( idpackage INTEGER PRIMARY KEY );')
def createInjectedTable(self):
self.cursor.execute('CREATE TABLE injected ( idpackage INTEGER PRIMARY KEY );')
def createProtectTable(self):
self.cursor.execute('DROP TABLE IF EXISTS configprotect;')
self.cursor.execute('DROP TABLE IF EXISTS configprotectmask;')
self.cursor.execute('DROP TABLE IF EXISTS configprotectreference;')
self.cursor.execute('CREATE TABLE configprotect ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );')
self.cursor.execute('CREATE TABLE configprotectmask ( idpackage INTEGER PRIMARY KEY, idprotect INTEGER );')
self.cursor.execute('CREATE TABLE configprotectreference ( idprotect INTEGER PRIMARY KEY AUTOINCREMENT, protect VARCHAR );')
def createInstalledTable(self):
self.cursor.execute('DROP TABLE IF EXISTS installedtable;')
self.cursor.execute('CREATE TABLE installedtable ( idpackage INTEGER PRIMARY KEY, repositoryname VARCHAR );')
def addDependRelationToDependsTable(self, iddependency, idpackage):
self.cursor.execute(
'INSERT into dependstable VALUES '
'(?,?)'
, ( iddependency,
idpackage,
)
)
if (self.entropyTools.is_user_in_entropy_group()) and \
(self.dbname.startswith(etpConst['serverdbid'])):
# force commit even if readonly, this will allow to automagically fix dependstable server side
self.connection.commit() # we don't care much about syncing the database since it's quite trivial
'''
@description: recreate dependstable table in the chosen database, it's used for caching searchDepends requests
@input Nothing
@output: Nothing
'''
def regenerateDependsTable(self, output = True):
self.createDependsTable()
depends = self.listAllDependencies()
count = 0
total = len(depends)
for depend in depends:
count += 1
atom = depend[1]
iddep = depend[0]
if output:
self.updateProgress(
red("Resolving %s") % (darkgreen(atom),),
importance = 0,
type = "info",
back = True,
count = (count,total)
)
match = self.atomMatch(atom)
if (match[0] != -1):
self.addDependRelationToDependsTable(iddep,match[0])
del depends
# now validate dependstable
self.sanitizeDependsTable()
########################################################
####
## Dependency handling functions
#
def __get_match_hash(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision):
c_hash = str(hash(atom)) + \
str(hash(matchSlot)) + \
str(hash(matchTag)) + \
str(hash(matchRevision)) + \
str(hash(tuple(matchBranches))) + \
str(hash(caseSensitive)) + \
str(hash(multiMatch)) + \
str(hash(packagesFilter)) + \
str(hash(matchRevision))
#c_hash = str(hash(c_hash))
return c_hash
def atomMatchFetchCache(self, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision):
if self.xcache:
c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
try:
cached = dumpTools.loadobj(etpCache['dbMatch']+"/"+self.dbname+"/"+c_hash)
if cached != None:
return cached
except (EOFError, IOError):
return None
def atomMatchStoreCache(self, result, atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision):
if self.xcache:
c_hash = self.__get_match_hash(atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
try:
sperms = False
if not os.path.isdir(os.path.join(etpConst['dumpstoragedir'],etpCache['dbMatch']+"/"+self.dbname)):
sperms = True
dumpTools.dumpobj(etpCache['dbMatch']+"/"+self.dbname+"/"+c_hash,result)
if sperms:
const_setup_perms(etpConst['dumpstoragedir'],etpConst['entropygid'])
except IOError:
pass
# function that validate one atom by reading keywords settings
# idpackageValidatorCache = {} >> function cache
def idpackageValidator(self,idpackage):
reponame = self.dbname[5:]
cached = idpackageValidatorCache.get((idpackage,reponame))
if cached != None: return cached
# check if user package.mask needs it masked
user_package_mask_ids = etpConst['packagemasking'].get(reponame+'mask_ids')
if user_package_mask_ids == None:
etpConst['packagemasking'][reponame+'mask_ids'] = set()
for atom in etpConst['packagemasking']['mask']:
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking'][reponame+'mask_ids'] |= matches[0]
user_package_mask_ids = etpConst['packagemasking'][reponame+'mask_ids']
if idpackage in user_package_mask_ids:
# sorry, masked
idpackageValidatorCache[(idpackage,reponame)] = -1,1
return -1,1
# see if we can unmask by just lookin into user package.unmask stuff -> etpConst['packagemasking']['unmask']
user_package_unmask_ids = etpConst['packagemasking'].get(reponame+'unmask_ids')
if user_package_unmask_ids == None:
etpConst['packagemasking'][reponame+'unmask_ids'] = set()
for atom in etpConst['packagemasking']['unmask']:
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking'][reponame+'unmask_ids'] |= matches[0]
user_package_unmask_ids = etpConst['packagemasking'][reponame+'unmask_ids']
if idpackage in user_package_unmask_ids:
idpackageValidatorCache[(idpackage,reponame)] = idpackage,3
return idpackage,3
# check if repository packages.db.mask needs it masked
repomask = etpConst['packagemasking']['repos_mask'].get(reponame)
if repomask != None:
# first, seek into generic masking, all branches
all_branches_mask = repomask.get("*")
if all_branches_mask:
all_branches_mask_ids = repomask.get("*_ids")
if all_branches_mask_ids == None:
etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] = set()
for atom in all_branches_mask:
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking']['repos_mask'][reponame]['*_ids'] |= matches[0]
all_branches_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['*_ids']
if idpackage in all_branches_mask_ids:
idpackageValidatorCache[(idpackage,reponame)] = -1,8
return -1,8
# no universal mask
branches_mask = repomask.get("branch")
if branches_mask:
for branch in branches_mask:
branch_mask_ids = branches_mask.get(branch+"_ids")
if branch_mask_ids == None:
etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] = set()
for atom in branches_mask[branch]:
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"] |= matches[0]
branch_mask_ids = etpConst['packagemasking']['repos_mask'][reponame]['branch'][branch+"_ids"]
if idpackage in branch_mask_ids:
if self.retrieveBranch(idpackage) == branch:
idpackageValidatorCache[(idpackage,reponame)] = -1,9
return -1,9
if etpConst['packagemasking']['license_mask']:
mylicenses = self.retrieveLicense(idpackage)
mylicenses = mylicenses.strip().split()
if mylicenses:
for mylicense in mylicenses:
if mylicense in etpConst['packagemasking']['license_mask']:
idpackageValidatorCache[(idpackage,reponame)] = -1,10
return -1,10
mykeywords = self.retrieveKeywords(idpackage)
# XXX WORKAROUND
if not mykeywords: mykeywords = [''] # ** is fine then
# firstly, check if package keywords are in etpConst['keywords'] (universal keywords have been merged from package.mask)
for key in etpConst['keywords']:
if key in mykeywords:
# found! all fine
idpackageValidatorCache[(idpackage,reponame)] = idpackage,2
return idpackage,2
# if we get here, it means we didn't find mykeywords in etpConst['keywords'], we need to seek etpConst['packagemasking']['keywords']
# seek in repository first
if reponame in etpConst['packagemasking']['keywords']['repositories']:
for keyword in etpConst['packagemasking']['keywords']['repositories'][reponame]:
if keyword in mykeywords:
keyword_data = etpConst['packagemasking']['keywords']['repositories'][reponame].get(keyword)
if keyword_data:
if "*" in keyword_data: # all packages in this repo with keyword "keyword" are ok
idpackageValidatorCache[(idpackage,reponame)] = idpackage,4
#self.storeInfoCache(idpackage,'idpackageValidator',(idpackage,4))
return idpackage,4
keyword_data_ids = etpConst['packagemasking']['keywords']['repositories'][reponame].get(keyword+"_ids")
if keyword_data_ids == None:
etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"] = set()
for atom in keyword_data:
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"] |= matches[0]
keyword_data_ids = etpConst['packagemasking']['keywords']['repositories'][reponame][keyword+"_ids"]
if idpackage in keyword_data_ids:
idpackageValidatorCache[(idpackage,reponame)] = idpackage,5
return idpackage,5
# if we get here, it means we didn't find a match in repositories
# so we scan packages, last chance
for keyword in etpConst['packagemasking']['keywords']['packages']:
# first of all check if keyword is in mykeywords
if keyword in mykeywords:
keyword_data = etpConst['packagemasking']['keywords']['packages'].get(keyword)
# check for relation
if keyword_data:
keyword_data_ids = etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"]
if keyword_data_ids == None:
etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] = set()
for atom in keyword_data:
# match atom
matches = self.atomMatch(atom, multiMatch = True, packagesFilter = False)
if matches[1] != 0:
continue
etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"] |= matches[0]
keyword_data_ids = etpConst['packagemasking']['keywords']['packages'][reponame+keyword+"_ids"]
if idpackage in keyword_data_ids:
# valid!
idpackageValidatorCache[(idpackage,reponame)] = idpackage,6
return idpackage,6
# holy crap, can't validate
idpackageValidatorCache[(idpackage,reponame)] = -1,7
return -1,7
# packages filter used by atomMatch, input must me foundIDs, a list like this:
# [(u'x11-libs/qt-4.3.2', 608), (u'x11-libs/qt-3.3.8-r4', 1867)]
def packagesFilter(self,results, atom):
# keywordsFilter ONLY FILTERS results if self.dbname.startswith(etpConst['dbnamerepoprefix']), repository database is open
if not self.dbname.startswith(etpConst['dbnamerepoprefix']):
return results
newresults = set()
for item in results:
rc = self.idpackageValidator(item[1])
if rc[0] != -1:
newresults.add(item)
else:
idreason = rc[1]
if not maskingReasonsStorage.has_key(atom):
maskingReasonsStorage[atom] = {}
if not maskingReasonsStorage[atom].has_key(idreason):
maskingReasonsStorage[atom][idreason] = set()
maskingReasonsStorage[atom][idreason].add((item[1],self.dbname[5:]))
return newresults
def __filterSlot(self, idpackage, slot):
if slot == None:
return idpackage
dbslot = self.retrieveSlot(idpackage)
if str(dbslot) == str(slot):
return idpackage
def __filterTag(self, idpackage, tag, operators):
if tag == None:
return idpackage
dbtag = self.retrieveVersionTag(idpackage)
compare = cmp(tag,dbtag)
if not operators or operators == "=":
if compare == 0:
return idpackage
else:
return self.__do_operator_compare(idpackage, operators, compare)
def __do_operator_compare(self, token, operators, compare):
if operators == ">" and compare == -1:
return token
elif operators == ">=" and compare < 1:
return token
elif operators == "<" and compare == 1:
return token
elif operators == "<=" and compare > -1:
return token
def __filterSlotTag(self, foundIDs, slot, tag, operators):
newlist = set()
for data in foundIDs:
idpackage = data[1]
idpackage = self.__filterSlot(idpackage, slot)
if not idpackage:
continue
idpackage = self.__filterTag(idpackage, tag, operators)
if not idpackage:
continue
newlist.add(data)
return newlist
'''
@description: matches the user chosen package name+ver, if possibile, in a single repository
@input atom: string, atom to match
@input caseSensitive: bool, should the atom be parsed case sensitive?
@input matchSlot: string, match atoms with the provided slot
@input multiMatch: bool, return all the available atoms
@input matchBranches: tuple or list, match packages only in the specified branches
@input matchTag: match packages only for the specified tag
@input packagesFilter: enable/disable package.mask/.keywords/.unmask filter
@output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = error
'''
def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = (), matchTag = None, packagesFilter = True, matchRevision = None):
if not atom:
return -1,1
cached = self.atomMatchFetchCache(atom,caseSensitive,matchSlot,multiMatch,matchBranches,matchTag,packagesFilter, matchRevision)
if cached != None:
return cached
atomTag = self.entropyTools.dep_gettag(atom)
atomSlot = self.entropyTools.dep_getslot(atom)
atomRev = self.entropyTools.dep_get_entropy_revision(atom)
# tag match
scan_atom = self.entropyTools.remove_tag(atom)
if (matchTag == None) and (atomTag != None):
matchTag = atomTag
# slot match
scan_atom = self.entropyTools.remove_slot(scan_atom)
if (matchSlot == None) and (atomSlot != None):
matchSlot = atomSlot
# revision match
scan_atom = self.entropyTools.remove_entropy_revision(scan_atom)
if (matchRevision == None) and (atomRev != None):
matchRevision = atomRev
# check for direction
strippedAtom = self.entropyTools.dep_getcpv(scan_atom)
if scan_atom[-1] == "*":
strippedAtom += "*"
direction = scan_atom[0:len(scan_atom)-len(strippedAtom)]
justname = self.entropyTools.isjustname(strippedAtom)
pkgversion = ''
if not justname:
# get version
data = self.entropyTools.catpkgsplit(strippedAtom)
if data == None:
return -1,1 # atom is badly formatted
pkgversion = data[2]+"-"+data[3]
pkgkey = self.entropyTools.dep_getkey(strippedAtom)
splitkey = pkgkey.split("/")
if (len(splitkey) == 2):
pkgname = splitkey[1]
pkgcat = splitkey[0]
else:
pkgname = splitkey[0]
pkgcat = "null"
if (matchBranches):
myBranchIndex = tuple(matchBranches) # force to tuple for security
else:
if (self.dbname == etpConst['clientdbid']):
# collect all available branches
myBranchIndex = tuple(self.listAllBranches())
elif self.dbname.startswith(etpConst['dbnamerepoprefix']): # repositories should match to any branch <= than the current if none specified
allbranches = set([x for x in self.listAllBranches() if x <= etpConst['branch']])
allbranches = list(allbranches)
allbranches.reverse()
if etpConst['branch'] not in allbranches:
allbranches.insert(0,etpConst['branch'])
myBranchIndex = tuple(allbranches)
else:
myBranchIndex = (etpConst['branch'],)
# IDs found in the database that match our search
foundIDs = set()
for idx in myBranchIndex:
results = self.searchPackagesByName(pkgname, sensitive = caseSensitive, branch = idx)
mypkgcat = pkgcat
mypkgname = pkgname
virtual = False
# if it's a PROVIDE, search with searchProvide
# there's no package with that name
if (not results) and (mypkgcat == "virtual"):
virtuals = self.searchProvide(pkgkey, branch = idx)
if (virtuals):
virtual = True
mypkgname = self.retrieveName(virtuals[0][1])
mypkgcat = self.retrieveCategory(virtuals[0][1])
results = virtuals
# now validate
if (not results):
continue # search into a stabler branch
elif (len(results) > 1):
# if it's because category differs, it's a problem
foundCat = ""
cats = set()
for result in results:
idpackage = result[1]
cat = self.retrieveCategory(idpackage)
cats.add(cat)
if (cat == mypkgcat) or ((not virtual) and (mypkgcat == "virtual") and (cat == mypkgcat)): # in case of virtual packages only (that they're not stored as provide)
foundCat = cat
# if we found something at least...
if (not foundCat) and (len(cats) == 1) and (mypkgcat in ("virtual","null")):
foundCat = list(cats)[0]
if (not foundCat):
# got the issue
continue
# we can use foundCat
mypkgcat = foundCat
# we need to search using the category
if (not multiMatch):
results = self.searchPackagesByNameAndCategory(name = mypkgname, category = mypkgcat, branch = idx, sensitive = caseSensitive)
# validate again
if (not results):
continue # search into another branch
# if we get here, we have found the needed IDs
foundIDs |= set(results)
break
else:
# if mypkgcat is virtual, we can force
if (mypkgcat == "virtual") and (not virtual): # in case of virtual packages only (that they're not stored as provide)
mypkgcat = self.entropyTools.dep_getkey(results[0][0]).split("/")[0]
# check if category matches
if mypkgcat != "null":
foundCat = self.retrieveCategory(results[0][1])
if mypkgcat == foundCat:
foundIDs.add(results[0])
else:
continue
else:
foundIDs.add(results[0])
break
### FILTERING
### FILTERING
### FILTERING
# filter slot and tag
foundIDs = self.__filterSlotTag(foundIDs, matchSlot, matchTag, direction)
if packagesFilter: # keyword filtering
foundIDs = self.packagesFilter(foundIDs, atom)
### END FILTERING
### END FILTERING
### END FILTERING
if not foundIDs:
# package not found
self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
return -1,1
### FILLING dbpkginfo
### FILLING dbpkginfo
### FILLING dbpkginfo
dbpkginfo = set()
# now we have to handle direction
if (direction) or (direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*")):
if (not justname) and \
((direction == "~") or (direction == "=") or \
(direction == '' and not justname) or (direction == '' and not justname and strippedAtom.endswith("*"))):
# any revision within the version specified OR the specified version
if (direction == '' and not justname):
direction = "="
# remove gentoo revision (-r0 if none)
if (direction == "="):
if (pkgversion.split("-")[-1] == "r0"):
pkgversion = self.entropyTools.remove_revision(pkgversion)
if (direction == "~"):
pkgrevision = self.entropyTools.dep_get_portage_revision(pkgversion)
pkgversion = self.entropyTools.remove_revision(pkgversion)
for data in foundIDs:
idpackage = data[1]
dbver = self.retrieveVersion(idpackage)
if (direction == "~"):
myrev = self.entropyTools.dep_get_portage_revision(dbver)
myver = self.entropyTools.remove_revision(dbver)
if myver == pkgversion and pkgrevision <= myrev:
# found
dbpkginfo.add((idpackage,dbver))
else:
# media-libs/test-1.2* support
if pkgversion[-1] == "*":
if dbver.startswith(pkgversion[:-1]):
dbpkginfo.add((idpackage,dbver))
elif (matchRevision != None) and (pkgversion == dbver):
dbrev = self.retrieveRevision(idpackage)
if dbrev == matchRevision:
dbpkginfo.add((idpackage,dbver))
elif (pkgversion == dbver) and (matchRevision == None):
dbpkginfo.add((idpackage,dbver))
elif (direction.find(">") != -1) or (direction.find("<") != -1):
if not justname:
# remove revision (-r0 if none)
if pkgversion.endswith("r0"):
# remove
self.entropyTools.remove_revision(pkgversion)
for data in foundIDs:
idpackage = data[1]
revcmp = 0
tagcmp = 0
if matchRevision != None:
dbrev = self.retrieveRevision(idpackage)
revcmp = cmp(matchRevision,dbrev)
if matchTag != None:
dbtag = self.retrieveVersionTag(idpackage)
tagcmp = cmp(matchTag,dbtag)
dbver = self.retrieveVersion(idpackage)
pkgcmp = self.entropyTools.compareVersions(pkgversion,dbver)
if isinstance(pkgcmp,tuple):
failed = pkgcmp[1]
if failed == 0:
failed = pkgversion
else:
failed = dbver
# I am sorry, but either pkgversion or dbver are invalid
self.updateProgress(
bold("atomMatch: ")+red("comparison between %s and %s failed. Wrong syntax for: %s") % (pkgversion,dbver,failed,),
importance = 1,
type = "error",
header = darkred(" !!! ")
)
raise exceptionTools.InvalidVersionString(
"InvalidVersionString: from atom: %s, cmp: %s, failed: %s" % (
atom, pkgcmp, failed, )
)
if direction == ">":
if pkgcmp < 0:
dbpkginfo.add((idpackage,dbver))
elif (matchRevision != None) and pkgcmp <= 0 and revcmp < 0:
#print "found >",self.retrieveAtom(idpackage)
dbpkginfo.add((idpackage,dbver))
elif (matchTag != None) and tagcmp < 0:
dbpkginfo.add((idpackage,dbver))
elif direction == "<":
if pkgcmp > 0:
dbpkginfo.add((idpackage,dbver))
elif (matchRevision != None) and pkgcmp >= 0 and revcmp > 0:
#print "found <",self.retrieveAtom(idpackage)
dbpkginfo.add((idpackage,dbver))
elif (matchTag != None) and tagcmp > 0:
dbpkginfo.add((idpackage,dbver))
elif direction == ">=":
if (matchRevision != None) and pkgcmp <= 0:
if pkgcmp == 0:
if revcmp <= 0:
dbpkginfo.add((idpackage,dbver))
#print "found >=",self.retrieveAtom(idpackage)
else:
dbpkginfo.add((idpackage,dbver))
elif pkgcmp <= 0 and matchRevision == None:
dbpkginfo.add((idpackage,dbver))
elif (matchTag != None) and tagcmp <= 0:
dbpkginfo.add((idpackage,dbver))
elif direction == "<=":
if (matchRevision != None) and pkgcmp >= 0:
if pkgcmp == 0:
if revcmp >= 0:
dbpkginfo.add((idpackage,dbver))
#print "found <=",self.retrieveAtom(idpackage)
else:
dbpkginfo.add((idpackage,dbver))
elif pkgcmp >= 0 and matchRevision == None:
dbpkginfo.add((idpackage,dbver))
elif (matchTag != None) and tagcmp >= 0:
dbpkginfo.add((idpackage,dbver))
else: # just the key
dbpkginfo = set([((x[1]),self.retrieveVersion(x[1])) for x in foundIDs])
### END FILLING dbpkginfo
### END FILLING dbpkginfo
### END FILLING dbpkginfo
if not dbpkginfo:
self.atomMatchStoreCache((-1,1), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
return -1,1
if multiMatch:
x = set([x[0] for x in dbpkginfo])
self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
return x,0
if len(dbpkginfo) == 1:
x = dbpkginfo.pop()
self.atomMatchStoreCache((x[0],0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
return x[0],0
dbpkginfo = list(dbpkginfo)
pkgdata = {}
versions = set()
for x in dbpkginfo:
info_tuple = (x[1],self.retrieveVersionTag(x[0]),self.retrieveRevision(x[0]))
versions.add(info_tuple)
pkgdata[info_tuple] = x[0]
newer = self.entropyTools.getEntropyNewerVersion(list(versions))[0]
x = pkgdata[newer]
self.atomMatchStoreCache((x,0), atom, caseSensitive, matchSlot, multiMatch, matchBranches, matchTag, packagesFilter, matchRevision)
return x,0
Reference in New Issue View Git Blame Copy Permalink