From f85dcf9687a2072e54dbbe93e9ca8f07e8b10474 Mon Sep 17 00:00:00 2001 From: "(no author)" <(no author)@cd1c1023-2f26-0410-ae45-c471fc1f0318> Date: Wed, 20 Feb 2008 13:46:56 +0000 Subject: [PATCH] - initial implementation of the licenses management infrastructure git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1257 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- TODO | 1 - libraries/activatorTools.py | 25 ++++++- libraries/databaseTools.py | 123 +++++++++++++++++++++++++++++++--- libraries/entropy.py | 3 + libraries/entropyConstants.py | 9 +++ libraries/entropyTools.py | 21 ++++++ libraries/serverConstants.py | 1 + 7 files changed, 169 insertions(+), 14 deletions(-) diff --git a/TODO b/TODO index 24c117df0..bde8ea207 100644 --- a/TODO +++ b/TODO @@ -8,7 +8,6 @@ TODO list: - packages.sabayonlinux.org interactivity (comments + images upload + connection to phpbb user db) - Community repositories - - separate packages by license restrictions - split RDEPEND and PDEPEND - write a ncurses interface to manage entropy database - find a way to better handle real smartapps deps (need split PDEPEND?) diff --git a/libraries/activatorTools.py b/libraries/activatorTools.py index 03c6e5622..828fd12de 100644 --- a/libraries/activatorTools.py +++ b/libraries/activatorTools.py @@ -1099,7 +1099,20 @@ def uploadDatabase(uris): else: print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) - # uploading package.mask (packages.db.mask) file + # uploading package licenses whitelist (packages.db.lic_whitelist) file + dblicwlfile = etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaselicwhitelistfile'] + if not os.path.isfile(dblicwlfile): + f = open(dblicwlfile,"w") + f.flush() + f.close() + print_info(green(" * ")+red("Uploading file ")+bold(dblicwlfile)+red(" ..."), back = True) + rc = ftp.uploadFile(dblicwlfile,True) + if (rc == True): + print_info(green(" * ")+red("Upload of ")+bold(dblicwlfile)+red(" completed.")) + else: + print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) + + # uploading packages mask list (packages.db.mask) file dbmaskfile = etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasemaskfile'] if not os.path.isfile(dbmaskfile): f = open(dbmaskfile,"w") @@ -1166,7 +1179,15 @@ def downloadDatabase(uri): else: print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) - # downloading package.mask (packages.db.mask) + # downloading package license whitelist (packages.db.lic_whitelist) + print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaselicwhitelistfile'])+red(" ..."), back = True) + rc = ftp.downloadFile(etpConst['etpdatabaselicwhitelistfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True) + if (rc == True): + print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabaselicwhitelistfile'])+red(" completed.")) + else: + print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check.")) + + # downloading package mask list (packages.db.mask) print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasemaskfile'])+red(" ..."), back = True) rc = ftp.downloadFile(etpConst['etpdatabasemaskfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True) if (rc == True): diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 266e50901..885f18f9d 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -760,6 +760,22 @@ class etpDatabase: # create category licid = self.addLicense(etpData['license']) + # insert license information + if not self.doesTableExist("licensedata"): + self.createLicensedataTable() + mylicenses = etpData['licensedata'].keys() + for mylicense in mylicenses: + found = self.isLicensedataKeyAvailable(mylicense) + if not found: + text = mylicenses[mylicense] + self.cursor.execute( + 'INSERT into licensedata VALUES ' + '(?,?,?)' + , ( mylicense, + buffer(text), + 0, + )) + # look for configured versiontag versiontag = "" if (etpData['versiontag']): @@ -1422,6 +1438,13 @@ class etpDatabase: self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot,idpackage,)) self.commitChanges() + def removeLicensedata(self, license_name): + if not self.doesTableExist("licensedata"): + if (etpConst['uid'] == 0) and (not self.readOnly): + self.createLicensedataTable() + return + self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,)) + def removeDependencies(self, idpackage): self.checkReadOnly() self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,)) @@ -1876,6 +1899,9 @@ class etpDatabase: data['size'] = mydata[17] data['revision'] = mydata[18] data['disksize'] = self.retrieveOnDiskSize(idpackage) # cannot do this too, for backward compat + + data['licensedata'] = self.retrieveLicensedata(idpackage) + return data def fetchall2set(self, item): @@ -2440,21 +2466,70 @@ class etpDatabase: def retrieveMirrorInfo(self, mirrorname): - self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) - mirrorlist = self.fetchall2set(self.cursor.fetchall()) - - return mirrorlist + self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,)) + mirrorlist = self.fetchall2set(self.cursor.fetchall()) + return mirrorlist def retrieveCategory(self, idpackage): - cache = self.fetchInfoCache(idpackage,'retrieveCategory') - if cache != None: return cache + cache = self.fetchInfoCache(idpackage,'retrieveCategory') + if cache != None: return cache - self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,)) - cat = self.cursor.fetchone()[0] + self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,)) + cat = self.cursor.fetchone()[0] - self.storeInfoCache(idpackage,'retrieveCategory',cat) - return cat + self.storeInfoCache(idpackage,'retrieveCategory',cat) + return cat + + def retrieveLicensedata(self, idpackage): + + cache = self.fetchInfoCache(idpackage,'retrieveLicensedata') + if cache != None: return cache + + # insert license information + if not self.doesTableExist("licensedata"): + if (etpConst['uid'] == 0) and (not self.readOnly): + self.createLicensedataTable() + return {} + licenses = self.retrieveLicense(idpackage) + licenses = licenses.split() + licdata = {} + for licname in licenses: + licname = licname.strip() + if not licname.isalnum(): + continue + self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,)) + lictext = self.cursor.fetchone() + if lictext != None: + lictext = lictext[0] + licdata[licname] = lictext + + self.storeInfoCache(idpackage,'retrieveLicensedata',licdata) + return licdata + + def retrieveLicensedataKeys(self, idpackage): + + cache = self.fetchInfoCache(idpackage,'retrieveLicensedataKeys') + if cache != None: return cache + + if not self.doesTableExist("licensedata"): + if (etpConst['uid'] == 0) and (not self.readOnly): + self.createLicensedataTable() + return set() + licenses = self.retrieveLicense(idpackage) + licenses = licenses.split() + licdata = set() + for licname in licenses: + licname = licname.strip() + if not licname.isalnum(): + continue + self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,)) + licidentifier = self.cursor.fetchone() + if licidentifier: + licdata.add(licidentifier[0]) + + self.storeInfoCache(idpackage,'retrieveLicensedataKeys',licdata) + return licdata def retrieveLicense(self, idpackage): @@ -2616,8 +2691,20 @@ class etpDatabase: return result + def isLicensedataKeyAvailable(self, license_name): + if not self.doesTableExist("licensedata"): + if (etpConst['uid'] == 0) and (not self.readOnly): + self.createLicensedataTable() + else: + return True + self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,)) + result = self.cursor.fetchone() + if not result: + return False + return True + def isLicenseAvailable(self,pkglicense): - if not pkglicense: # workaround for packages without a license but just garbage + if not pkglicense or not pkglicense.isalnum(): # workaround for packages without a license but just garbage pkglicense = ' ' self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,)) result = self.cursor.fetchone() @@ -3376,6 +3463,7 @@ class etpDatabase: self.createExtrainfoIndex() self.createNeededIndex() self.createUseflagsIndex() + self.createLicensedataIndex() def createNeededIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: @@ -3397,6 +3485,16 @@ class etpDatabase: self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, versiontag, slot, branch, revision )') self.commitChanges() + def createLicensedataIndex(self): + if self.dbname != etpConst['serverdbid'] and self.indexing: + if not self.doesTableExist("licensedata"): + if (etpConst['uid'] == 0) and (not self.readOnly): + self.createLicensedataTable() + else: + return + self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )') + self.commitChanges() + def createKeywordsIndex(self): if self.dbname != etpConst['serverdbid'] and self.indexing: self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsindex ON keywords ( idpackage, idkeyword )') @@ -3517,6 +3615,9 @@ class etpDatabase: except: pass + def createLicensedataTable(self): + self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );') + def createTriggerTable(self): self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );') diff --git a/libraries/entropy.py b/libraries/entropy.py index da72b2bbd..0cdca3f60 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -2684,6 +2684,9 @@ class PackageInterface: data['injected'] = False data['counter'] = -1 # gentoo counter will be set in self.__install_package_into_gentoo_database() + # FIXME: this will be removed create all indexes on the client db + self.Entropy.clientDbconn.createAllIndexes() + idpk, rev, x, status = self.Entropy.clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision']) del x del data diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 642a02aa5..6846976f7 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -67,6 +67,7 @@ etpData = { 'needed': u"", # runtime libraries needed by the package 'trigger': u"", # this will become a bool, containing info about external trigger presence 'injected': bool, # if the package has been injected manually, this will be true + 'licensedata': dict # dictionary that contains license text } ''' @@ -304,6 +305,12 @@ CREATE TABLE binkeywords ( idkeyword INTEGER ); +CREATE TABLE licensedata ( + licensename VARCHAR UNIQUE, + text BLOB, + compressed INTEGER +); + """ # ETP_ARCH_CONST setup @@ -319,6 +326,7 @@ etpSys = { 'rootdir': "", 'maxthreads': 100, 'dirstoclean': set(), + 'serverside': False, } etpUi = { @@ -520,6 +528,7 @@ def initConfig_entropyConstants(rootdir): 'entropyxpakfilename': "metadata.xpak", # Gentoo xpak metadata file name 'etpdatabasemaskfile': ETP_DBFILE+".mask", # the local/remote database revision file + 'etpdatabaselicwhitelistfile': ETP_DBFILE+".lic_whitelist", # the local/remote database revision file 'etpdatabaserevisionfile': ETP_DBFILE+".revision", # the local/remote database revision file 'etpdatabasehashfile': ETP_DBFILE+".md5", # its checksum 'etpdatabasedumphashfilebz2': ETP_DBFILE+".dump.bz2.md5", diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index d7a8ebc6e..5112bf334 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -2101,6 +2101,27 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje pass ''' + # Get License text if possible + licenses_dir = None + try: + from portageTools import getPortageEnv + licenses_dir = os.path.join(getPortageEnv('PORTDIR'),'licenses') + except: + pass + data['licensedata'] = {} + if licenses_dir: + licdata = [x.strip() for x in data['license'].split() if x.strip().isalnum()] + for mylicense in licdata: + licfile = os.path.join(licenses_dir,mylicense) + if os.access(licfile,os.R_OK): + if istextfile(licfile): + f = open(licfile,"rb") + f.seek(0,2) + size = f.tell() + f.seek(0) + data['licensedata'][mylicense] = f.read(size) + f.close() + if not silent: print_info(yellow(" * ")+red(info_package+"Getting package mirrors list..."),back = True) # manage data['sources'] to create data['mirrorlinks'] # =mirror://openoffice|link1|link2|link3 diff --git a/libraries/serverConstants.py b/libraries/serverConstants.py index 5b6892aaf..3f7f78ce6 100644 --- a/libraries/serverConstants.py +++ b/libraries/serverConstants.py @@ -21,6 +21,7 @@ ''' from entropyConstants import * +etpSys['serverside'] = True def initConfig_serverConstants():