- initial implementation of the licenses management infrastructure

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1257 cd1c1023-2f26-0410-ae45-c471fc1f0318

TODO

@@ -8,7 +8,6 @@ TODO list:
 
   - packages.sabayonlinux.org interactivity (comments + images upload + connection to phpbb user db)
   - Community repositories
-  - separate packages by license restrictions
   - split RDEPEND and PDEPEND
   - write a ncurses interface to manage entropy database
   - find a way to better handle real smartapps deps (need split PDEPEND?)

libraries/activatorTools.py

@@ -1099,7 +1099,20 @@ def uploadDatabase(uris):
         else:
             print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
 
-        # uploading package.mask (packages.db.mask) file
+        # uploading package licenses whitelist (packages.db.lic_whitelist) file
+        dblicwlfile = etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaselicwhitelistfile']
+        if not os.path.isfile(dblicwlfile):
+            f = open(dblicwlfile,"w")
+            f.flush()
+            f.close()
+        print_info(green(" * ")+red("Uploading file ")+bold(dblicwlfile)+red(" ..."), back = True)
+        rc = ftp.uploadFile(dblicwlfile,True)
+        if (rc == True):
+            print_info(green(" * ")+red("Upload of ")+bold(dblicwlfile)+red(" completed."))
+        else:
+            print_warning(brown(" * ")+red("Cannot properly upload to ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
+
+        # uploading packages mask list (packages.db.mask) file
         dbmaskfile = etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabasemaskfile']
         if not os.path.isfile(dbmaskfile):
             f = open(dbmaskfile,"w")
@@ -1166,7 +1179,15 @@ def downloadDatabase(uri):
     else:
         print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
 
-    # downloading package.mask (packages.db.mask)
+    # downloading package license whitelist (packages.db.lic_whitelist)
+    print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabaselicwhitelistfile'])+red(" ..."), back = True)
+    rc = ftp.downloadFile(etpConst['etpdatabaselicwhitelistfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
+    if (rc == True):
+        print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabaselicwhitelistfile'])+red(" completed."))
+    else:
+        print_warning(brown(" * ")+red("Cannot properly download from ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Please check."))
+
+    # downloading package mask list (packages.db.mask)
     print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasemaskfile'])+red(" ..."), back = True)
     rc = ftp.downloadFile(etpConst['etpdatabasemaskfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
     if (rc == True):

libraries/databaseTools.py

@@ -760,6 +760,22 @@ class etpDatabase:
             # create category
             licid = self.addLicense(etpData['license'])
 
+        # insert license information
+        if not self.doesTableExist("licensedata"):
+            self.createLicensedataTable()
+        mylicenses = etpData['licensedata'].keys()
+        for mylicense in mylicenses:
+            found = self.isLicensedataKeyAvailable(mylicense)
+            if not found:
+                text = mylicenses[mylicense]
+                self.cursor.execute(
+                    'INSERT into licensedata VALUES '
+                    '(?,?,?)'
+                    , (	mylicense,
+                            buffer(text),
+                            0,
+                ))
+
         # look for configured versiontag
         versiontag = ""
         if (etpData['versiontag']):
@@ -1422,6 +1438,13 @@ class etpDatabase:
         self.cursor.execute('UPDATE baseinfo SET slot = (?) WHERE idpackage = (?)', (slot,idpackage,))
         self.commitChanges()
 
+    def removeLicensedata(self, license_name):
+        if not self.doesTableExist("licensedata"):
+            if (etpConst['uid'] == 0) and (not self.readOnly):
+                self.createLicensedataTable()
+            return
+        self.cursor.execute('DELETE FROM licensedata WHERE licensename = (?)', (license_name,))
+
     def removeDependencies(self, idpackage):
         self.checkReadOnly()
         self.cursor.execute("DELETE FROM dependencies WHERE idpackage = (?)", (idpackage,))
@@ -1876,6 +1899,9 @@ class etpDatabase:
         data['size'] = mydata[17]
         data['revision'] = mydata[18]
         data['disksize'] = self.retrieveOnDiskSize(idpackage) # cannot do this too, for backward compat
+
+        data['licensedata'] = self.retrieveLicensedata(idpackage)
+
         return data
 
     def fetchall2set(self, item):
@@ -2440,21 +2466,70 @@ class etpDatabase:
 
     def retrieveMirrorInfo(self, mirrorname):
 
-	self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,))
-	mirrorlist = self.fetchall2set(self.cursor.fetchall())
-
-	return mirrorlist
+        self.cursor.execute('SELECT "mirrorlink" FROM mirrorlinks WHERE mirrorname = (?)', (mirrorname,))
+        mirrorlist = self.fetchall2set(self.cursor.fetchall())
+        return mirrorlist
 
     def retrieveCategory(self, idpackage):
 
-	cache = self.fetchInfoCache(idpackage,'retrieveCategory')
-	if cache != None: return cache
+        cache = self.fetchInfoCache(idpackage,'retrieveCategory')
+        if cache != None: return cache
 
-	self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
-	cat = self.cursor.fetchone()[0]
+        self.cursor.execute('SELECT category FROM baseinfo,categories WHERE baseinfo.idpackage = (?) and baseinfo.idcategory = categories.idcategory', (idpackage,))
+        cat = self.cursor.fetchone()[0]
 
-	self.storeInfoCache(idpackage,'retrieveCategory',cat)
-	return cat
+        self.storeInfoCache(idpackage,'retrieveCategory',cat)
+        return cat
+
+    def retrieveLicensedata(self, idpackage):
+
+        cache = self.fetchInfoCache(idpackage,'retrieveLicensedata')
+        if cache != None: return cache
+
+        # insert license information
+        if not self.doesTableExist("licensedata"):
+            if (etpConst['uid'] == 0) and (not self.readOnly):
+                self.createLicensedataTable()
+            return {}
+        licenses = self.retrieveLicense(idpackage)
+        licenses = licenses.split()
+        licdata = {}
+        for licname in licenses:
+            licname = licname.strip()
+            if not licname.isalnum():
+                continue
+            self.cursor.execute('SELECT text FROM licensedata WHERE licensename = (?)', (licname,))
+            lictext = self.cursor.fetchone()
+            if lictext != None:
+                lictext = lictext[0]
+                licdata[licname] = lictext
+
+        self.storeInfoCache(idpackage,'retrieveLicensedata',licdata)
+        return licdata
+
+    def retrieveLicensedataKeys(self, idpackage):
+
+        cache = self.fetchInfoCache(idpackage,'retrieveLicensedataKeys')
+        if cache != None: return cache
+
+        if not self.doesTableExist("licensedata"):
+            if (etpConst['uid'] == 0) and (not self.readOnly):
+                self.createLicensedataTable()
+            return set()
+        licenses = self.retrieveLicense(idpackage)
+        licenses = licenses.split()
+        licdata = set()
+        for licname in licenses:
+            licname = licname.strip()
+            if not licname.isalnum():
+                continue
+            self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (licname,))
+            licidentifier = self.cursor.fetchone()
+            if licidentifier:
+                licdata.add(licidentifier[0])
+
+        self.storeInfoCache(idpackage,'retrieveLicensedataKeys',licdata)
+        return licdata
 
     def retrieveLicense(self, idpackage):
 
@@ -2616,8 +2691,20 @@ class etpDatabase:
 
         return result
 
+    def isLicensedataKeyAvailable(self, license_name):
+        if not self.doesTableExist("licensedata"):
+            if (etpConst['uid'] == 0) and (not self.readOnly):
+                self.createLicensedataTable()
+            else:
+                return True
+        self.cursor.execute('SELECT licensename FROM licensedata WHERE licensename = (?)', (license_name,))
+        result = self.cursor.fetchone()
+        if not result:
+            return False
+        return True
+
     def isLicenseAvailable(self,pkglicense):
-        if not pkglicense: # workaround for packages without a license but just garbage
+        if not pkglicense or not pkglicense.isalnum(): # workaround for packages without a license but just garbage
             pkglicense = ' '
         self.cursor.execute('SELECT idlicense FROM licenses WHERE license = (?)', (pkglicense,))
         result = self.cursor.fetchone()
@@ -3376,6 +3463,7 @@ class etpDatabase:
         self.createExtrainfoIndex()
         self.createNeededIndex()
         self.createUseflagsIndex()
+        self.createLicensedataIndex()
 
     def createNeededIndex(self):
         if self.dbname != etpConst['serverdbid'] and self.indexing:
@@ -3397,6 +3485,16 @@ class etpDatabase:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS baseindex ON baseinfo ( idpackage, atom, name, version, versiontag, slot, branch, revision )')
             self.commitChanges()
 
+    def createLicensedataIndex(self):
+        if self.dbname != etpConst['serverdbid'] and self.indexing:
+            if not self.doesTableExist("licensedata"):
+                if (etpConst['uid'] == 0) and (not self.readOnly):
+                    self.createLicensedataTable()
+                else:
+                    return
+            self.cursor.execute('CREATE INDEX IF NOT EXISTS licensedataindex ON licensedata ( licensename )')
+            self.commitChanges()
+
     def createKeywordsIndex(self):
         if self.dbname != etpConst['serverdbid'] and self.indexing:
             self.cursor.execute('CREATE INDEX IF NOT EXISTS keywordsindex ON keywords ( idpackage, idkeyword )')
@@ -3517,6 +3615,9 @@ class etpDatabase:
         except:
             pass
 
+    def createLicensedataTable(self):
+        self.cursor.execute('CREATE TABLE licensedata ( licensename VARCHAR UNIQUE, text BLOB, compressed INTEGER );')
+
     def createTriggerTable(self):
         self.cursor.execute('CREATE TABLE triggers ( idpackage INTEGER PRIMARY KEY, data BLOB );')
 

libraries/entropy.py

@@ -2684,6 +2684,9 @@ class PackageInterface:
         data['injected'] = False
         data['counter'] = -1 # gentoo counter will be set in self.__install_package_into_gentoo_database()
 
+        # FIXME: this will be removed create all indexes on the client db
+        self.Entropy.clientDbconn.createAllIndexes()
+
         idpk, rev, x, status = self.Entropy.clientDbconn.handlePackage(etpData = data, forcedRevision = data['revision'])
         del x
         del data

libraries/entropyConstants.py

@@ -67,6 +67,7 @@ etpData = {
     'needed': u"", # runtime libraries needed by the package
     'trigger': u"", # this will become a bool, containing info about external trigger presence
     'injected': bool, # if the package has been injected manually, this will be true
+    'licensedata': dict # dictionary that contains license text
 }
 '''
 
@@ -304,6 +305,12 @@ CREATE TABLE binkeywords (
     idkeyword INTEGER
 );
 
+CREATE TABLE licensedata (
+    licensename VARCHAR UNIQUE,
+    text BLOB,
+    compressed INTEGER
+);
+
 """
 
 # ETP_ARCH_CONST setup
@@ -319,6 +326,7 @@ etpSys = {
     'rootdir': "",
     'maxthreads': 100,
     'dirstoclean': set(),
+    'serverside': False,
 }
 
 etpUi = {
@@ -520,6 +528,7 @@ def initConfig_entropyConstants(rootdir):
         'entropyxpakfilename': "metadata.xpak", # Gentoo xpak metadata file name
 
         'etpdatabasemaskfile': ETP_DBFILE+".mask", # the local/remote database revision file
+        'etpdatabaselicwhitelistfile': ETP_DBFILE+".lic_whitelist", # the local/remote database revision file
         'etpdatabaserevisionfile': ETP_DBFILE+".revision", # the local/remote database revision file
         'etpdatabasehashfile': ETP_DBFILE+".md5", # its checksum
         'etpdatabasedumphashfilebz2': ETP_DBFILE+".dump.bz2.md5",

libraries/entropyTools.py

@@ -2101,6 +2101,27 @@ def extractPkgData(package, etpBranch = etpConst['branch'], silent = False, inje
             pass
     '''
 
+    # Get License text if possible
+    licenses_dir = None
+    try:
+        from portageTools import getPortageEnv
+        licenses_dir = os.path.join(getPortageEnv('PORTDIR'),'licenses')
+    except:
+        pass
+    data['licensedata'] = {}
+    if licenses_dir:
+        licdata = [x.strip() for x in data['license'].split() if x.strip().isalnum()]
+        for mylicense in licdata:
+            licfile = os.path.join(licenses_dir,mylicense)
+            if os.access(licfile,os.R_OK):
+                if istextfile(licfile):
+                    f = open(licfile,"rb")
+                    f.seek(0,2)
+                    size = f.tell()
+                    f.seek(0)
+                    data['licensedata'][mylicense] = f.read(size)
+                    f.close()
+
     if not silent: print_info(yellow(" * ")+red(info_package+"Getting package mirrors list..."),back = True)
     # manage data['sources'] to create data['mirrorlinks']
     # =mirror://openoffice|link1|link2|link3

libraries/serverConstants.py

@@ -21,6 +21,7 @@
 '''
 
 from entropyConstants import *
+etpSys['serverside'] = True
 
 def initConfig_serverConstants():