Entropy/EntropyDatabaseInterface:

- implement table structure for supporting package sets - add methods to handle them: clearPackageSets(), insertPackageSets(), retrievePackageSets() Entropy/Socket Server Interface: - if command is not valid, return a CL answer Entropy/Socket Server Client Interface: - handle CL answers on receive() Entropy/ServerInterface: - add support for package sets, loading data from sets/ database dir subdirectory - automatically handle them by updating the db at every change Entropy/Repository Daemon: - add EAPI3 support for package sets Entropy/RepoInterface/EquoInterface: - add support for package sets on sync - add package sets match and search functions more tomorrow git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@2735 cd1c1023-2f26-0410-ae45-c471fc1f0318
2008-12-07 22:38:42 +00:00
parent 22d3400e7a
commit fd85e10164
2 changed files with 298 additions and 38 deletions
@@ -1865,6 +1865,65 @@ class EquoInterface(TextInterface):

        return dbpkginfo

+    def packageSetList(self, server_repos = [], serverInstance = None):
+        return self.packageSetMatch('', server_repos = server_repos, serverInstance = serverInstance, search = True)
+
+    def packageSetSearch(self, package_set, server_repos = [], serverInstance = None):
+        return self.packageSetMatch(package_set, server_repos = server_repos, serverInstance = serverInstance, search = True)
+
+    def packageSetMatch(self, package_set, multiMatch = False, server_repos = [], serverInstance = None, search = False):
+
+        if server_repos:
+            if not serverInstance:
+                t = _("server_repos needs serverInstance")
+                raise exceptionTools.IncorrectParameter("IncorrectParameter: %s" % (t,))
+            valid_repos = server_repos[:]
+        else:
+            valid_repos = self.validRepositories
+
+        def open_db(repoid):
+            if server_repos:
+                dbconn = serverInstance.openServerDatabase(just_reading = True, repo = repoid)
+            else:
+                dbconn = self.openRepositoryDatabase(repoid)
+            return dbconn
+
+        # if we search, we return all the matches available
+        if search: multiMatch = True
+
+        set_data = []
+
+        while 1:
+
+            # check inside PackageSettings
+            if not server_repos:
+                if search:
+                    mysets = [x for x in self.PackageSettings['system_package_sets'].keys() if (x.find(package_set) != -1)]
+                    for myset in mysets:
+                        mydata = self.PackageSettings['system_package_sets'].get(myset)
+                        set_data.append((etpConst['userpackagesetsid'], myset, mydata.copy(),))
+                else:
+                    mydata = self.PackageSettings['system_package_sets'].get(package_set)
+                    if mydata != None: set_data.append((etpConst['userpackagesetsid'], package_set, mydata,))
+                    if not multiMatch: break
+
+            for repoid in valid_repos:
+                dbconn = open_db(repoid)
+                if search:
+                    mysets = dbconn.searchSets(package_set)
+                    for myset in mysets:
+                        mydata = dbconn.retrievePackageSet(myset)
+                        set_data.append((repoid, myset, mydata.copy(),))
+                else:
+                    mydata = dbconn.retrievePackageSet(package_set)
+                    if mydata: set_data.append((repoid, package_set, mydata,))
+                    if not multiMatch: break
+
+            break
+
+        if multiMatch: return set_data
+        if not set_data: return {}
+        return set_data.pop(0)

    def repository_move_clear_cache(self, repoid = None):
        self.clear_dump_cache(etpCache['world_available'])
@@ -6656,6 +6715,14 @@ class RepoInterface:
        if not isinstance(data,dict): return None,None
        return data.get('digest'), data.get('actions')

+    def get_eapi3_package_sets(self, eapi3_interface, repo, session):
+        self.socket.setdefaulttimeout(self.big_socket_timeout)
+        data = eapi3_interface.CmdInterface.get_package_sets(
+            session, repo, etpConst['currentarch'], etpConst['product']
+        )
+        if not isinstance(data,dict): return {}
+        return data
+
    def handle_eapi3_database_sync(self, repo, threshold = 1500, chunk_size = 12):

        def prepare_exit(mysock, session = None):
@@ -6716,35 +6783,6 @@ class RepoInterface:
            prepare_exit(eapi3_interface, session)
            return False

-        # get treeupdates stuff
-        dbdigest, treeupdates_actions = self.get_eapi3_database_treeupdates(eapi3_interface, repo, session)
-        if dbdigest == None:
-            mydbconn.closeDB()
-            prepare_exit(eapi3_interface, session)
-            mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("treeupdates data not available")),)
-            self.Entropy.updateProgress(
-                mytxt,
-                importance = 0,
-                type = "info",
-                header = blue("  # "),
-            )
-            return None
-
-        try:
-            mydbconn.setRepositoryUpdatesDigest(repo, dbdigest)
-            mydbconn.bumpTreeUpdatesActions(treeupdates_actions)
-        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
-            mydbconn.closeDB()
-            prepare_exit(eapi3_interface, session)
-            mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update treeupdates data")),)
-            self.Entropy.updateProgress(
-                mytxt,
-                importance = 0,
-                type = "info",
-                header = blue("  # "),
-            )
-            return None
-
        count = 0
        added_segments = []
        mytmp = set()
@@ -6844,6 +6882,53 @@ class RepoInterface:

        del added_segments

+        # get treeupdates stuff
+        dbdigest, treeupdates_actions = self.get_eapi3_database_treeupdates(eapi3_interface, repo, session)
+        if dbdigest == None:
+            mydbconn.closeDB()
+            prepare_exit(eapi3_interface, session)
+            mytxt = "%s: %s" % ( blue(_("EAPI3 Service status")), darkred(_("treeupdates data not available")),)
+            self.Entropy.updateProgress(
+                mytxt,
+                importance = 0,
+                type = "info",
+                header = blue("  # "),
+            )
+            return None
+
+        try:
+            mydbconn.setRepositoryUpdatesDigest(repo, dbdigest)
+            mydbconn.bumpTreeUpdatesActions(treeupdates_actions)
+        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
+            mydbconn.closeDB()
+            prepare_exit(eapi3_interface, session)
+            mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update treeupdates data")),)
+            self.Entropy.updateProgress(
+                mytxt,
+                importance = 0,
+                type = "info",
+                header = blue("  # "),
+            )
+            return None
+
+
+        # get updated package sets
+        repo_sets = self.get_eapi3_package_sets(eapi3_interface, repo, session)
+        try:
+            mydbconn.clearPackageSets()
+            mydbconn.insertPackageSets(repo_sets)
+        except (self.dbapi2.DatabaseError,self.dbapi2.IntegrityError,self.dbapi2.OperationalError,):
+            mydbconn.closeDB()
+            prepare_exit(eapi3_interface, session)
+            mytxt = "%s: %s" % (blue(_("EAPI3 Service status")), darkred(_("cannot update package sets data")),)
+            self.Entropy.updateProgress(
+                mytxt,
+                importance = 0,
+                type = "info",
+                header = blue("  # "),
+            )
+            return None
+
        # I don't need you anymore
        # disconnect socket
        prepare_exit(eapi3_interface, session)
@@ -10165,6 +10250,7 @@ timeout=10

 class PackageSettings:

+    import entropyTools
    def __init__(self, EquoInstance):

        if not isinstance(EquoInstance,EquoInterface):
@@ -10180,6 +10266,7 @@ class PackageSettings:
            'repos_system_mask': {},
            'repos_mask': {},
            'repos_license_whitelist': {},
+            'system_package_sets': {},
        }

        self.etpMtimeFiles = {
@@ -10190,6 +10277,7 @@ class PackageSettings:
            'repos_system_mask': {},
            'repos_mask': {},
            'repos_license_whitelist': {},
+            'system_package_sets': {},
        }

        self.__settings = None
@@ -10204,6 +10292,9 @@ class PackageSettings:
    def clear(self):
        self.__settings = None

+    def keys(self):
+        return self.__settings.keys()
+
    def scan(self):

        self.__settings = self.parse()
@@ -10239,6 +10330,9 @@ class PackageSettings:
        if self.__settings == None: self.scan()
        return mykey in self.__settings

+    def __cmp__(self, other):
+        return cmp(self.__settings,other)
+
    def get(self, mykey):
        if self.__settings == None: self.scan()
        return self.__settings.get(mykey)
@@ -10250,26 +10344,34 @@ class PackageSettings:
    def parse(self):

        # append repositories mask files
+        # append repositories mtime files
        for repoid in etpRepositoriesOrder:
            maskpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasemaskfile'])
            wlpath = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabaselicwhitelistfile'])
            sm_path = os.path.join(etpRepositories[repoid]['dbpath'],etpConst['etpdatabasesytemmaskfile'])
            if os.path.isfile(maskpath) and os.access(maskpath,os.R_OK):
                self.etpMaskFiles['repos_mask'][repoid] = maskpath
+                self.etpMtimeFiles['repos_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasemaskfile']+".mtime"
            if os.path.isfile(wlpath) and os.access(wlpath,os.R_OK):
                self.etpMaskFiles['repos_license_whitelist'][repoid] = wlpath
+                self.etpMtimeFiles['repos_license_whitelist'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabaselicwhitelistfile']+".mtime"
            if os.path.isfile(sm_path) and os.access(sm_path,os.R_OK):
                self.etpMaskFiles['repos_system_mask'][repoid] = sm_path
-
-        # append repositories mtime files
-        for repoid in etpRepositoriesOrder:
-            if repoid in self.etpMaskFiles['repos_mask']:
-                self.etpMtimeFiles['repos_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasemaskfile']+".mtime"
-            if repoid in self.etpMaskFiles['repos_license_whitelist']:
-                self.etpMtimeFiles['repos_license_whitelist'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabaselicwhitelistfile']+".mtime"
-            if repoid in self.etpMaskFiles['repos_system_mask']:
                self.etpMtimeFiles['repos_system_mask'][repoid] = etpConst['dumpstoragedir']+"/repo_"+repoid+"_"+etpConst['etpdatabasesytemmaskfile']+".mtime"

+        # user defined package sets
+        sets_dir = etpConst['confsetsdir']
+        if (os.path.isdir(sets_dir) and os.access(sets_dir,os.R_OK)):
+            set_files = [x for x in os.listdir(sets_dir) if (os.path.isfile(os.path.join(sets_dir,x)) and os.access(os.path.join(sets_dir,x),os.R_OK))]
+            for set_file in set_files:
+                try:
+                    set_file = str(set_file)
+                except (UnicodeDecodeError,UnicodeEncodeError,):
+                    continue
+                set_filepath = os.path.join(sets_dir,set_file)
+                self.etpMaskFiles['system_package_sets'][set_file] = set_filepath
+                self.etpMtimeFiles['system_package_sets'][set_file] = etpConst['dumpstoragedir']+"/system_package_set_"+set_file+".mtime"
+
        data = {}
        for item in self.etpMaskFiles:
            myattr = '%s_parser' % (item,)
@@ -10443,6 +10545,16 @@ class PackageSettings:

        return data

+    def system_package_sets_parser(self):
+
+        data = {}
+        for set_name in self.etpMaskFiles['system_package_sets']:
+            set_filepath = self.etpMaskFiles['system_package_sets'][set_name]
+            set_elements = self.entropyTools.extract_packages_from_set_file(set_filepath)
+            if set_elements: data[set_name] = set_elements.copy()
+
+        return data
+

    '''
    internal functions
@@ -13484,6 +13596,8 @@ class SocketHostInterface:

                cmd = self.server.processor.process(self.buffered_data, self.request, self.client_address)
                if cmd == 'close':
+                    # send KAPUTT signal JA!
+                    self.transmit(self.HostInterface.answers['cl'])
                    return True
                self.buffered_data = ''
                return False
@@ -15429,6 +15543,14 @@ class ServerInterface(TextInterface):
                use_branch,
                lock_remote
            )] = conn
+
+        # auto-update package sets
+        if not read_only:
+            cur_sets = conn.retrievePackageSets()
+            sys_sets = self.get_configured_package_sets(repo)
+            if cur_sets != sys_sets:
+                self.update_database_package_sets(repo)
+
        return conn

    def deps_tester(self):
@@ -17369,6 +17491,18 @@ class ServerInterface(TextInterface):

        return sets_data

+    def update_database_package_sets(self, repo = None):
+
+        if repo == None: repo = self.default_repository
+
+        package_sets = self.get_configured_package_sets(repo)
+        dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
+        # clear old
+        dbconn.clearPackageSets()
+        if package_sets: dbconn.insertPackageSets(package_sets)
+        dbconn.commitChanges()
+
+
 class RemoteDbSkelInterface:

    def __init__(self):
@@ -20590,6 +20724,16 @@ class RepositorySocketServerInterface(SocketHostInterface):
                    'syntax': "<SESSION_ID> repository_server:treeupdates <repository> <arch> <product> <branch>",
                    'from': unicode(self), # from what class
                },
+                'repository_server:get_package_sets': {
+                    'auth': False,
+                    'built_in': False,
+                    'cb': self.docmd_package_sets,
+                    'args': ["myargs"],
+                    'as_user': False,
+                    'desc': "returns repository package sets metadata",
+                    'syntax': "<SESSION_ID> repository_server:get_package_sets <repository> <arch> <product> <branch>",
+                    'from': unicode(self), # from what class
+                }
            }

        def trash_old_databases(self):
@@ -20629,6 +20773,41 @@ class RepositorySocketServerInterface(SocketHostInterface):

            return {'removed': removed_ids, 'added': added_ids, 'checksum': mychecksum}

+        def docmd_package_sets(self, myargs):
+
+            self.trash_old_databases()
+
+            if len(myargs) < 4:
+                return None
+            repository = myargs[0]
+            arch = myargs[1]
+            product = myargs[2]
+            try:
+                branch = str(myargs[3])
+            except (UnicodeEncodeError,UnicodeDecodeError,):
+                return None
+
+            x = (repository,arch,product,branch,)
+            valid = self.HostInterface.is_repository_available(x)
+            if not valid:
+                return valid
+
+            cached = self.HostInterface.get_dcache((repository, arch, product, branch, 'docmd_package_sets'), repository)
+            if cached != None:
+                return cached
+
+            dbpath = self.get_database_path(repository, arch, product, branch)
+            dbconn = self.HostInterface.open_db(dbpath, docache = False)
+
+            # get data
+            data = dbconn.retrievePackageSets()
+
+            self.HostInterface.set_dcache((repository, arch, product, branch, 'docmd_package_sets'), data, repository)
+            dbconn.closeDB()
+
+            return data
+
+
        def docmd_treeupdates(self, myargs):

            self.trash_old_databases()
@@ -21233,6 +21412,18 @@ class RepositorySocketClientCommands(EntropySocketClientCommands):
        )
        return self.do_generic_handler(cmd, session_id, tries = 5)

+    def get_package_sets(self, session_id, repository, arch, product):
+
+        cmd = "%s %s %s %s %s %s" % (
+            session_id,
+            'repository_server:get_package_sets',
+            repository,
+            arch,
+            product,
+            etpConst['branch'],
+        )
+        return self.do_generic_handler(cmd, session_id, tries = 5)
+
    def get_package_information(self, session_id, idpackages, repository, arch, product):

        cmd = "%s %s %s %s %s %s %s %s" % (
@@ -21811,7 +22002,23 @@ class SystemSocketClientInterface:
                data = do_receive()
                if self.buffer_length == None:
                    self.buffered_data = ''
-                    if len(data) < len(myeos):
+                    if (data == '') or (data == self.answers['cl']):
+                        # nein! no support, KAPUTT!
+                        # RAUSS!
+                        if not self.quiet:
+                            mytxt = _("command not supported. receive aborted")
+                            self.Entropy.updateProgress(
+                                "[%s:%s] %s" % (
+                                        brown(self.hostname),
+                                        bold(str(self.hostport)),
+                                        blue(mytxt),
+                                ),
+                                importance = 1,
+                                type = "warning",
+                                header = self.output_header
+                            )
+                        return None
+                    elif len(data) < len(myeos):
                        if not self.quiet:
                            mytxt = _("malformed EOS. receive aborted")
                            self.Entropy.updateProgress(
@@ -28469,6 +28676,7 @@ class ServerMirrorsInterface:
            )

            self.sync_database_treeupdates(repo)
+            self.Entropy.update_database_package_sets(repo)
            self.Entropy.close_server_databases()

            # backup current database to avoid re-indexing
@@ -31894,6 +32102,34 @@ class EntropyDatabaseInterface:
            return True
        return False

+    def clearPackageSets(self):
+        self.checkReadOnly()
+        self.cursor.execute('DELETE FROM packagesets')
+
+    def insertPackageSets(self, sets_data):
+        self.checkReadOnly()
+
+        def myiter():
+            for setname in sets_data:
+                for dependency in sets_data[setname]:
+                    yield (setname,dependency)
+
+        with self.WriteLock:
+            self.cursor.executemany('INSERT into packagesets VALUES (?,?)', myiter())
+
+    def retrievePackageSets(self):
+        self.cursor.execute('SELECT setname,dependency FROM packagesets')
+        data = self.fetchall()
+        sets = {}
+        for setname, dependency in data:
+            if not sets.has_key(setname):
+                sets[setname] = set()
+            sets[setname].add(dependency)
+        return sets
+
+    def retrievePackageSet(self, setname):
+        self.cursor.execute('SELECT dependency FROM packagesets WHERE setname = (?)', (setname,))
+        return self.fetchall2set(self.fetchall())

    def retrieveAtom(self, idpackage):
        self.cursor.execute('SELECT atom FROM baseinfo WHERE idpackage = (?)', (idpackage,))
@@ -32577,6 +32813,10 @@ class EntropyDatabaseInterface:
        self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = (?)', (iddep,))
        return self.fetchall2set(self.cursor.fetchall())

+    def searchSets(self, keyword):
+        self.cursor.execute('SELECT DISTINCT(setname) FROM packagesets WHERE setname LIKE (?)', ("%"+keyword+"%",))
+        return self.fetchall2set(self.cursor.fetchall())
+
    def searchPackages(self, keyword, sensitive = False, slot = None, tag = None, branch = None, order_by = 'atom'):

        searchkeywords = ["%"+keyword+"%"]
@@ -32976,6 +33216,9 @@ class EntropyDatabaseInterface:
        if not self.doesTableExist("categoriesdescription"):
            self.createCategoriesdescriptionTable()

+        if not self.doesTableExist('packagesets'):
+            self.createPackagesetsTable()
+
        # these are the tables moved to INTEGER PRIMARY KEY AUTOINCREMENT
        autoincrement_tables = [
            'treeupdatesactions',
@@ -33484,7 +33727,13 @@ class EntropyDatabaseInterface:
        self.createEclassesIndex()
        self.createCategoriesIndex()
        self.createCompileFlagsIndex()
+        self.createPackagesetsIndex()

+    def createPackagesetsIndex(self):
+        if self.indexing:
+            with self.WriteLock:
+                self.cursor.execute('CREATE INDEX IF NOT EXISTS packagesetsindex ON packagesets ( setname )')
+                self.commitChanges()

    def createNeededIndex(self):
        if self.indexing:
@@ -33702,6 +33951,10 @@ class EntropyDatabaseInterface:
        self.cursor.execute('ALTER TABLE counterstemp RENAME TO counters')
        self.commitChanges()

+    def createPackagesetsTable(self):
+        with self.WriteLock:
+            self.cursor.execute('CREATE TABLE packagesets ( setname VARCHAR, dependency VARCHAR );')
+
    def createCategoriesdescriptionTable(self):
        with self.WriteLock:
            self.cursor.execute('CREATE TABLE categoriesdescription ( category VARCHAR, locale VARCHAR, description VARCHAR );')
@@ -119,6 +119,7 @@ DROP TABLE IF EXISTS licenses_accepted;
 DROP TABLE IF EXISTS trashedcounters;
 DROP TABLE IF EXISTS entropy_misc_counters;
 DROP TABLE IF EXISTS categoriesdescription;
+DROP TABLE IF EXISTS packagesets;
 """

 etpSQLInit = """
@@ -337,6 +338,11 @@ CREATE TABLE categoriesdescription (
    description VARCHAR
 );

+CREATE TABLE packagesets (
+    setname VARCHAR,
+    dependency VARCHAR
+);
+
 """

 # ETP_ARCH_CONST setup
@@ -603,6 +609,7 @@ def const_defaultSettings(rootdir):
        'rss-managing-editor': "lxnay@sabayonlinux.org", # updates submitter
        'rss-notice-board': "notice.rss", # repository RSS-based notice board content

+        'userpackagesetsid': '__user__',
        'packagesext': ".tbz2",
        'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file
        'packagesexpirationfileext': ".expired", # Extension of the file that "contains" expiration mtime