Entropy:

- fix some possible encoding issues on QAInterface.content_test()
- preparing to implement the Community Repositories interface


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1959 cd1c1023-2f26-0410-ae45-c471fc1f0318

libraries/entropy.py

@@ -6092,6 +6092,7 @@ class QAInterface:
         count = 0
         maxcount = len(idpackages)
         for idpackage in idpackages:
+            count += 1
             atom = dbconn.retrieveAtom(idpackage)
             scan_msg = "%s, %s:" % (blue(_("scanning for broken depends")),darkgreen(atom),)
             self.Entropy.updateProgress(
@@ -6265,6 +6266,7 @@ class QAInterface:
 
         mylibs = {}
         for myfile in mycontent:
+            myfile = myfile.encode('raw_unicode_escape')
             if not os.access(myfile,os.R_OK):
                 continue
             if not os.path.isfile(myfile):
@@ -6415,6 +6417,9 @@ class QAInterface:
 '''
 class FtpInterface:
 
+    import ftplib
+    import entropyTools
+    import socket
     # this must be run before calling the other functions
     def __init__(self, ftpuri, EntropyInterface, verbose = True):
 
@@ -6424,14 +6429,7 @@ class FtpInterface:
                 raise exceptionTools.IncorrectParameter("IncorrectParameter: %s, (! %s !)" % (EntropyInterface,mytxt,))
 
         self.Entropy = EntropyInterface
-        import entropyTools
-        self.entropyTools = entropyTools
-        import ftplib
-        self.ftplib = ftplib
-        import socket
-        self.socket = socket
         self.verbose = verbose
-
         self.oldprogress = 0.0
 
         # import FTP modules
@@ -6460,7 +6458,7 @@ class FtpInterface:
         self.ftpport = ftpuri.split(":")[-1]
         try:
             self.ftpport = int(self.ftpport)
-        except:
+        except ValueError:
             self.ftpport = 21
 
         self.ftpdir = ftpuri.split("ftp://")[-1]
@@ -6499,8 +6497,7 @@ class FtpInterface:
                 type = "info",
                 header = darkgreen(" * ")
             )
-        self.ftpconn.cwd(self.ftpdir)
-        self.currentdir = self.ftpdir
+        self.setCWD(self.ftpdir, dodir = True)
 
     def setBasedir(self):
         rc = self.setCWD(self.ftpdir)
@@ -6552,7 +6549,7 @@ class FtpInterface:
         pwd = self.ftpconn.pwd()
         return pwd
 
-    def setCWD(self,mydir):
+    def setCWD(self, mydir, dodir = False):
         if self.verbose:
             mytxt = _("switching to")
             self.Entropy.updateProgress(
@@ -6561,7 +6558,14 @@ class FtpInterface:
                 type = "info",
                 header = darkgreen(" * ")
             )
-        self.ftpconn.cwd(mydir)
+        try:
+            self.ftpconn.cwd(mydir)
+        except self.ftplib.error_perm, e:
+            if e[0][:3] == '550' and dodir:
+                self.recursiveMkdir(mydir)
+                self.ftpconn.cwd(mydir)
+            else:
+                raise
         self.currentdir = self.getCWD()
 
     def setPASV(self,bool):
@@ -6618,9 +6622,16 @@ class FtpInterface:
         except:
             return False
 
+    def recursiveMkdir(self, mypath):
+        mydirs = [x for x in mypath.split("/") if x]
+        mycurpath = ""
+        for mydir in mydirs:
+            mycurpath = os.path.join(mycurpath,mydir)
+            if not self.isFileAvailable(mycurpath):
+                self.mkdir(mycurpath)
+
     def mkdir(self,directory):
-        # FIXME: add rc
-        self.ftpconn.mkd(directory)
+        return self.ftpconn.mkd(directory)
 
     # this function also supports callback, because storbinary doesn't
     def advancedStorBinary(self, cmd, fp, callback=None):
@@ -12382,7 +12393,7 @@ class SocketUrlFetcher(urlFetcher):
 
 class ServerInterface(TextInterface):
 
-    def __init__(self, default_repository = None, save_repository = False):
+    def __init__(self, default_repository = None, save_repository = False, community_repo = False):
 
         if etpConst['uid'] != 0:
             mytxt = _("Entropy ServerInterface must be run as root")
@@ -12394,6 +12405,7 @@ class ServerInterface(TextInterface):
             header = "[server]"
         )
 
+        self.community_repo = community_repo
         self.dbapi2 = dbapi2 # export for third parties
         # settings
         etpSys['serverside'] = True
@@ -12407,7 +12419,7 @@ class ServerInterface(TextInterface):
         self.do_save_repository = save_repository
         self.default_repository = default_repository
         if self.default_repository == None:
-            self.default_repository = etpConst['officialrepositoryid']
+            self.default_repository = etpConst['officialserverrepositoryid']
 
         if self.default_repository not in etpConst['server_repositories']:
             raise exceptionTools.PermissionDenied("PermissionDenied: %s %s" % (
@@ -12441,7 +12453,7 @@ class ServerInterface(TextInterface):
         backup_list = [
             'etpdatabaseclientfilepath',
             'clientdbid',
-            'officialrepositoryid'
+            'officialserverrepositoryid'
         ]
         for setting in backup_list:
             if setting not in self.settings_to_backup:
@@ -12476,7 +12488,7 @@ class ServerInterface(TextInterface):
                     )
             )
         self.close_server_databases()
-        etpConst['officialrepositoryid'] = repoid
+        etpConst['officialserverrepositoryid'] = repoid
         self.default_repository = repoid
         self.setup_services()
         if save:
@@ -12484,6 +12496,38 @@ class ServerInterface(TextInterface):
 
         self.show_interface_status()
 
+        if not self.is_repository_initialized(self.default_repository):
+            mytxt = blue("%s.") % (_("Your default repository is not initialized"),)
+            self.updateProgress(
+                "[%s:%s] %s" % (
+                        brown("repo"),
+                        purple(self.default_repository),
+                        mytxt,
+                ),
+                importance = 1,
+                type = "warning",
+                header = darkred(" !!! ")
+            )
+            answer = self.askQuestion(_("Do you want to initialize your default repository ?"))
+            if answer == "No":
+                mytxt = red("%s.") % (_("You have taken the risk to continue with an uninitialized repository"),)
+                self.updateProgress(
+                    "[%s:%s] %s" % (
+                            brown("repo"),
+                            purple(self.default_repository),
+                            mytxt,
+                    ),
+                    importance = 1,
+                    type = "warning",
+                    header = darkred(" !!! ")
+                )
+            else:
+                # move empty database for security sake
+                dbfile = self.get_local_database_file(self.default_repository)
+                if os.path.isfile(dbfile):
+                    shutil.move(dbfile,dbfile+".backup")
+                self.initialize_server_database(empty = True, repo = self.default_repository, warnings = False)
+
 
     def show_interface_status(self):
         mytxt = _("Entropy Server Interface Instance on repository") # ..on repository: <repository_name>
@@ -12511,29 +12555,29 @@ class ServerInterface(TextInterface):
 
 
     def save_default_repository(self, repoid):
-        if os.path.isfile(etpConst['repositoriesconf']):
-            f = open(etpConst['repositoriesconf'],"r")
+        if os.path.isfile(etpConst['serverconf']):
+            f = open(etpConst['serverconf'],"r")
             content = f.readlines()
             f.close()
             content = [x.strip() for x in content]
             found = False
             new_content = []
             for line in content:
-                if line.strip().startswith("officialrepositoryid|"):
-                    line = "officialrepositoryid|%s" % (repoid,)
+                if line.strip().startswith("officialserverrepositoryid|"):
+                    line = "officialserverrepositoryid|%s" % (repoid,)
                     found = True
                 new_content.append(line)
             if not found:
-                new_content.append("officialrepositoryid|%s" % (repoid,))
-            f = open(etpConst['repositoriesconf']+".save_default_repo_tmp","w")
+                new_content.append("officialserverrepositoryid|%s" % (repoid,))
+            f = open(etpConst['serverconf']+".save_default_repo_tmp","w")
             for line in new_content:
                 f.write(line+"\n")
             f.flush()
             f.close()
-            shutil.move(etpConst['repositoriesconf']+".save_default_repo_tmp",etpConst['repositoriesconf'])
+            shutil.move(etpConst['serverconf']+".save_default_repo_tmp",etpConst['serverconf'])
         else:
-            f = open(etpConst['repositoriesconf'],"w")
-            f.write("officialrepositoryid|%s\n" % (repoid,))
+            f = open(etpConst['serverconf'],"w")
+            f.write("officialserverrepositoryid|%s\n" % (repoid,))
             f.flush()
             f.close()
 
@@ -12572,7 +12616,17 @@ class ServerInterface(TextInterface):
             self.ClientService.backup_setting(setting)
 
 
-    def openServerDatabase(self, read_only = True, no_upload = True, just_reading = False, repo = None, indexing = True):
+    def is_repository_initialized(self, repo):
+        dbc = self.openServerDatabase(just_reading = True, repo = repo)
+        valid = True
+        try:
+            dbc.validateDatabase()
+        except exceptionTools.SystemDatabaseError:
+            valid = False
+        self.close_server_database(dbc)
+        return valid
+
+    def openServerDatabase(self, read_only = True, no_upload = True, just_reading = False, repo = None, indexing = True, warnings = True):
 
         if repo == None:
             repo = self.default_repository
@@ -12621,7 +12675,7 @@ class ServerInterface(TextInterface):
             # sometimes, when filling a new server db, we need to avoid tree updates
             if valid:
                 conn.serverUpdatePackagesData()
-            else:
+            elif warnings:
                 mytxt = _( "Entropy database is probably empty. If you don't agree with what I'm saying, " + \
                            "then it's probably corrupted! I won't stop you here btw..."
                 )
@@ -13545,7 +13599,7 @@ class ServerInterface(TextInterface):
         dbconn.setCounter(idpackage,counter)
         dbconn.setInjected(idpackage)
 
-    def initialize_server_database(self, empty = True, repo = None):
+    def initialize_server_database(self, empty = True, repo = None, warnings = True):
 
         if repo == None:
             repo = self.default_repository
@@ -13568,7 +13622,7 @@ class ServerInterface(TextInterface):
 
         if os.path.isfile(self.get_local_database_file(repo)):
 
-            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo)
+            dbconn = self.openServerDatabase(read_only = True, no_upload = True, repo = repo, warnings = warnings)
 
             if dbconn.doesTableExist("baseinfo") and dbconn.doesTableExist("extrainfo"):
                 idpackages = dbconn.listAllIdpackages()
@@ -13610,9 +13664,9 @@ class ServerInterface(TextInterface):
         # initialize
         dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
         dbconn.initializeDatabase()
-        myQA = self.QA()
 
         if not empty:
+
             revisions_file = "/entropy-revisions-dump.txt"
             # dump revisions - as a backup
             if revisions_match:
@@ -13663,7 +13717,8 @@ class ServerInterface(TextInterface):
                 pkg_branch_dir = os.path.join(self.get_local_packages_directory(repo),mybranch)
                 pkglist = os.listdir(pkg_branch_dir)
                 # filter .md5 and .expired packages
-                pkglist = [x for x in pkglist if x[-5:] == etpConst['packagesext'] and not os.path.isfile(os.path.join(pkg_branch_dir,x+etpConst['packagesexpirationfileext']))]
+                pkglist = [x for x in pkglist if x[-5:] == etpConst['packagesext'] and not \
+                    os.path.isfile(os.path.join(pkg_branch_dir,x+etpConst['packagesexpirationfileext']))]
 
                 if not pkglist:
                     continue
@@ -13735,9 +13790,11 @@ class ServerInterface(TextInterface):
 
             self.depends_table_initialize(repo)
 
-        if idpackages_added:
-            dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
-            myQA.scan_missing_dependencies(idpackages_added, dbconn, ask = True, repo = repo)
+            myQA = self.QA()
+
+            if idpackages_added:
+                dbconn = self.openServerDatabase(read_only = False, no_upload = True, repo = repo)
+                myQA.scan_missing_dependencies(idpackages_added, dbconn, ask = True, repo = repo, self_check = True)
 
         dbconn.commitChanges()
         self.close_server_databases()
@@ -14459,6 +14516,11 @@ class ServerInterface(TextInterface):
 
         return switched, already_switched, ignored, not_found, no_checksum
 
+class CommunityServerInterface(ServerInterface):
+
+    def __init__(self):
+        ServerInterface.__init__(self, community_repo = True)
+
 
 class ServerMirrorsInterface:
 
@@ -14520,7 +14582,7 @@ class ServerMirrorsInterface:
             )
 
             ftp = self.FtpInterface(uri, self.Entropy)
-            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo), dodir = True)
 
             if lock and ftp.isFileAvailable(etpConst['etpdatabaselockfile']):
                 self.Entropy.updateProgress(
@@ -14595,7 +14657,7 @@ class ServerMirrorsInterface:
             )
 
             ftp = self.FtpInterface(uri, self.Entropy)
-            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo), dodir = True)
 
             if lock and ftp.isFileAvailable(etpConst['etpdatabasedownloadlockfile']):
                 self.Entropy.updateProgress(
@@ -14640,12 +14702,12 @@ class ServerMirrorsInterface:
 
         if not ftp_connection:
             ftp_connection = self.FtpInterface(uri, self.Entropy)
-            ftp_connection.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            ftp_connection.setCWD(self.Entropy.get_remote_database_relative_path(repo), dodir = True)
         else:
             mycwd = ftp_connection.getCWD()
             if mycwd != self.Entropy.get_remote_database_relative_path(repo):
                 ftp_connection.setBasedir()
-                ftp_connection.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+                ftp_connection.setCWD(self.Entropy.get_remote_database_relative_path(repo), dodir = True)
 
         crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
         lock_string = ''
@@ -14940,7 +15002,22 @@ class ServerMirrorsInterface:
         for uri in self.Entropy.get_remote_mirrors(repo):
 
             ftp = self.FtpInterface(uri, self.Entropy)
-            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            try:
+                ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            except ftp.ftplib.error_perm:
+                crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
+                self.Entropy.updateProgress(
+                    "[repo:%s|%s] %s !" % (
+                            brown(repo),
+                            darkgreen(crippled_uri),
+                            blue(_("mirror doesn't have a valid directory structure")),
+                    ),
+                    importance = 1,
+                    type = "warning",
+                    header = darkred(" !!! ")
+                )
+                ftp.closeConnection()
+                continue
             cmethod = etpConst['etpdatabasecompressclasses'].get(etpConst['etpdatabasefileformat'])
             if cmethod == None:
                 raise exceptionTools.InvalidDataType("InvalidDataType: %s." % (
@@ -14996,7 +15073,11 @@ class ServerMirrorsInterface:
         for uri in self.Entropy.get_remote_mirrors(repo):
             data = [uri,False,False]
             ftp = FtpInterface(uri, self.Entropy)
-            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            try:
+                ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
+            except ftp.ftplib.error_perm:
+                ftp.closeConnection()
+                continue
             if ftp.isFileAvailable(etpConst['etpdatabaselockfile']):
                 # upload locked
                 data[1] = True
@@ -15339,7 +15420,7 @@ class ServerMirrorsInterface:
                     type = "info",
                     header = blue(" @@ ")
                 )
-                ftp.setCWD(self.ftp_basedir)
+                ftp.setCWD(self.ftp_basedir, dodir = True)
 
                 maxcount = len(self.myfiles)
                 counter = 0
@@ -15564,12 +15645,7 @@ class ServerMirrorsInterface:
         gave_up = False
         crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
         ftp = self.FtpInterface(uri, self.Entropy)
-        try:
-            ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo))
-        except:
-            self.create_mirror_directories(ftp,self.Entropy.get_remote_database_relative_path(repo))
-            ftp.closeConnection()
-            return gave_up # no errors, mirror is unlocked
+        ftp.setCWD(self.Entropy.get_remote_database_relative_path(repo), dodir = True)
 
         lock_file = self.get_database_lockfile(repo)
         if not os.path.isfile(lock_file) and ftp.isFileAvailable(os.path.basename(lock_file)):
@@ -16250,11 +16326,7 @@ class ServerMirrorsInterface:
         remote_packages_data = {}
 
         def do_cwd():
-            try:
-                ftp_connection.setCWD(self.Entropy.get_remote_packages_relative_path(repo))
-            except:
-                self.create_mirror_directories(ftp_connection,self.Entropy.get_remote_packages_relative_path(repo))
-                ftp_connection.setCWD(self.Entropy.get_remote_packages_relative_path(repo))
+            ftp_connection.setCWD(self.Entropy.get_remote_packages_relative_path(repo), dodir = True)
             if not ftp_connection.isFileAvailable(branch):
                 ftp_connection.mkdir(branch)
             ftp_connection.setCWD(branch)

libraries/entropyConstants.py

@@ -626,6 +626,7 @@ def const_defaultSettings(rootdir):
             '/usr/lib/openmotif-2.2/libMrm.so.3',
             '/usr/lib/openmotif-2.2/libXm.so.3'
         ],
+        'officialserverrepositoryid': "sabayonlinux.org", # our official repository name
         'officialrepositoryid': "sabayonlinux.org", # our official repository name
         'conntestlink': "http://www.google.com", 
         'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes)
@@ -1219,6 +1220,9 @@ def const_readServerSettings():
             if etpConst['branch'] not in etpConst['branches']:
                 etpConst['branches'].append(etpConst['branch'])
 
+        elif (line.find("officialserverrepositoryid|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2):
+            etpConst['officialserverrepositoryid'] = line.split("|")[1].strip()
+
         elif line.startswith("repository|") and (len(line.split("|")) == 5):
 
             repoid = line.split("|")[1].strip()
@@ -1298,7 +1302,7 @@ def const_setupWithEnvironment():
 
     shell_repoid = os.getenv('ETP_REPO')
     if shell_repoid:
-        etpConst['officialrepositoryid'] = shell_repoid
+        etpConst['officialserverrepositoryid'] = shell_repoid
 
     expiration_days = os.getenv('ETP_EXPIRATION_DAYS')
     if expiration_days: