diff --git a/client/equo b/client/equo index 78eb7c41a..3aa325336 100644 --- a/client/equo +++ b/client/equo @@ -352,7 +352,14 @@ except Exception: print print_error(darkred("!!! If the error is about missing tables in the database, just run 'equo database generate' again !!!")) print_error(blue("Ok, back here. Let me see if you are connected to the Internet. Yes, I am blue now, so?")) - conntest = entropyTools.get_remote_data("http://svn.sabayonlinux.org") + testlink = "http://www.google.com" + if etpRepositories.has_key(etpConst['officialrepositoryid']): + try: + testlink = etpRepositories[etpConst['officialrepositoryid']]['database'] + except KeyError: + pass + + conntest = entropyTools.get_remote_data(testlink) if (conntest != False): print_error(darkgreen("Of course you are on the Internet...")) rc = Text.askQuestion(" Erm... Can I send the error to my creators so they can fix me?") diff --git a/client/text_repositories.py b/client/text_repositories.py index e69a3c227..8126dd884 100644 --- a/client/text_repositories.py +++ b/client/text_repositories.py @@ -64,7 +64,7 @@ def showRepositories(): sourcecount += 1 print_info(red("\t\tPackages Mirror #"+str(sourcecount)+" : ")+darkgreen(pkgrepo)) print_info(red("\t\tDatabase URL: ")+darkgreen(etpRepositories[repo]['database'])) - print_info(red("\t\tRepository name: ")+bold(repo)) + print_info(red("\t\tRepository identifier: ")+bold(repo)) print_info(red("\t\tRepository database path: ")+blue(etpRepositories[repo]['dbpath'])) return 0 diff --git a/client/text_rescue.py b/client/text_rescue.py index f9dabe0fa..b6b0902ba 100644 --- a/client/text_rescue.py +++ b/client/text_rescue.py @@ -516,7 +516,7 @@ def getinfo(dict = False): info['Entropy unpack directory'] = etpConst['entropyunpackdir'] info['Entropy packages directory'] = etpConst['packagesbindir'] info['Entropy logging directory'] = etpConst['logdir'] - info['Entropy Official Repository name'] = etpConst['officialrepositoryname'] + info['Entropy Official Repository identifier'] = etpConst['officialrepositoryid'] info['Entropy API'] = etpConst['etpapi'] info['Equo pidfile'] = etpConst['pidfile'] info['Entropy database tag'] = etpConst['databasestarttag'] diff --git a/conf/repositories.conf b/conf/repositories.conf index 3f04fb715..ac8c76f9e 100644 --- a/conf/repositories.conf +++ b/conf/repositories.conf @@ -44,3 +44,11 @@ product|standard # # Branch default setting branch|3.5 + +# +# syntax for branch: +# +# officialrepositoryid: Repository identifier for the official main distribution repository +# officialrepositoryid| +# +officialrepositoryid|sabayonlinux.org diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index 0adec330c..ed4563ff1 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -340,7 +340,7 @@ class etpDatabase(TextInterface): etpConst['treeupdatescalled'] = True - repository = etpConst['officialrepositoryname'] + repository = etpConst['officialrepositoryid'] doRescan = False if repositoryUpdatesDigestCache_db.has_key(repository): diff --git a/libraries/entropy.py b/libraries/entropy.py index a4ebb5f53..a342aa9a4 100644 --- a/libraries/entropy.py +++ b/libraries/entropy.py @@ -152,7 +152,9 @@ class EquoInterface(TextInterface): def loadRepositoryDatabase(self, repositoryName, xcache = True, indexing = True): dbfile = etpRepositories[repositoryName]['dbpath']+"/"+etpConst['etpdatabasefile'] if not os.path.isfile(dbfile): - self.fetch_repository_if_not_available(repositoryName) + rc = self.fetch_repository_if_not_available(repositoryName) + if rc != 0: + raise exceptionTools.RepositoryError("RepositoryError: problems fetching database for repo id: "+reponame) conn = self.databaseTools.etpDatabase(readOnly = True, dbFile = dbfile, clientDatabase = True, dbname = etpConst['dbnamerepoprefix']+repositoryName, xcache = xcache, indexing = indexing) # initialize CONFIG_PROTECT if (etpRepositories[repositoryName]['configprotect'] == None) or \ @@ -624,7 +626,10 @@ class EquoInterface(TextInterface): mhash = "-1" return mhash + def fetch_repository_if_not_available(self, reponame): + if fetch_repository_if_not_available_cache.has_key(reponame): + return fetch_repository_if_not_available_cache.get(reponame) # open database rc = 0 dbfile = etpRepositories[reponame]['dbpath']+"/"+etpConst['etpdatabasefile'] @@ -632,11 +637,10 @@ class EquoInterface(TextInterface): # sync repoConn = self.Repositories(reponames = [reponame]) rc = repoConn.sync() - if rc != 0: - raise exceptionTools.RepositoryError("RepositoryError: cannot fetch database for repo id: "+reponame) del repoConn - if not os.path.isfile(dbfile): - raise exceptionTools.RepositoryError("RepositoryError: cannot fetch database for repo id: "+reponame) + if os.path.isfile(dbfile): + rc = 0 + fetch_repository_if_not_available_cache[reponame] = rc return rc ''' @@ -658,12 +662,10 @@ class EquoInterface(TextInterface): pass repoResults = {} - exitErrors = {} for repo in etpRepositories: # sync database if not available rc = self.fetch_repository_if_not_available(repo) if (rc != 0): - exitErrors[repo] = -1 continue # open database dbconn = self.openRepositoryDatabase(repo) @@ -856,6 +858,79 @@ class EquoInterface(TextInterface): atomMatchCache[atom]['etpRepositories'] = etpRepositories.copy() return repoResults[reponame],reponame + def addRepository(self, repodata): + # update etpRepositories + try: + etpRepositories[repodata['repoid']] = {} + etpRepositories[repodata['repoid']]['description'] = repodata['description'] + etpRepositories[repodata['repoid']]['packages'] = repodata['packages'][:] + etpRepositories[repodata['repoid']]['configprotect'] = None + etpRepositories[repodata['repoid']]['configprotectmask'] = None + except KeyError: + raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted") + + if repodata['repoid'].endswith(".tbz2"): # dynamic repository + try: + etpRepositories[repodata['repoid']]['smartpackage'] = repodata['smartpackage'] + etpRepositories[repodata['repoid']]['dbpath'] = repodata['dbpath'] + etpRepositories[repodata['repoid']]['pkgpath'] = repodata['pkgpath'] + except KeyError: + raise exceptionTools.InvalidData("InvalidData: repodata dictionary is corrupted") + # put at top priority, shift others + myrepo_order = set([(x[0]+1,x[1]) for x in etpRepositoriesOrder]) + etpRepositoriesOrder.clear() + etpRepositoriesOrder.update(myrepo_order) + etpRepositoriesOrder.add((1,repodata['repoid'])) + else: + # XXX it's boring to keep this in sync with entropyConstants stuff, solutions? + etpRepositories[repodata['repoid']]['database'] = repodata['database'] + etpRepositories[repodata['repoid']]['dbcformat'] = repodata['dbcformat'] + etpRepositories[repodata['repoid']]['dbpath'] = etpConst['etpdatabaseclientdir'] + "/" + repodata['repoid'] + "/" + etpConst['product'] + "/" + etpConst['currentarch'] + # set dbrevision + myrev = self.get_repository_revision(repodata['repoid']) + if myrev == -1: + myrev = 0 + etpRepositories[repodata['repoid']]['dbrevision'] = str(myrev) + myrepocount = max([x[0] for x in etpRepositoriesOrder])+1 + etpRepositoriesOrder.add((myrepocount,repodata['repoid'])) + # clean world_available cache + self.dumpTools.dumpobj(etpCache['world_available'], {}) + # clean world_update cache + self.dumpTools.dumpobj(etpCache['world_update'], {}) + # clean check_update_package_cache + check_package_update_cache.clear() + self.dumpTools.dumpobj(etpCache['check_package_update'],{}) + # save new etpRepositories to file + self.entropyTools.saveRepositoriesSettings(etpRepositories) + + def removeRepository(self, repoid): + # update etpRepositories + done = False + try: + del etpRepositories[repoid] + done = True + except: + pass + + if done: + diff = set() + for data in etpRepositoriesOrder: + if data[1] == repoid: + diff.add(data) + etpRepositoriesOrder.difference_update(diff) + # it's not vital to reset etpRepositoriesOrder counters + + # clean world_available cache + self.dumpTools.dumpobj(etpCache['world_available'], {}) + # clean world_update cache + self.dumpTools.dumpobj(etpCache['world_update'], {}) + # clean check_update_package_cache + check_package_update_cache.clear() + self.dumpTools.dumpobj(etpCache['check_package_update'],{}) + # save new etpRepositories to file + self.entropyTools.saveRepositoriesSettings(etpRepositories) + + ''' @description: filter the already installed dependencies @input dependencies: list of dependencies to check @@ -1461,19 +1536,14 @@ class EquoInterface(TextInterface): return -1,atoms_contained etpSys['dirstoclean'].add(os.path.dirname(dbfile)) # add dbfile - etpRepositories[basefile] = {} - etpRepositories[basefile]['description'] = "Dynamic database from "+basefile - etpRepositories[basefile]['packages'] = [] - etpRepositories[basefile]['dbpath'] = os.path.dirname(dbfile) - etpRepositories[basefile]['pkgpath'] = os.path.realpath(tbz2file) # extra info added - etpRepositories[basefile]['configprotect'] = set() - etpRepositories[basefile]['configprotectmask'] = set() - etpRepositories[basefile]['smartpackage'] = False # extra info added - # put at top priority, shift others - myrepo_order = set([(x[0]+1,x[1]) for x in etpRepositoriesOrder]) - etpRepositoriesOrder.clear() - etpRepositoriesOrder.update(myrepo_order) - etpRepositoriesOrder.add((1,basefile)) + repodata = {} + repodata['repoid'] = basefile + repodata['description'] = "Dynamic database from "+basefile + repodata['packages'] = [] + repodata['dbpath'] = os.path.dirname(dbfile) + repodata['pkgpath'] = os.path.realpath(tbz2file) # extra info added + repodata['smartpackage'] = False # extra info added + self.addRepository(repodata) mydbconn = self.openGenericDatabase(dbfile) # read all idpackages try: @@ -3389,6 +3459,7 @@ class RepoInterface: self.dbupdated = False self.newEquo = False self.alreadyUpdated = 0 + self.notAvailable = 0 # check if I am root if (not self.Entropy.entropyTools.isRoot()): @@ -3556,7 +3627,7 @@ class RepoInterface: def sync(self): - # clse them + # close them self.Entropy.closeAllRepositoryDatabases() # let's dance! @@ -3566,6 +3637,7 @@ class RepoInterface: header = darkred(" @@ ") ) + self.dbupdated = False repocount = 0 repolength = len(self.reponames) for repo in self.reponames: @@ -3612,8 +3684,6 @@ class RepoInterface: self.Entropy.cycleDone() continue - # database is going to be updated - self.dbupdated = True # clear database interface cache belonging to this repository self.clear_repository_cache(repo) cmethod = self.__validate_compression_method(repo) @@ -3634,8 +3704,12 @@ class RepoInterface: header = "\t" ) self.Entropy.cycleDone() + self.notAvailable += 1 continue + # database is going to be updated + self.dbupdated = True + # unpack database self.Entropy.updateProgress( red("Unpacking database to ") + darkgreen(etpConst['etpdatabasefile'])+red(" ..."), importance = 0, @@ -3747,6 +3821,10 @@ class RepoInterface: header = darkred(" !! ") ) + if (self.notAvailable > 0): + return 1 + elif (self.notAvailable >= len(self.reponames)): + return 2 return 0 ''' @@ -4123,7 +4201,7 @@ class urlFetcher: self.speedUpdater.start() # set timeout - socket.setdefaulttimeout(60) + socket.setdefaulttimeout(20) # get file size if available try: diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index a6f9a5fb5..29345ad4d 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -416,6 +416,8 @@ linkerPaths = set() repositoryUpdatesDigestCache_db = {} repositoryUpdatesDigestCache_disk = {} +fetch_repository_if_not_available_cache = {} + ### Application disk cache def const_resetCache(): for item in dbCacheStore: @@ -443,6 +445,7 @@ def const_resetCache(): repositoryUpdatesDigestCache_db.clear() repositoryUpdatesDigestCache_disk.clear() check_package_update_cache.clear() + fetch_repository_if_not_available_cache.clear() # Inside it you'll find instantiated vartree classes portageRoots = {} @@ -620,7 +623,7 @@ def initConfig_entropyConstants(rootdir): '/var/lib/scrollkeeper', '/usr/src', '/etc/skel', '/etc/ssh', '/etc/ssl', '/var/run', '/var/spool/cron', '/var/lib/init.d', '/lib/modules', '/etc/env.d', '/etc/gconf', '/etc/runlevels', '/lib/splash/cache', '/usr/share/mime', '/etc/portage' ], - 'officialrepositoryname': "sabayonlinux.org", # our official repository name + 'officialrepositoryid': "sabayonlinux.org", # our official repository name 'databasestarttag': "|ENTROPY:PROJECT:DB:MAGIC:START|", # tag to append to .tbz2 file before entropy database (must be 32bytes) 'pidfile': "/var/run/equo.pid", 'applicationlock': False, @@ -849,6 +852,10 @@ def initConfig_entropyConstants(rootdir): import exceptionTools raise exceptionTools.DirectoryNotFound("DirectoryNotFound: please run this as root at least once or create: "+etpConst['packagesbindir']+"/"+branch) + elif (line.find("officialrepositoryid|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): + officialreponame = line.split("|")[1] + etpConst['officialrepositoryid'] = officialreponame + # align etpConst['binaryurirelativepath'] and etpConst['etpurirelativepath'] with etpConst['product'] etpConst['binaryurirelativepath'] = etpConst['product']+"/"+etpConst['binaryurirelativepath'] etpConst['etpurirelativepath'] = etpConst['product']+"/"+etpConst['etpurirelativepath'] diff --git a/libraries/entropyTools.py b/libraries/entropyTools.py index c113ff839..d85ece864 100644 --- a/libraries/entropyTools.py +++ b/libraries/entropyTools.py @@ -191,7 +191,7 @@ def md5sum(filepath): block = readfile.read(1024) while block: m.update(block) - block = readfile.read(1024) + block = readfile.read(1024) return m.hexdigest() def md5sum_directory(directory): @@ -273,38 +273,39 @@ def unpackXpak(xpakfile, tmpdir = None): except: return None return tmpdir - + + def suckXpak(tbz2file, outputpath): - + xpakpath = outputpath+"/"+os.path.basename(tbz2file)[:-5]+".xpak" old = open(tbz2file,"rb") db = open(xpakpath,"wb") db_tmp = open(xpakpath+".reverse","wb") allowWrite = False - + # position old to the end old.seek(0,2) # read backward until we find bytes = old.tell() counter = bytes - + while counter >= 0: - old.seek(counter-bytes,2) - byte = old.read(1) - if byte == "P" or byte == "K": - old.seek(counter-bytes-7,2) - chunk = old.read(7)+byte - if chunk == "XPAKPACK": - allowWrite = False + old.seek(counter-bytes,2) + byte = old.read(1) + if byte == "P" or byte == "K": + old.seek(counter-bytes-7,2) + chunk = old.read(7)+byte + if chunk == "XPAKPACK": + allowWrite = False db_tmp.write(chunk[::-1]) - break - elif chunk == "XPAKSTOP": - allowWrite = True - old.seek(counter-bytes,2) - if (allowWrite): - db_tmp.write(byte) - counter -= 1 - + break + elif chunk == "XPAKSTOP": + allowWrite = True + old.seek(counter-bytes,2) + if (allowWrite): + db_tmp.write(byte) + counter -= 1 + db_tmp.flush() db_tmp.close() db_tmp = open(xpakpath+".reverse","rb") @@ -317,7 +318,7 @@ def suckXpak(tbz2file, outputpath): byte = db_tmp.read(1) db.write(byte) counter -= 1 - + db.flush() db.close() db_tmp.close() @@ -477,37 +478,37 @@ def allocateMaskedFile(file, fromfile): previousfile = "" while 1: - counter += 1 - txtcounter = str(counter) - oldtxtcounter = str(counter-1) - for x in range(4-len(txtcounter)): - txtcounter = "0"+txtcounter - oldtxtcounter = "0"+oldtxtcounter - newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file) - if counter > 0: - previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file) - else: - previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file) - if not os.path.exists(newfile): - break + counter += 1 + txtcounter = str(counter) + oldtxtcounter = str(counter-1) + for x in range(4-len(txtcounter)): + txtcounter = "0"+txtcounter + oldtxtcounter = "0"+oldtxtcounter + newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file) + if counter > 0: + previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file) + else: + previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file) + if not os.path.exists(newfile): + break if not newfile: - newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file) + newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file) else: - + if os.path.exists(previousfile): - + # compare fromfile with previousfile new = md5sum(fromfile) old = md5sum(previousfile) if new == old: return previousfile, False - - # compare old and new, if they match, suggest previousfile directly - new = md5sum(file) - old = md5sum(previousfile) - if (new == old): - return previousfile, False - + + # compare old and new, if they match, suggest previousfile directly + new = md5sum(file) + old = md5sum(previousfile) + if (new == old): + return previousfile, False + return newfile, True def extractElog(file): @@ -1441,28 +1442,57 @@ def umountProc(): return True class lifobuffer: - + def __init__(self): self.counter = -1 self.buf = {} - + def push(self,item): self.counter += 1 self.buf[self.counter] = item - + def pop(self): if self.counter == -1: return None self.counter -= 1 return self.buf[self.counter+1] -def writeNewBranch(branch): - +def read_repositories_conf(): content = [] if os.path.isfile(etpConst['repositoriesconf']): f = open(etpConst['repositoriesconf']) content = f.readlines() f.close() + return content + +def saveRepositoriesSettings(repodata): + + import shutil + content = read_repositories_conf() + content = [x for x in content if not x.startswith("repository|")] + # inject new repodata + for repoid in repodata: + if repoid.endswith(".tbz2"): # skip live repositories + continue + line = "repository|%s|%s|%s|%s#%s\n" % ( repoid, + repodata[repoid]['description'], + ' '.join(repodata[repoid]['packages']), + repodata[repoid]['database'], + repodata[repoid]['dbcformat'], + ) + content.append(line) + if os.path.isfile(etpConst['repositoriesconf']): + if os.path.isfile(etpConst['repositoriesconf']+".old"): + os.remove(etpConst['repositoriesconf']+".old") + shutil.copy2(etpConst['repositoriesconf'],etpConst['repositoriesconf']+".old") + f = open(etpConst['repositoriesconf'],"w") + f.writelines(content) + f.flush() + f.close() + +def writeNewBranch(branch): + + content = read_repositories_conf() found = False new_content = [] diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 8b62f3022..6942bdfd4 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -53,10 +53,10 @@ def generator(package, dbconnection = None, enzymeRequestBranch = etpConst['bran idpk, revision, etpDataUpdated, accepted = dbconn.handlePackage(mydata) - # add package info to our official repository etpConst['officialrepositoryname'] + # add package info to our official repository etpConst['officialrepositoryid'] if (accepted): dbconn.removePackageFromInstalledTable(idpk) - dbconn.addPackageToInstalledTable(idpk,etpConst['officialrepositoryname']) + dbconn.addPackageToInstalledTable(idpk,etpConst['officialrepositoryid']) if dbconnection is None: dbconn.commitChanges() diff --git a/spritz/src/spritz.glade b/spritz/src/spritz.glade index d8f2739b6..953611066 100644 --- a/spritz/src/spritz.glade +++ b/spritz/src/spritz.glade @@ -4816,12 +4816,13 @@ gz 10 - + True True True GTK_RELIEF_NORMAL True + diff --git a/spritz/src/spritz.py b/spritz/src/spritz.py index 470092dc7..3136cada0 100644 --- a/spritz/src/spritz.py +++ b/spritz/src/spritz.py @@ -161,6 +161,14 @@ class SpritzController(Controller): def on_repoCancel_clicked( self, widget ): self.addrepo_ui.addRepoWin.hide() + def on_repoInsert_clicked( self, widget ): + text = inputBox(self.addrepo_ui.addRepoWin, _("Insert Repository"), _("Insert Repository identification string")+" ") + if text: + if (text.startswith("repository|")) and (len(text.split("|")) == 5): + print text,"is valid" + else: + okDialog( self.addrepo_ui.addRepoWin, _("This Repository identification string is malformed") ) + def on_removeRepo_clicked( self, widget ): # get selected repo selection = self.repoView.view.get_selection()