diff --git a/TODO b/TODO index c4293b2c8..0c8c0ab5a 100644 --- a/TODO +++ b/TODO @@ -1,19 +1,17 @@ TODO list: - reagent: complete smartapps section CLIENT: - - regenerate client database from / content - - triggers: - - alternatives eclass - :: sqlite needs it (/usr/bin/lemon) - + - add external triggerable hooks support - branch reorganization based on the release: - install only looks inside the configured branch - move branches to version naming (3.5, 3.6, 3.7) instead of using stable, unstable... + - atom|tag ? - add scripts directory that contains pre/post-install-remove scripts that is synced using activator - include more triggers - optimize database interface - add multithreading support + - limit items/size in on-disk cache? Project Status: - reagent: complete. Stabilization mode. diff --git a/client/confTools.py b/client/confTools.py index 84831f785..acdaf6b3f 100644 --- a/client/confTools.py +++ b/client/confTools.py @@ -347,7 +347,7 @@ def scanfs(quiet = True, dcache = True): scandata[counter] = mydict.copy() try: - if (not quiet): print_info("("+blue(str(counter))+") "+red(" file: ")+filepath) + if (not quiet): print_info("("+blue(str(counter))+") "+red(" file: ")+os.path.dirname(filepath)+"/"+os.path.basename(filepath)[10:]) except: pass # possible encoding issues # store data diff --git a/client/equo b/client/equo index f6938ece3..979c95b1b 100644 --- a/client/equo +++ b/client/equo @@ -40,9 +40,7 @@ def print_help(): print_info(" \t"+blue("status")+brown("\t\t show respositories status")) print_info(" \t"+blue("search")+brown("\t\t search a package trough repositories")) - print_info(" \t"+blue("world")+brown("\t\t update packages to the latest version")) - print_info(" \t\t"+darkgreen("update")+red("\t\t\t update the system in the current branch")) - print_info(" \t\t"+darkgreen("upgrade")+red("\t\t\t upgrade the system to the latest branch")) + print_info(" \t"+blue("world")+brown("\t\t update system with the latest available packages")) print_info(" \t\t"+red("--ask")+"\t\t\t ask before making any changes") print_info(" \t\t"+red("--fetch")+"\t\t\t just download packages without doing the install") print_info(" \t\t"+red("--pretend")+"\t\t just show what would be done") @@ -85,12 +83,13 @@ def print_help(): print_info(" \t"+blue("database")+brown("\t handle installed packages database")) print_info(" \t\t"+darkgreen("generate")+red("\t\t generate installed packages database")) + print_info(" \t\t"+darkgreen("resurrect")+red("\t\t generate installed packages database using system content [last hope]")) print_info(" \t\t"+darkgreen("depends")+red("\t\t\t to regenerate/generate depends caching table")) print_info(" \t"+blue("cache")+brown("\t\t handle Equo on-disk cache")) print_info(" \t\t"+darkgreen("clean")+red("\t\t\t clean on-disk cache")) print_info(" \t\t"+darkgreen("generate")+red("\t\t generate on-disk cache (to speed up Equo)")) print_info(" \t\t"+red("--quiet")+"\t\t\t show less details (useful for scripting)") - print_info(" \t\t"+red("--verbose")+"\t\t\t show more details about what's going on") + print_info(" \t\t"+red("--verbose")+"\t\t show more details about what's going on") print_info(" \t"+blue("cleanup")+brown("\t\t remove downloaded packages and clean temporary directories (not cache)")) print_info(" \t"+blue("--info")+brown("\t\t show system information")) diff --git a/client/equoTools.py b/client/equoTools.py index ff75ce60a..d5795e4c8 100644 --- a/client/equoTools.py +++ b/client/equoTools.py @@ -328,11 +328,11 @@ def fetchRepositoryIfNotAvailable(reponame): ''' @description: matches the package that user chose, using dbconnection.atomMatch searching in all available repositories. @input atom: user choosen package name - @output: the matched selection, list: [package id,repository name] | if nothing found, returns: [ -1,1 ] + @output: the matched selection, list: [package id,repository name] | if nothing found, returns: ( -1,1 ) @ exit errors: -1 => repository cannot be fetched online ''' -def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = [], xcache = True): +def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = (), xcache = True): # no one seems to use matchBranches :D #print atom if xcache: @@ -356,7 +356,7 @@ def atomMatch(atom, caseSentitive = True, matchSlot = None, matchBranches = [], dbconn = openRepositoryDatabase(repo, xcache = xcache) # search - query = dbconn.atomMatch(atom, caseSentitive, matchSlot, matchBranches = matchBranches) + query = dbconn.atomMatch(atom, caseSensitive = caseSentitive, matchSlot = matchSlot, matchBranches = matchBranches) if query[1] == 0: # package found, add to our dictionary repoResults[repo] = query[0] @@ -574,11 +574,11 @@ def getDependencies(packageInfo): dbconn = openRepositoryDatabase(reponame) # retrieve dependencies - depend = dbconn.retrieveDependencies(idpackage) + depend = set(dbconn.retrieveDependencies(idpackage)) # and conflicts - conflicts = dbconn.retrieveConflicts(idpackage) + conflicts = set(dbconn.retrieveConflicts(idpackage)) for x in conflicts: - depend.append("!"+x) + depend.add("!"+x) dbconn.closeDB() ''' caching ''' @@ -1476,7 +1476,7 @@ def installPackageIntoGentooDatabase(infoDict,packageFile): #print _portage_getInstalledAtom(key) atomsfound = set() for xdir in os.listdir(portDbDir): - if xdir == (infoDict['category']): + if (xdir == infoDict['category']): for ydir in os.listdir(portDbDir+"/"+xdir): if (key == dep_getkey(xdir+"/"+ydir)): atomsfound.add(xdir+"/"+ydir) @@ -1574,7 +1574,6 @@ def installPackageIntoDatabase(idpackage, repository): clientDbconn.addDependRelationToDependsTable(iddep,match[0]) except: - print "DEBUG!!! dependstable not found" clientDbconn.regenerateDependsTable() clientDbconn.closeDB() @@ -1671,13 +1670,8 @@ def package(options): rc = 127 elif (options[0] == "world"): - myopts = options[1:] - doupgrade = False - for opt in myopts: - if opt == "upgrade": - doupgrade = True loadCaches() - rc, status = worldUpdate(ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, onlyfetch = equoRequestOnlyFetch, upgrade = doupgrade) + rc, status = worldUpdate(ask = equoRequestAsk, pretend = equoRequestPretend, verbose = equoRequestVerbose, onlyfetch = equoRequestOnlyFetch) elif (options[0] == "remove"): if len(myopts) > 0: @@ -1706,9 +1700,10 @@ def database(options): if (options[0] == "generate"): - print_warning(bold("####### ATTENTION -> ")+red("The installed package database will be regenerated, this will take a LOT of time.")) + print_warning(bold("####### ATTENTION -> ")+red("The installed package database will be regenerated.")) print_warning(bold("####### ATTENTION -> ")+red("Sabayon Linux Officially Repository MUST be on top of the repositories list in ")+etpConst['repositoriesconf']) print_warning(bold("####### ATTENTION -> ")+red("This method is only used for testing at the moment and you need Portage installed. Don't worry about Portage warnings.")) + print_warning(bold("####### ATTENTION -> ")+red("Please use this function ONLY if you are using an Entropy-enabled Sabayon distribution.")) rc = askquestion(" Can I continue ?") if rc == "No": sys.exit(0) @@ -1718,6 +1713,11 @@ def database(options): rc = askquestion(" Do you even know what you're doing ?") if rc == "No": sys.exit(0) + + # clean caches + import cacheTools + cacheTools.cleanCache(quiet = True) + const_resetCache() # ok, he/she knows it... hopefully # if exist, copy old database @@ -1791,6 +1791,101 @@ def database(options): clientDbconn.closeDB() + elif (options[0] == "resurrect"): + + print_warning(bold("####### ATTENTION -> ")+red("The installed package database will be resurrected, this will take a LOT of time.")) + print_warning(bold("####### ATTENTION -> ")+red("Please use this function ONLY if you are using an Entropy-enabled Sabayon distribution.")) + rc = askquestion(" Can I continue ?") + if rc == "No": + sys.exit(0) + rc = askquestion(" Are you REALLY sure ?") + if rc == "No": + sys.exit(0) + rc = askquestion(" Do you even know what you're doing ?") + if rc == "No": + sys.exit(0) + + # clean caches + import cacheTools + cacheTools.cleanCache(quiet = True) + const_resetCache() + + # ok, he/she knows it... hopefully + # if exist, copy old database + print_info(red(" @@ ")+blue("Creating backup of the previous database, if exists.")+red(" @@")) + newfile = backupClientDatabase() + if (newfile): + print_info(red(" @@ ")+blue("Previous database copied to file ")+newfile+red(" @@")) + + # Now reinitialize it + print_info(darkred(" Initializing the new database at "+bold(etpConst['etpdatabaseclientfilepath'])), back = True) + clientDbconn = openClientDatabase() + clientDbconn.initializeDatabase() + print_info(darkgreen(" Database reinitialized correctly at "+bold(etpConst['etpdatabaseclientfilepath']))) + + print_info(red(" Collecting installed files. Writing: "+etpConst['packagestmpfile']+" Please wait..."), back = True) + + # since we use find, see if it's installed + find = os.system("which find &> /dev/null") + if find != 0: + print_error(darkred("Attention: ")+red("You must have 'find' installed!")) + return + # spawn process + if os.path.isfile(etpConst['packagestmpfile']): + os.remove(etpConst['packagestmpfile']) + os.system("find / -mount 1> "+etpConst['packagestmpfile']) + if not os.path.isfile(etpConst['packagestmpfile']): + print_error(darkred("Attention: ")+red("find couldn't generate an output file.")) + return + + f = open(etpConst['packagestmpfile'],"r") + # creating list of files + filelist = set() + file = f.readline().strip() + while file: + filelist.add(file) + file = f.readline().strip() + f.close() + entries = len(filelist) + + print_info(red(" Found "+str(entries)+" files on the system. Assigning packages...")) + atoms = {} + pkgsfound = set() + + for repo in etpRepositories: + print_info(red(" Matching in repository: ")+etpRepositories[repo]['description']) + # get all idpackages + dbconn = openRepositoryDatabase(repo) + idpackages = dbconn.listAllIdpackages(branch = etpConst['branch']) + count = str(len(idpackages)) + cnt = 0 + for idpackage in idpackages: + cnt += 1 + idpackageatom = dbconn.retrieveAtom(idpackage) + print_info(" ("+str(cnt)+"/"+count+")"+red(" Matching files from packages..."), back = True) + # content + content = dbconn.retrieveContent(idpackage) + for file in content: + if file in filelist: + pkgsfound.add((idpackage,repo)) + atoms[(idpackage,repo)] = idpackageatom + filelist.difference_update(set(content)) + break + dbconn.closeDB() + + print_info(red(" Found "+str(len(pkgsfound))+" packages. Filling database...")) + count = str(len(pkgsfound)) + cnt = 0 + #XXXos.remove(etpConst['packagestmpfile']) + + for pkgfound in pkgsfound: + cnt += 1 + print_info(" ("+str(cnt)+"/"+count+") "+red("Adding: ")+atoms[pkgfound], back = True) + installPackageIntoDatabase(pkgfound[0],pkgfound[1]) + + print_info(red(" Database resurrected successfully.")) + print_warning(red(" Keep in mind that virtual/meta packages couldn't be matched. They don't own any files.")) + elif (options[0] == "depends"): print_info(red(" Regenerating depends caching table...")) clientDbconn = openClientDatabase() @@ -1917,18 +2012,14 @@ def openClientDatabase(xcache = True): # -def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False, upgrade = False): +def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False): # check if I am root if (not isRoot()) and (not pretend): print_error(red("You must run this function as superuser.")) return 1,-1 - if (not upgrade): - branches = [etpConst['branch']] - else: - branches = etpConst['branches'] - + branches = (etpConst['branch'],) updateList = [] fineList = set() removedList = set() @@ -1982,7 +2073,7 @@ def worldUpdate(ask = False, pretend = False, verbose = False, onlyfetch = False if (updateList): print_info(red(" @@ ")+blue("Calculating queue...")) - rc = installPackages(atomsdata = updateList, ask = ask, pretend = pretend, verbose = verbose, onlyfetch = onlyfetch, deepdeps = upgrade) + rc = installPackages(atomsdata = updateList, ask = ask, pretend = pretend, verbose = verbose, onlyfetch = onlyfetch) if rc[0] != 0: return rc else: diff --git a/client/queryTools.py b/client/queryTools.py index 8a994689e..8e854d788 100644 --- a/client/queryTools.py +++ b/client/queryTools.py @@ -27,7 +27,7 @@ from entropyConstants import * from clientConstants import * from outputTools import * from databaseTools import etpDatabase -from entropyTools import dep_getkey +from entropyTools import dep_getkey, dep_getslot, remove_slot from equoTools import openClientDatabase, openRepositoryDatabase, printPackageInfo, generateDependsTree, atomMatch # move them away? ######################################################## @@ -101,7 +101,9 @@ def searchInstalledPackages(packages, idreturn = False, quiet = False): dataInfo = [] # when idreturn is True for package in packages: - result = clientDbconn.searchPackages(package) + slot = dep_getslot(package) + package = remove_slot(package) + result = clientDbconn.searchPackages(package, slot = slot) if (result): # print info if (not idreturn) and (not quiet): @@ -132,6 +134,8 @@ def searchBelongs(files, idreturn = False, quiet = False): clientDbconn = openClientDatabase() dataInfo = [] # when idreturn is True + results = {} + flatresults = {} for file in files: like = False if file.find("*") != -1: @@ -139,9 +143,18 @@ def searchBelongs(files, idreturn = False, quiet = False): out = re.subn("\*","%",file) file = out[0] like = True - result = clientDbconn.searchBelongs(file, like) - if (result): + results[file] = set() + idpackages = clientDbconn.searchBelongs(file, like) + for idpackage in idpackages: + if not flatresults.get(idpackage): + results[file].add(idpackage) + flatresults[idpackage] = True + + if (results): + for result in results: # print info + file = result + result = results[result] if (not idreturn) and (not quiet): print_info(blue(" Keyword: ")+bold("\t"+file)) print_info(blue(" Found: ")+bold("\t"+str(len(result)))+red(" entries")) @@ -466,10 +479,12 @@ def searchPackage(packages, idreturn = False): dbconn = openRepositoryDatabase(repo) dataInfo = [] # when idreturn is True for package in packages: - result = dbconn.searchPackages(package) + slot = dep_getslot(package) + package = remove_slot(package) + result = dbconn.searchPackages(package, slot = slot) if (not result): # look for provide - provide = dbconn.searchProvide(package) + provide = dbconn.searchProvide(package, slot = slot) if (provide): result = [[provide[0],provide[1]]] @@ -484,14 +499,6 @@ def searchPackage(packages, idreturn = False): idpackage = pkg[1] atom = pkg[0] branch = dbconn.retrieveBranch(idpackage) - # does the package exist in the selected branch? - if etpConst['branch'] != branch: - # get branch name position in branches - myBranchIndex = etpConst['branches'].index(etpConst['branch']) - foundBranchIndex = etpConst['branches'].index(branch) - if foundBranchIndex > myBranchIndex: - # package found in branch more unstable than the selected one, for us, it does not exist - continue if (idreturn): dataInfo.append([idpackage,repo]) else: diff --git a/client/triggerTools.py b/client/triggerTools.py index deef6aa1c..7ed2af2b2 100644 --- a/client/triggerTools.py +++ b/client/triggerTools.py @@ -64,6 +64,9 @@ def postinstall(pkgdata): if pkgdata['category']+"/"+pkgdata['name'] == "dev-lang/python": functions.add("pythoninst") + if pkgdata['category']+"/"+pkgdata['name'] == "dev-db/sqlite": + functions.add('sqliteinst') + # kde package ? if "kde" in pkgdata['eclasses']: functions.add("kbuildsycoca") @@ -79,6 +82,9 @@ def postinstall(pkgdata): functions.add('gconfinstallschemas') functions.add('gconfreload') + if pkgdata['name'] == "pygtk": + functions.add('pygtksetup') + # prepare content mycnt = set(pkgdata['content']) @@ -127,12 +133,26 @@ def postremove(pkgdata): # opengl configuration if pkgdata['category'] == "x11-drivers": - functions.add("openglsetup") + functions.add("openglsetup_xorg") # kde package ? if "kde" in pkgdata['eclasses']: functions.add("kbuildsycoca") + if pkgdata['name'] == "pygtk": + functions.add('pygtkremove') + + if pkgdata['category']+"/"+pkgdata['name'] == "dev-db/sqlite": + functions.add('sqliteinst') + + # python configuration + if pkgdata['category']+"/"+pkgdata['name'] == "dev-lang/python": + functions.add("pythoninst") + + # fonts configuration + if pkgdata['category'] == "media-fonts": + functions.add("fontconfig") + # prepare content mycnt = set(pkgdata['removecontent']) @@ -243,13 +263,18 @@ def kernelmod(pkgdata): update_moduledb(item) print_info(" "+brown("[POST] Running depmod...")) kos = [x for x in pkgdata['content'] if x.startswith("/lib/modules") and x.endswith(".ko")] - run_depmod(kos, pkgdata['versiontag']) + run_depmod() def pythoninst(pkgdata): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring Python...") print_info(" "+brown("[POST] Configuring Python...")) python_update_symlink() +def sqliteinst(pkgdata): + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Configuring SQLite...") + print_info(" "+brown("[POST] Configuring SQLite...")) + sqlite_update_symlink() + def initdisable(pkgdata): mycnt = set(pkgdata['removecontent']) for file in mycnt: @@ -282,6 +307,16 @@ def openglsetup(pkgdata): equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Eselect NOT found, cannot run OpenGL trigger") print_info(" "+brown("[POST] Eselect NOT found, cannot run OpenGL trigger")) +def openglsetup_xorg(pkgdata): + eselect = os.system("eselect opengl &> /dev/null") + if eselect == 0: + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Reconfiguring OpenGL to fallback xorg-x11 ...") + print_info(" "+brown("[POST] Reconfiguring OpenGL...")) + os.system("eselect opengl set xorg-x11") + else: + equoLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"[POST] Eselect NOT found, cannot run OpenGL trigger") + print_info(" "+brown("[POST] Eselect NOT found, cannot run OpenGL trigger")) + # FIXME: this only supports grub (no lilo support) def addbootablekernel(pkgdata): kernels = [x for x in pkgdata['content'] if x.startswith("/boot/kernel-")] @@ -349,6 +384,7 @@ def gconfinstallschemas(pkgdata): gtest = os.system("which gconftool-2 &> /dev/null") if gtest == 0: schemas = [x for x in pkgdata['content'] if x.startswith("/etc/gconf/schemas") and x.endswith(".schemas")] + print_info(" "+brown("[POST] Installing GConf2 schemas...")) for schema in schemas: os.system(""" unset GCONF_DISABLE_MAKEFILE_SCHEMA_INSTALL @@ -356,6 +392,18 @@ def gconfinstallschemas(pkgdata): gconftool-2 --makefile-install-rule """+schema+""" 1>/dev/null """) +def pygtksetup(pkgdata): + python_sym_files = [x for x in pkgdata['content'] if x.startswith("/usr/lib/python") and (x.endswith("pygtk.py-2.0") or x.endswith("pygtk.pth-2.0"))] + for file in python_sym_files: + if os.path.isfile(file): + os.symlink(file,file[:-4]) + +def pygtkremove(pkgdata): + python_sym_files = [x for x in pkgdata['content'] if x.startswith("/usr/lib/python") and (x.endswith("pygtk.py-2.0") or x.endswith("pygtk.pth-2.0"))] + for file in python_sym_files: + if os.path.isfile(file[:-4]): + os.remove(file[:-4]) + ######################################################## #### ## Internal functions @@ -479,10 +527,9 @@ def update_moduledb(item): @description: insert kernel object into kernel modules db @output: returns int() as exit status ''' -def run_depmod(files, kv): +def run_depmod(): if os.access('/sbin/depmod',os.X_OK): - for file in files: - rc = os.system('/sbin/depmod -a -v '+kv+' '+file+' &> /dev/null') + os.system('/sbin/depmod -a &> /dev/null') return 0 ''' @@ -493,11 +540,23 @@ def python_update_symlink(): bins = [x for x in os.listdir("/usr/bin") if x.startswith("python2.")] versions = [x[6:] for x in bins] versions.sort() - latest = versions[len(versions)-1] + latest = versions[-1] os.system('ln -sf /usr/bin/python'+str(latest)+' /usr/bin/python') os.system('ln -sf /usr/bin/python'+str(latest)+' /usr/bin/python2') return 0 +''' + @description: update /usr/bin/lemon symlink + @output: returns int() as exit status +''' +def sqlite_update_symlink(): + bins = [x for x in os.listdir("/usr/bin") if x.startswith("lemon-")] + versions = [x[6:] for x in bins] + versions.sort() + latest = versions[-1] + os.system('ln -sf /usr/bin/lemon-'+str(latest)+' /usr/bin/lemon') + return 0 + ''' @description: shuts down selected init script, and remove from runlevel @output: returns int() as exit status @@ -521,6 +580,12 @@ def configure_boot_grub(kernel,initramfs): grub = open("/boot/grub/grub.conf","aw") # get boot dev boot_dev = get_grub_boot_dev() + # test if entry has been already added + grubtest = open("/boot/grub/grub.conf","r") + content = grubtest.readlines() + if "title="+etpConst['systemname']+" ("+os.path.basename(kernel)+")\n" in content: + grubtest.close() + return else: # create boot_dev = "(hd0,0)" diff --git a/conf/repositories.conf b/conf/repositories.conf index 5af33f9bf..8cf7799c4 100644 --- a/conf/repositories.conf +++ b/conf/repositories.conf @@ -15,13 +15,13 @@ repository|sabayonlinux.org|Sabayon Linux Official Repository|http://svn.sabayon # # syntax for branch: # -# branch: in Sabayon Linux there are two trees, stable and unstable. Stable packages are the ones of the latest stable release. -# branch|name of the branch (no spaces!) +# branch: in Sabayon Linux each release has its branch that is the version name (eg. 3.4, 3.5) +# branch|actual branch string (no spaces!) # # example: -# branch|stable +# branch|3.5 # or: -# branch|unstable +# branch|3.6 # # Branch default setting branch|unstable diff --git a/libraries/clientConstants.py b/libraries/clientConstants.py index 628e96dd9..440eb4bb9 100644 --- a/libraries/clientConstants.py +++ b/libraries/clientConstants.py @@ -35,49 +35,6 @@ etpInstallTriggers = {} ### structure same as above etpRemovalTriggers = {} - -# Client packages/database repositories -# used by equo -etpRepositories = {} -etpRepositoriesOrder = [] -if os.path.isfile(etpConst['repositoriesconf']): - f = open(etpConst['repositoriesconf'],"r") - repositoriesconf = f.readlines() - f.close() - - for line in repositoriesconf: - line = line.strip() - # populate etpRepositories - if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5): - reponame = line.split("|")[1] - repodesc = line.split("|")[2] - repopackages = line.split("|")[3] - repodatabase = line.split("|")[4] - if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")): - etpRepositories[reponame] = {} - etpRepositoriesOrder.append(reponame) - etpRepositories[reponame]['description'] = repodesc - etpRepositories[reponame]['packages'] = [] - for x in repopackages.split(): - etpRepositories[reponame]['packages'].append(x) - etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['currentarch'] - etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['currentarch'] - # initialize CONFIG_PROTECT - will be filled the first time the db will be opened - etpRepositories[reponame]['configprotect'] = set() - etpRepositories[reponame]['configprotectmask'] = set() - elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): - branch = line.split("|")[1] - etpConst['branch'] = branch - if branch not in etpConst['branches']: - etpConst['branches'].append(branch) - if not os.path.isdir(etpConst['packagesbindir']+"/"+branch): - if os.getuid() == 0: - os.makedirs(etpConst['packagesbindir']+"/"+branch) - else: - print "ERROR: please run equo as root at least once or create: "+str(etpConst['packagesbindir']+"/"+branch) - sys.exit(49) - - # equo section if (not os.path.isfile(etpConst['equoconf'])): print "ERROR: "+etpConst['equoconf']+" does not exist" diff --git a/libraries/databaseTools.py b/libraries/databaseTools.py index b678bcd64..4b9da21a9 100644 --- a/libraries/databaseTools.py +++ b/libraries/databaseTools.py @@ -271,146 +271,101 @@ def database(options): connection.close() print_info(green(" * ")+red("Entropy database file ")+bold(mypath[0])+red(" successfully initialized.")) - elif (options[0] == "stabilize") or (options[0] == "unstabilize"): # FIXME: adapt to the new branches structure + elif (options[0] == "switchbranch"): + + if (len(options) < 2): + print_error(yellow(" * ")+red("Not enough parameters")) + sys.exit(302) - if options[0] == "stabilize": - stable = True - else: - stable = False - - if (stable): - print_info(green(" * ")+red("Collecting packages that would be marked stable ..."), back = True) - else: - print_info(green(" * ")+red("Collecting packages that would be marked unstable ..."), back = True) - - myatoms = options[1:] - if len(myatoms) == 0: + switchbranch = options[1] + print_info(green(" * ")+red("Collecting packages that would be marked '"+switchbranch+"' ..."), back = True) + + myatoms = options[2:] + if not myatoms: print_error(yellow(" * ")+red("Not enough parameters")) sys.exit(303) + + dbconn = etpDatabase(readOnly = False, noUpload = True) # is world? if myatoms[0] == "world": - # open db in read only - dbconn = etpDatabase(readOnly = True) - if (stable): - pkglist = dbconn.listUnstablePackages() - else: - pkglist = dbconn.listStablePackages() - # This is the list of all the packages available in Entropy - dbconn.closeDB() + pkglist = set(dbconn.listAllIdpackages()) else: - pkglist = [] + pkglist = set() for atom in myatoms: # validate atom - dbconn = etpDatabase(readOnly = True) - if (stable): - pkg = dbconn.searchPackagesInBranch(atom,"unstable") + match = dbconn.atomMatch(atom) + if match == -1: + print_warning(yellow(" * ")+red("Cannot match: ")+bold(atom)) else: - pkg = dbconn.searchPackagesInBranch(atom,"stable") - for x in pkg: - pkglist.append(x[0]) + pkglist.add(match[0]) - # filter dups - pkglist = list(set(pkglist)) # check if atoms were found - if len(pkglist) == 0: + if not pkglist: print print_error(yellow(" * ")+red("No packages found.")) sys.exit(303) # show what would be done - if (stable): - print_info(green(" * ")+red("These are the packages that would be marked stable:")) - else: - print_info(green(" * ")+red("These are the packages that would be marked unstable:")) + print_info(green(" * ")+red("These are the packages that would be marked '"+switchbranch+"':")) for pkg in pkglist: - print_info(red("\t (*) ")+bold(pkg)) - - # ask to continue + atom = dbconn.retrieveAtom(pkg) + print_info(red("\t (*) ")+bold(atom)) + rc = entropyTools.askquestion(" Would you like to continue ?") if rc == "No": sys.exit(0) - # now mark them as stable - print_info(green(" * ")+red("Marking selected packages ...")) - - # open db - dbconn = etpDatabase(readOnly = False, noUpload = True) + # sync packages + import activatorTools + activatorTools.packages(["sync","--ask"]) + + print_info(green(" * ")+red("Switching selected packages ...")) import re + for pkg in pkglist: - print_info(green(" * ")+red("Marking package: ")+bold(pkg)+red(" ..."), back = True) - rc, action = dbconn.stabilizePackage(pkg,stable) - # @rc: True if updated, False if not - # @action: action taken: "stable" for stabilized package, "unstable" for unstabilized package - if (rc): - - print_info(green(" * ")+red("Package: ")+bold(pkg)+red(" needs to be marked ")+bold(action), back = True) - - # change download database parameter name - download = dbconn.retrievePackageVar(pkg, "download", branch = action) - # change action with the opposite: - if action == "stable": - # move to unstable - oppositeAction = "unstable" - else: - oppositeAction = "stable" - - oldpkgfilename = os.path.basename(download) - download = re.subn("-"+oppositeAction,"-"+action, download) - - if download[1]: # if the name has been converted - - newpkgfilename = os.path.basename(download[0]) - - # change download parameter in the database entry - dbconn.writePackageParameter(pkg, "download", download[0], action) - - print_info(green(" * ")+yellow("Updating local package name")) - - # change filename locally - if os.path.isfile(etpConst['packagesbindir']+"/"+oldpkgfilename): - os.rename(etpConst['packagesbindir']+"/"+oldpkgfilename,etpConst['packagesbindir']+"/"+newpkgfilename) - - print_info(green(" * ")+yellow("Updating local package checksum")) - - # update md5 - if os.path.isfile(etpConst['packagesbindir']+"/"+oldpkgfilename+etpConst['packageshashfileext']): - - f = open(etpConst['packagesbindir']+"/"+oldpkgfilename+etpConst['packageshashfileext']) - oldMd5 = f.readline().strip() - f.close() - newMd5 = re.subn(oldpkgfilename, newpkgfilename, oldMd5) - if newMd5[1]: - f = open(etpConst['packagesbindir']+"/"+newpkgfilename+etpConst['packageshashfileext'],"w") - f.write(newMd5[0]+"\n") - f.flush() - f.close() - # remove old - os.remove(etpConst['packagesbindir']+"/"+oldpkgfilename+etpConst['packageshashfileext']) - - else: # old md5 does not exist - - entropyTools.createHashFile(etpConst['packagesbindir']+"/"+newpkgfilename) - - - print_info(green(" * ")+yellow("Updating remote package information")) - - # change filename remotely - ftp = mirrorTools.handlerFTP(uri) - ftp.setCWD(etpConst['binaryurirelativepath']) - if (ftp.isFileAvailable(etpConst['packagesbindir']+"/"+oldpkgfilename)): - # rename tbz2 - ftp.renameFile(oldpkgfilename,newpkgfilename) - # remove old .md5 - ftp.deleteFile(oldpkgfilename+etpConst['packageshashfileext']) - # upload new .md5 if found - if os.path.isfile(etpConst['packagesbindir']+"/"+newpkgfilename+etpConst['packageshashfileext']): - ftp.uploadFile(etpConst['packagesbindir']+"/"+newpkgfilename+etpConst['packageshashfileext'],ascii = True) - - - dbconn.commitChanges() - print_info(green(" * ")+red("All the selected packages have been marked as requested. Have fun.")) + atom = dbconn.retrieveAtom(pkg) + currentbranch = dbconn.retrieveBranch(pkg) + currentdownload = dbconn.retrieveDownloadURL(pkg) + + if currentbranch == switchbranch: + print_warning(green(" * ")+red("Ignoring ")+bold(atom)+red(" since it is already in the chosen branch")) + continue + + print_info(green(" * ")+darkred(atom+": ")+red("Configuring package information..."), back = True) + # change branch and download URL + dbconn.switchBranch(pkg,switchbranch) + + # rename locally + filename = os.path.basename(dbconn.retrieveDownloadURL(pkg)) + topath = etpConst['packagesbindir']+"/"+switchbranch + if not os.path.isdir(topath): + os.makedirs(topath) + print_info(green(" * ")+darkred(atom+": ")+red("Moving file locally..."), back = True) + #print etpConst['entropyworkdir']+"/"+currentdownload+" --> "+topath+"/"+newdownload + os.rename(etpConst['entropyworkdir']+"/"+currentdownload,topath+"/"+filename) + # md5 + os.rename(etpConst['entropyworkdir']+"/"+currentdownload+etpConst['packageshashfileext'],topath+"/"+filename+etpConst['packageshashfileext']) + + # XXX: we can barely ignore branch info injected into .tbz2 since they'll be ignored too + + # rename remotely + print_info(green(" * ")+darkred(atom+": ")+red("Moving file remotely..."), back = True) + # change filename remotely + ftp = mirrorTools.handlerFTP(uri) + ftp.setCWD(etpConst['binaryurirelativepath']) + # create directory if it doesn't exist + if (not ftp.isFileAvailable(switchbranch)): + ftp.mkdir(switchbranch) + # rename tbz2 + ftp.renameFile(currentbranch+"/"+filename,switchbranch+"/"+filename) + # rename md5 + ftp.renameFile(currentbranch+"/"+filename+etpConst['packageshashfileext'],switchbranch+"/"+filename+etpConst['packageshashfileext']) + ftp.closeConnection() + dbconn.closeDB() + print_info(green(" * ")+red("All the selected packages have been marked as requested. Remember to run activator.")) + elif (options[0] == "remove"): @@ -418,16 +373,10 @@ def database(options): myopts = options[1:] _myopts = [] - branch = '' + branch = None for opt in myopts: if (opt.startswith("--branch=")) and (len(opt.split("=")) == 2): - - try: - branch = opt.split("=")[1] - idx = etpConst['branches'].index(branch) - etpConst['branch'] = branch - except: - pass + branch = opt.split("=")[1] else: _myopts.append(opt) myopts = _myopts @@ -436,18 +385,19 @@ def database(options): print_error(yellow(" * ")+red("Not enough parameters")) sys.exit(303) - pkglist = [] - dbconn = etpDatabase(readOnly = True) + pkglist = set() + dbconn = etpDatabase(readOnly = False, noUpload = True) for atom in myopts: - pkg = dbconn.atomMatch(atom) + if (branch): + pkg = dbconn.atomMatch(atom, matchBranches = (branch,)) + else: + pkg = dbconn.atomMatch(atom) if pkg[0] != -1: - pkglist.append(pkg[0]) + pkglist.add(pkg[0]) - # filter dups - pkglist = list(set(pkglist)) # check if atoms were found - if len(pkglist) == 0: + if not pkglist: print dbconn.closeDB() print_error(yellow(" * ")+red("No packages found.")) @@ -460,8 +410,6 @@ def database(options): branch = dbconn.retrieveBranch(pkg) print_info(red("\t (*) ")+bold(pkgatom)+blue(" [")+red(branch)+blue("]")) - dbconn.closeDB() - # ask to continue rc = entropyTools.askquestion(" Would you like to continue ?") if rc == "No": @@ -471,31 +419,14 @@ def database(options): print_info(green(" * ")+red("Removing selected packages ...")) # open db - dbconn = etpDatabase(readOnly = False, noUpload = True) for pkg in pkglist: pkgatom = dbconn.retrieveAtom(pkg) print_info(green(" * ")+red("Removing package: ")+bold(pkgatom)+red(" ..."), back = True) dbconn.removePackage(pkg) - dbconn.commitChanges() print_info(green(" * ")+red("All the selected packages have been removed as requested. To remove online binary packages, just run Activator.")) dbconn.closeDB() # used by reagent - elif (options[0] == "statistics"): - print_info(green(" [LOCAL DB STATISTIC]\t\t")+red("Information")) - # fetch total packages - dbconn = etpDatabase(readOnly = True) - totalpkgs = len(dbconn.listAllPackages()) - totalstablepkgs = len(dbconn.listStablePackages()) - totalunstablepkgs = len(dbconn.listUnstablePackages()) - print_info(green(" Total Installed Packages\t\t")+red(str(totalpkgs))) - print_info(green(" Total Stable Packages\t\t")+red(str(totalstablepkgs))) - print_info(green(" Total Unstable Packages\t\t")+red(str(totalunstablepkgs))) - activatorTools.syncRemoteDatabases(justStats = True) - dbconn.closeDB() - - # used by reagent - # FIXME: complete this with some automated magic elif (options[0] == "md5check"): print_info(green(" * ")+red("Integrity verification of the selected packages:")) @@ -1849,6 +1780,18 @@ class etpDatabase: data['disksize'] = self.retrieveOnDiskSize(idpackage) return data + def fetchall2set(self, item): + return set(sum(tuple(item), ())) + + def fetchall2list(self, item): + return list(sum(tuple(item), ())) + + def fetchone2list(self, item): + return list(item) + + def fetchone2set(self, item): + return set(item) + def retrieveAtom(self, idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveAtom: retrieving Atom for package ID "+str(idpackage)) @@ -1862,8 +1805,10 @@ class etpDatabase: else: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} - self.cursor.execute('SELECT "atom" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - atom = self.cursor.fetchone()[0] + self.cursor.execute('SELECT "atom" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') + atom = self.cursor.fetchone() + if atom: + atom = atom[0] ''' caching ''' if (self.xcache): @@ -1884,7 +1829,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "branch" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - br = self.cursor.fetchone()[0] + br = self.cursor.fetchone() + if br: + br = br[0] ''' caching ''' if (self.xcache): @@ -1905,7 +1852,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "download" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') - download = self.cursor.fetchone()[0] + download = self.cursor.fetchone() + if download: + download = download[0] ''' caching ''' if (self.xcache): @@ -1926,7 +1875,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "description" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') - description = self.cursor.fetchone()[0] + description = self.cursor.fetchone() + if description: + description = description[0] ''' caching ''' if (self.xcache): @@ -1947,7 +1898,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "homepage" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') - home = self.cursor.fetchone()[0] + home = self.cursor.fetchone() + if home: + home = home[0] ''' caching ''' if (self.xcache): @@ -1970,7 +1923,9 @@ class etpDatabase: counter = -1 try: self.cursor.execute('SELECT "counter" FROM counters WHERE idpackage = "'+str(idpackage)+'"') - counter = self.cursor.fetchone()[0] + counter = self.cursor.fetchone() + if counter: + counter = counter[0] except: pass @@ -2073,7 +2028,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "digest" FROM extrainfo WHERE idpackage = "'+str(idpackage)+'"') - digest = self.cursor.fetchone()[0] + digest = self.cursor.fetchone() + if digest: + digest = digest[0] ''' caching ''' if (self.xcache): @@ -2094,7 +2051,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "name" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - name = self.cursor.fetchone()[0] + name = self.cursor.fetchone() + if name: + name = name[0] ''' caching ''' if (self.xcache): @@ -2115,7 +2074,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "version" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - ver = self.cursor.fetchone()[0] + ver = self.cursor.fetchone() + if ver: + ver = ver[0] ''' caching ''' if (self.xcache): @@ -2136,7 +2097,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "revision" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - rev = self.cursor.fetchone()[0] + rev = self.cursor.fetchone() + if rev: + rev = rev[0] ''' caching ''' if (self.xcache): @@ -2205,7 +2168,8 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT flagname FROM useflags,useflagsreference WHERE useflags.idpackage = "'+str(idpackage)+'" and useflags.idflag = useflagsreference.idflag') - flags = self.cursor.fetchall() + flags = self.fetchall2set(self.cursor.fetchall()) + ''' caching ''' if (self.xcache): @@ -2226,7 +2190,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT classname FROM eclasses,eclassesreference WHERE eclasses.idpackage = "'+str(idpackage)+'" and eclasses.idclass = eclassesreference.idclass') - classes = self.cursor.fetchall() + classes = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2246,8 +2210,8 @@ class etpDatabase: else: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} - self.cursor.execute('SELECT library FROM needed,neededreference WHERE ineeded.dpackage = "'+str(idpackage)+'" and needed.idneeded = neededreference.idneeded') - needed = self.cursor.fetchall() + self.cursor.execute('SELECT library FROM needed,neededreference WHERE needed.idpackage = "'+str(idpackage)+'" and needed.idneeded = neededreference.idneeded') + needed = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2268,7 +2232,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "conflict" FROM conflicts WHERE idpackage = "'+str(idpackage)+'"') - confl = self.cursor.fetchall() + confl = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2289,7 +2253,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "atom" FROM provide WHERE idpackage = "'+str(idpackage)+'"') - provide = self.cursor.fetchall() + provide = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2310,7 +2274,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT dependenciesreference.dependency FROM dependencies,dependenciesreference WHERE idpackage = "'+str(idpackage)+'" and dependencies.iddependency = dependenciesreference.iddependency') - deps = self.cursor.fetchall() + deps = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2331,7 +2295,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT iddependency FROM dependencies WHERE idpackage = "'+str(idpackage)+'"') - iddeps = self.cursor.fetchall() + iddeps = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2352,7 +2316,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT keywordname FROM binkeywords,keywordsreference WHERE binkeywords.idpackage = "'+str(idpackage)+'" and binkeywords.idkeyword = keywordsreference.idkeyword') - kw = self.cursor.fetchall() + kw = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2373,7 +2337,7 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"retrieveKeywords: retrieving Keywords for package ID "+str(idpackage)) self.cursor.execute('SELECT keywordname FROM keywords,keywordsreference WHERE keywords.idpackage = "'+str(idpackage)+'" and keywords.idkeyword = keywordsreference.idkeyword') - kw = self.cursor.fetchall() + kw = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' @@ -2399,9 +2363,11 @@ class etpDatabase: if not protect: protect = '' else: - ''' caching ''' - if (self.xcache): - dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)]['retrieveProtect'] = protect + protect = protect[0] + + ''' caching ''' + if (self.xcache): + dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)]['retrieveProtect'] = protect return protect def retrieveProtectMask(self, idpackage): @@ -2422,9 +2388,11 @@ class etpDatabase: if not protect: protect = '' else: - ''' caching ''' - if (self.xcache): - dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)]['retrieveProtectMask'] = protect + protect = protect[0] + + ''' caching ''' + if (self.xcache): + dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)]['retrieveProtectMask'] = protect return protect def retrieveSources(self, idpackage): @@ -2440,8 +2408,8 @@ class etpDatabase: else: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} - self.cursor.execute('SELECT sourcesreference.source FROM sources,sourcesreference WHERE idpackage = "'+str(idpackage)+'" sources.idsource = sourcesreference.idsource') - sources = self.corsor.fetchall() + self.cursor.execute('SELECT sourcesreference.source FROM sources,sourcesreference WHERE idpackage = "'+str(idpackage)+'" and sources.idsource = sourcesreference.idsource') + sources = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2462,7 +2430,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "file" FROM content WHERE idpackage = "'+str(idpackage)+'"') - fl = self.cursor.fetchall() + fl = self.fetchall2set(self.cursor.fetchall()) ''' caching ''' if (self.xcache): @@ -2483,7 +2451,9 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT "slot" FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - ver = self.cursor.fetchone()[0] + ver = self.cursor.fetchone() + if ver: + ver = ver[0] ''' caching ''' if (self.xcache): @@ -2553,7 +2523,7 @@ class etpDatabase: else: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} - self.cursor.execute('SELECT license FROM baseinfo,licenses WHERE baseinfo.idpackage = "'+str(idpackage)+'" baseinfo.idlicense = licenses.idlicense') + self.cursor.execute('SELECT license FROM baseinfo,licenses WHERE baseinfo.idpackage = "'+str(idpackage)+'" and baseinfo.idlicense = licenses.idlicense') licname = self.cursor.fetchone()[0] ''' caching ''' @@ -2621,7 +2591,7 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isPackageAvailable: called.") pkgkey = entropyTools.removePackageOperators(pkgkey) self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = "'+pkgkey+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isPackageAvailable: "+pkgkey+" not available.") return False @@ -2631,7 +2601,7 @@ class etpDatabase: def isIDPackageAvailable(self,idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isIDPackageAvailable: called.") self.cursor.execute('SELECT idpackage FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isIDPackageAvailable: "+str(idpackage)+" not available.") return False @@ -2643,7 +2613,7 @@ class etpDatabase: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSpecificPackageAvailable: called.") pkgkey = entropyTools.removePackageOperators(pkgkey) self.cursor.execute('SELECT idpackage FROM baseinfo WHERE atom = "'+pkgkey+'" AND branch = "'+branch+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isSpecificPackageAvailable: "+pkgkey+" | branch: "+branch+" -> not found.") return False @@ -2653,82 +2623,82 @@ class etpDatabase: def isCategoryAvailable(self,category): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isCategoryAvailable: called.") self.cursor.execute('SELECT idcategory FROM categories WHERE category = "'+category+'"') - result = self.cursor.fetchall() - if not result???: + result = self.cursor.fetchone() + if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isCategoryAvailable: "+category+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isCategoryAvailable: "+category+" available.") - return result + return result[0] def isProtectAvailable(self,protect): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isProtectAvailable: called.") self.cursor.execute('SELECT idprotect FROM configprotectreference WHERE protect = "'+protect+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isProtectAvailable: "+protect+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isProtectAvailable: "+protect+" available.") - return result + return result[0] def isSourceAvailable(self,source): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSourceAvailable: called.") self.cursor.execute('SELECT idsource FROM sourcesreference WHERE source = "'+source+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isSourceAvailable: "+source+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSourceAvailable: "+source+" available.") - return result + return result[0] def isDependencyAvailable(self,dependency): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isDependencyAvailable: called.") self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = "'+dependency+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isDependencyAvailable: "+dependency+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isDependencyAvailable: "+dependency+" available.") - return result + return result[0] def isKeywordAvailable(self,keyword): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isKeywordAvailable: called.") self.cursor.execute('SELECT idkeyword FROM keywordsreference WHERE keywordname = "'+keyword+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isKeywordAvailable: "+keyword+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isKeywordAvailable: "+keyword+" available.") - return result + return result[0] def isUseflagAvailable(self,useflag): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isUseflagAvailable: called.") self.cursor.execute('SELECT idflag FROM useflagsreference WHERE flagname = "'+useflag+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isUseflagAvailable: "+useflag+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isUseflagAvailable: "+useflag+" available.") - return result + return result[0] def isEclassAvailable(self,eclass): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isEclassAvailable: called.") self.cursor.execute('SELECT idclass FROM eclassesreference WHERE classname = "'+eclass+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isEclassAvailable: "+eclass+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isEclassAvailable: "+eclass+" available.") - return result + return result[0] def isNeededAvailable(self,needed): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isNeededAvailable: called.") self.cursor.execute('SELECT idneeded FROM neededreference WHERE library = "'+needed+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isNeededAvailable: "+needed+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isNeededAvailable: "+needed+" available.") - return result + return result[0] def isCounterAvailable(self,counter): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isCounterAvailable: called.") @@ -2745,12 +2715,12 @@ class etpDatabase: def isLicenseAvailable(self,license): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isLicenseAvailable: called.") self.cursor.execute('SELECT idlicense FROM licenses WHERE license = "'+license+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"isLicenseAvailable: "+license+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isLicenseAvailable: "+license+" available.") - return result + return result[0] def isSystemPackage(self,idpackage): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSystemPackage: called.") @@ -2766,7 +2736,7 @@ class etpDatabase: dbCacheStore[etpCache['dbInfo']+self.dbname][int(idpackage)] = {} self.cursor.execute('SELECT idpackage FROM systempackages WHERE idpackage = "'+str(idpackage)+'"') - result = self.cursor.fetchone()[0] + result = self.cursor.fetchone() rslt = False if result: dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"isSystemPackage: package is in system.") @@ -2782,79 +2752,91 @@ class etpDatabase: def areCompileFlagsAvailable(self,chost,cflags,cxxflags): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"areCompileFlagsAvailable: called.") self.cursor.execute('SELECT idflags FROM flags WHERE chost = "'+chost+'" AND cflags = "'+cflags+'" AND cxxflags = "'+cxxflags+'"') - result = self.cursor.fetchall() + result = self.cursor.fetchone() if not result: dbLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"areCompileFlagsAvailable: flags tuple "+chost+"|"+cflags+"|"+cxxflags+" not available.") - return result + return -1 dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"areCompileFlagsAvailable: flags tuple "+chost+"|"+cflags+"|"+cxxflags+" available.") - return result + return result[0] - def searchBelongs(self, file, like = False): + def searchBelongs(self, file, like = False, branch = None): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchBelongs: called for "+file) + + branchstring = '' + if branch: + branchstring = ' and baseinfo.branch = "'+branch+'"' + if (like): - self.cursor.execute('SELECT idpackage FROM content WHERE file LIKE "'+file+'"') + self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file LIKE "'+file+'" and content.idpackage = baseinfo.idpackage '+branchstring) else: - self.cursor.execute('SELECT idpackage FROM content WHERE file = "'+file+'"') - return self.cursor.fetchall() + self.cursor.execute('SELECT content.idpackage FROM content,baseinfo WHERE file = "'+file+'" and content.idpackage = baseinfo.idpackage '+branchstring) + + return self.fetchall2set(self.cursor.fetchall()) ''' search packages that need the specified library (in neededreference table) specified by keyword ''' def searchNeeded(self, keyword): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchNeeded: called for "+keyword) - idpackages = set() self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference WHERE library = "'+keyword+'" and needed.idneeded = neededreference.idneeded') - for row in self.cursor: - idpackages.add(row[0]) - return idpackages + return self.fetchall2set(self.cursor.fetchall()) ''' same as above but with branch support ''' def searchNeededInBranch(self, keyword, branch): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchNeeded: called for "+keyword+" and branch: "+branch) - idpackages = set() self.cursor.execute('SELECT needed.idpackage FROM needed,neededreference,baseinfo WHERE library = "'+keyword+'" and needed.idneeded = neededreference.idneeded and baseinfo.branch = "'+branch+'"') - for row in self.cursor: - idpackages.add(row[0]) - return idpackages + return self.fetchall2set(self.cursor.fetchall()) ''' search dependency string inside dependenciesreference table and retrieve iddependency ''' def searchDependency(self, dep): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchDependency: called for "+dep) self.cursor.execute('SELECT iddependency FROM dependenciesreference WHERE dependency = "'+dep+'"') - iddep = self.cursor.fetchall() + iddep = self.cursor.fetchone() if not iddep: iddep = -1 - return iddep + return iddep[0] ''' search iddependency inside dependencies table and retrieve idpackages ''' def searchIdpackageFromIddependency(self, iddep): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchIdpackageFromIddependency: called for "+str(iddep)) self.cursor.execute('SELECT idpackage FROM dependencies WHERE iddependency = "'+str(iddep)+'"') - return self.cursor.fetchall() + return self.fetchall2set(self.cursor.fetchall()) - def searchPackages(self, keyword, sensitive = False): + def searchPackages(self, keyword, sensitive = False, slot = None): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchPackages: called for "+keyword) - result = [] + slotstring = '' + if slot: + slotstring = ' and slot = "'+slot+'"' if (sensitive): - self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE "%'+keyword+'%"') + self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE atom LIKE "%'+keyword+'%"'+slotstring) else: - self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"') + self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE LOWER(atom) LIKE "%'+string.lower(keyword)+'%"'+slotstring) return self.cursor.fetchall() - def searchProvide(self, keyword): + def searchProvide(self, keyword, slot = None): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchProvide: called for "+keyword) - idpackage = [] + slotstring = '' + if slot: + slotstring = ' and slot = "'+slot+'"' + + idpackage = '' self.cursor.execute('SELECT idpackage FROM provide WHERE atom = "'+keyword+'"') - idpackage = self.cursor.fetchone()[0] - self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') - result = self.cursor.fetchone()[0] + idpackage = self.cursor.fetchone() + if not idpackage: + return '' + + self.cursor.execute('SELECT atom,idpackage FROM baseinfo WHERE idpackage = "'+str(idpackage[0])+'"'+slotstring) + result = self.cursor.fetchone() + if result: + return result[0] - return result + return '' def searchProvideInBranch(self, keyword, branch): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchProvideInBranch: called for "+keyword+" and branch: "+branch) self.cursor.execute('SELECT idpackage FROM provide WHERE atom = "'+keyword+'"') - idpackages = self.cursor.fetchall() + results = set() + idpackages = self.fetchall2set(self.cursor.fetchall()) for idpackage in idpackages: self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo WHERE idpackage = "'+str(idpackage)+'"') for row in self.cursor: @@ -2863,7 +2845,7 @@ class etpDatabase: idpackage = data[1] pkgbranch = data[2] if (branch == pkgbranch): - results.append((atom,idpackage)) + results.add((atom,idpackage)) return results def searchPackagesInBranch(self, keyword, branch, sensitive = False): @@ -2975,7 +2957,7 @@ class etpDatabase: # this function search packages with the same pkgcat/pkgname # you must provide something like: media-sound/amarok # optionally, you can add version too. - def searchSimilarPackages(self, atom, branch = "unstable"): + def searchSimilarPackages(self, atom, branch = etpConst['branch']): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"searchSimilarPackages: called for "+atom+" | branch: "+branch) category = atom.split("/")[0] name = atom.split("/")[1] @@ -2990,33 +2972,36 @@ class etpDatabase: def listAllPackages(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllPackages: called.") self.cursor.execute('SELECT atom,idpackage,branch FROM baseinfo') - result = [] + result = set() for row in self.cursor: - result.append(row) + result.add(row) return result def listAllCounters(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllCounters: called.") self.cursor.execute('SELECT counter,idpackage FROM counters') - result = [] + result = set() for row in self.cursor: - result.append(row) + result.add(row) return result - def listAllIdpackages(self): + def listAllIdpackages(self, branch = None): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllIdpackages: called.") - self.cursor.execute('SELECT idpackage FROM baseinfo') - result = [] + branchstring = '' + if branch: + branchstring = ' where branch = "'+branch+'"' + self.cursor.execute('SELECT idpackage FROM baseinfo'+branchstring) + result = set() for row in self.cursor: - result.append(row[0]) + result.add(row[0]) return result def listAllDependencies(self): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllDependencies: called.") self.cursor.execute('SELECT * FROM dependenciesreference') - result = [] + result = set() for row in self.cursor: - result.append(row) + result.add(row) return result def listIdpackageDependencies(self, idpackage): @@ -3032,22 +3017,6 @@ class etpDatabase: result.append(row) return result - #FIXME: DEPRECATED - def listAllPackagesTbz2(self): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listAllPackagesTbz2: called.") - result = [] - pkglist = self.listAllPackages() - for pkg in pkglist: - idpackage = pkg[1] - url = self.retrieveDownloadURL(idpackage) - if url: - result.append(url) - # filter dups? - if (result): - result = list(set(result)) - result.sort() - return result - def listBranchPackagesTbz2(self, branch): dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"listBranchPackagesTbz2: called with "+str(branch)) result = [] @@ -3127,50 +3096,32 @@ class etpDatabase: dirs.sort() return dirs - # FIXME: get it working with the new branch layout - def stabilizePackage(self,atom,stable = True): + def switchBranch(self, idpackage, tobranch): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: called for "+atom+" | branch stable? -> "+str(stable)) + dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"switchBranch: called for ID "+str(idpackage)+" | branch -> "+str(tobranch)) + + mycat = self.retreveCategory(idpackage) + myname = self.retrieveName(idpackage) + myslot = self.retrieveSlot(idpackage) + mybranch = self.retrieveBranch(idpackage) + mydownload = self.retrieveDownloadURL(idpackage) + import re + out = re.subn('/'+mybranch+'/','/'+tobranch+'/',mydownload) + newdownload = out[0] + + # remove package with the same key+slot and tobranch if exists + match = self.atomMatch(mycat+"/"+myname, matchSlot = myslot, matchBranches = (tobranch,)) + if match[0] != -1: + self.removePackage(match[0]) + + # now switch selected idpackage to the new branch + self.cursor.execute('UPDATE baseinfo SET branch = "'+str(tobranch)+'" WHERE idpackage = "'+str(idpackage)+'"') + self.cursor.execute('UPDATE extrainfo SET download = "'+newdownload+'" WHERE idpackage = "'+str(idpackage)+'"') + self.commitChanges() + # clean cursor - NEEDED? + for row in self.cursor: + x = row - action = "unstable" - removeaction = "stable" - if (stable): - action = "stable" - removeaction = "unstable" - - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: add action: "+action+" | remove action: "+removeaction) - - if (self.isSpecificPackageAvailable(atom, removeaction)): - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: there's something old that needs to be removed.") - idpackage = self.getIDPackage(atom, branch = removeaction) - - pkgname = self.retrieveName(idpackage) - # get its pkgcat - category = self.retrieveCategory(idpackage) - # search packages with similar pkgcat/name marked as stable - slot = self.retrieveSlot(idpackage) - - # we need to get rid of them - results = self.searchStablePackages(category+"/"+pkgname) - - removelist = [] - for result in results: - myidpackage = result[1] - # have a look if the slot matches - #print result - myslot = self.retrieveSlot(myidpackage) - if (myslot == slot): - removelist.append(result[1]) - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: removelist: "+str(removelist)) - for pkg in removelist: - self.removePackage(pkg) - dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"stabilizePackage: updating "+atom+" setting branch: "+action) - - self.cursor.execute('UPDATE baseinfo SET branch = "'+action+'" WHERE idpackage = "'+idpackage+'"') - self.commitChanges() - - return True,action - return False,action ######################################################## #### @@ -3346,7 +3297,7 @@ class etpDatabase: @input multiMatch: bool, return all the available atoms @output: the package id, if found, otherwise -1 plus the status, 0 = ok, 1 = not found, 2 = need more info, 3 = cannot use direction without specifying version ''' - def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = []): + def atomMatch(self, atom, caseSensitive = True, matchSlot = None, multiMatch = False, matchBranches = ()): if (self.xcache): cached = dbCacheStore[etpCache['dbMatch']+self.dbname].get(atom) if cached: @@ -3371,12 +3322,7 @@ class etpDatabase: direction = atom[0:len(atom)-len(strippedAtom)] #print direction - #print strippedAtom - #print entropyTools.isspecific(strippedAtom) - #print direction - justname = entropyTools.isjustname(strippedAtom) - #print justname pkgversion = '' if (not justname): # strip tag @@ -3405,15 +3351,14 @@ class etpDatabase: #print dep_getkey(strippedAtom) if (matchBranches): - myBranchIndex = matchBranches + myBranchIndex = tuple(matchBranches) # force to tuple for security else: - myBranchIndex = [etpConst['branch']] + myBranchIndex = (etpConst['branch'],) # IDs found in the database that match our search foundIDs = [] for idx in myBranchIndex: # myBranchIndex is ordered by importance - #print "Searching into -> "+etpConst['branches'][idx] # search into the less stable, if found, break, otherwise continue results = self.searchPackagesInBranchByName(pkgname, idx, caseSensitive) diff --git a/libraries/entropyConstants.py b/libraries/entropyConstants.py index 01cff3f9b..1dbb8fc25 100644 --- a/libraries/entropyConstants.py +++ b/libraries/entropyConstants.py @@ -389,8 +389,8 @@ etpConst = { 'preinstallscript': "preinstall.sh", # used by the client to run some pre-install actions 'postinstallscript': "postinstall.sh", # used by the client to run some post-install actions - 'branches': ["stable","unstable"], # available branches, do not scramble! - 'branch': "unstable", # choosen branch + 'branches': ["3.5","2008"], # available branches, this only exists for the server part + 'branch': "3.5", # choosen branch 'gentoo-compat': False, # Gentoo compatibility (/var/db/pkg + Portage availability) 'filesystemdirs': ['/bin','/boot','/emul','/etc','/lib','/lib32','/lib64','/opt','/sbin','/usr','/var'], # directory of the filesystem 'filesystemdirsmask': [ @@ -562,6 +562,45 @@ else: import time time.sleep(5) +# Client packages/database repositories +etpRepositories = {} +etpRepositoriesOrder = [] +if os.path.isfile(etpConst['repositoriesconf']): + f = open(etpConst['repositoriesconf'],"r") + repositoriesconf = f.readlines() + f.close() + + for line in repositoriesconf: + line = line.strip() + # populate etpRepositories + if (line.find("repository|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 5): + reponame = line.split("|")[1] + repodesc = line.split("|")[2] + repopackages = line.split("|")[3] + repodatabase = line.split("|")[4] + if (repopackages.startswith("http://") or repopackages.startswith("ftp://")) and (repodatabase.startswith("http://") or repodatabase.startswith("ftp://")): + etpRepositories[reponame] = {} + etpRepositoriesOrder.append(reponame) + etpRepositories[reponame]['description'] = repodesc + etpRepositories[reponame]['packages'] = [] + for x in repopackages.split(): + etpRepositories[reponame]['packages'].append(x) + etpRepositories[reponame]['database'] = repodatabase+"/"+etpConst['currentarch'] + etpRepositories[reponame]['dbpath'] = etpConst['etpdatabaseclientdir']+"/"+reponame+"/"+etpConst['currentarch'] + # initialize CONFIG_PROTECT - will be filled the first time the db will be opened + etpRepositories[reponame]['configprotect'] = set() + etpRepositories[reponame]['configprotectmask'] = set() + elif (line.find("branch|") != -1) and (not line.startswith("#")) and (len(line.split("|")) == 2): + branch = line.split("|")[1] + etpConst['branch'] = branch + if branch not in etpConst['branches']: + etpConst['branches'].append(branch) + if not os.path.isdir(etpConst['packagesbindir']+"/"+branch): + if os.getuid() == 0: + os.makedirs(etpConst['packagesbindir']+"/"+branch) + else: + print "ERROR: please run equo as root at least once or create: "+str(etpConst['packagesbindir']+"/"+branch) + sys.exit(49) # database section if (not os.path.isfile(etpConst['databaseconf'])): diff --git a/libraries/mirrorTools.py b/libraries/mirrorTools.py index ab6f26774..b6a776e8a 100644 --- a/libraries/mirrorTools.py +++ b/libraries/mirrorTools.py @@ -39,7 +39,6 @@ mirrorLog = logTools.LogFile(level=etpConst['mirrorsloglevel'],filename = etpCon class handlerFTP: - # ftp://linuxsabayon:asdasd@silk.dreamhost.com/sabayon.org # this must be run before calling the other functions def __init__(self, ftpuri, debug = None): @@ -48,8 +47,6 @@ class handlerFTP: debug = True else: debug = False - # FIXME: remove this - #debug = True # import FTP modules import timeoutsocket @@ -208,6 +205,11 @@ class handlerFTP: mirrorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"handlerFTP.deleteFile: result -> False (exception occured!!!)") return False + def mkdir(self,directory): + mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.mkdir: called for -> "+str(directory)) + # FIXME: add rc + self.ftpconn.mkd(directory) + # this function also supports callback, because storbinary doesn't def advancedStorBinary(self, cmd, fp, callback=None, blocksize=8192): mirrorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"handlerFTP.advancedStorBinary: called with -> "+str(cmd)) diff --git a/libraries/reagentTools.py b/libraries/reagentTools.py index 30267970d..5bc6d219e 100644 --- a/libraries/reagentTools.py +++ b/libraries/reagentTools.py @@ -98,89 +98,126 @@ def update(options): # collect differences between the packages in the database and the ones on the system reagentRequestSeekStore = False + reagentRequestRepackage = False + repackageItems = [] _options = [] for opt in options: if opt.startswith("--seekstore"): reagentRequestSeekStore = True + elif opt.startswith("--repackage"): + reagentRequestRepackage = True else: + if (reagentRequestRepackage) and (not opt.startswith("--")): + if not opt in repackageItems: + repackageItems.append(opt) + continue _options.append(opt) options = _options - if (not reagentRequestSeekStore): - print_info(yellow(" * ")+red("Scanning the database for differences...")) dbconn = databaseTools.etpDatabase(readOnly = True, noUpload = True) - from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot - installedPackages = getInstalledPackagesCounters() - installedCounters = {} - databasePackages = dbconn.listAllPackages() - toBeAdded = [] - toBeRemoved = [] - - # fill lists - - # packages to be added - for x in installedPackages[0]: - installedCounters[x[1]] = 1 - counter = dbconn.isCounterAvailable(x[1]) - if (not counter): - toBeAdded.append(x) + + if (not reagentRequestRepackage): + print_info(yellow(" * ")+red("Scanning the database for differences...")) + from portageTools import getInstalledPackagesCounters, quickpkg, getPackageSlot + installedPackages = getInstalledPackagesCounters() + installedCounters = {} + databasePackages = dbconn.listAllPackages() + toBeAdded = [] + toBeRemoved = [] + + # packages to be added + for x in installedPackages[0]: + installedCounters[x[1]] = 1 + counter = dbconn.isCounterAvailable(x[1]) + if (not counter): + toBeAdded.append(x) - # packages to be removed from the database - databaseCounters = dbconn.listAllCounters() - for x in databaseCounters: - match = installedCounters.get(x[0], None) - #print match - if (not match): - # check if the package is in toBeAdded - if (toBeAdded): - atomkey = dep_getkey(dbconn.retrieveAtom(x[1])) - atomslot = dbconn.retrieveSlot(x[1]) - add = True - for pkgdata in toBeAdded: - addslot = getPackageSlot(pkgdata[0]) - addkey = dep_getkey(pkgdata[0]) - # workaround for ebuilds not having slot - if addslot == None: - addslot = '' - if (atomkey == addkey) and (atomslot == addslot): - # do not add to toBeRemoved - add = False - break - if add: - toBeRemoved.append(x[1]) - else: - toBeRemoved.append(x[1]) - - if (not toBeRemoved) and (not toBeAdded): - print_info(yellow(" * ")+red("Nothing to do, check later.")) - # then exit gracefully - sys.exit(0) - - if (toBeRemoved): - print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:")) - for x in toBeRemoved: - atom = dbconn.retrieveAtom(x) - print_info(yellow(" # ")+red(atom)) - rc = askquestion(">> Would you like to remove them now ?") - if rc == "Yes": - rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) - for x in toBeRemoved: - atom = rwdbconn.retrieveAtom(x) - print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True) - rwdbconn.removePackage(x) - rwdbconn.closeDB() - print_info(yellow(" @@ ")+blue("Database removal complete.")) - - if (toBeAdded): - print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:")) - for x in toBeAdded: - print_info(yellow(" # ")+red(x[0])) - rc = askquestion(">> Would you like to packetize them now ?") - if rc == "No": + # packages to be removed from the database + databaseCounters = dbconn.listAllCounters() + for x in databaseCounters: + match = installedCounters.get(x[0], None) + #print match + if (not match): + # check if the package is in toBeAdded + if (toBeAdded): + atomkey = dep_getkey(dbconn.retrieveAtom(x[1])) + atomslot = dbconn.retrieveSlot(x[1]) + add = True + for pkgdata in toBeAdded: + addslot = getPackageSlot(pkgdata[0]) + addkey = dep_getkey(pkgdata[0]) + # workaround for ebuilds not having slot + if addslot == None: + addslot = '' + if (atomkey == addkey) and (atomslot == addslot): + # do not add to toBeRemoved + add = False + break + if add: + toBeRemoved.append(x[1]) + else: + toBeRemoved.append(x[1]) + + if (not toBeRemoved) and (not toBeAdded): + print_info(yellow(" * ")+red("Nothing to do, check later.")) + # then exit gracefully sys.exit(0) + if (toBeRemoved): + print_info(yellow(" @@ ")+blue("These are the packages that would be removed from the database:")) + for x in toBeRemoved: + atom = dbconn.retrieveAtom(x) + print_info(yellow(" # ")+red(atom)) + rc = askquestion(">> Would you like to remove them now ?") + if rc == "Yes": + rwdbconn = databaseTools.etpDatabase(readOnly = False, noUpload = True) + for x in toBeRemoved: + atom = rwdbconn.retrieveAtom(x) + print_info(yellow(" @@ ")+blue("Removing from database: ")+red(atom), back = True) + rwdbconn.removePackage(x) + rwdbconn.closeDB() + print_info(yellow(" @@ ")+blue("Database removal complete.")) + + if (toBeAdded): + print_info(yellow(" @@ ")+blue("These are the packages that would be added/updated to the add list:")) + for x in toBeAdded: + print_info(yellow(" # ")+red(x[0])) + rc = askquestion(">> Would you like to packetize them now ?") + if rc == "No": + sys.exit(0) + + else: + if not repackageItems: + print_info(yellow(" * ")+red("Nothing to do, check later.")) + # then exit gracefully + sys.exit(0) + + from portageTools import getPortageAppDbPath,quickpkg + appdb = getPortageAppDbPath() + + packages = [] + for item in repackageItems: + match = dbconn.atomMatch(item) + if match[0] == -1: + print_warning(darkred(" !!! ")+red("Cannot match ")+bold(item)) + else: + cat = dbconn.retrieveCategory(match[0]) + name = dbconn.retrieveName(match[0]) + version = dbconn.retrieveVersion(match[0]) + slot = dbconn.retrieveSlot(match[0]) + if os.path.isdir(appdb+"/"+cat+"/"+name+"-"+version): + packages.append([cat+"/"+name+"-"+version,0]) + + # FIXME: complete this + if not packages: + print_info(yellow(" * ")+red("Nothing to do, check later.")) + # then exit gracefully + sys.exit(0) + + toBeAdded = packages + # package them print_info(yellow(" @@ ")+blue("Compressing packages...")) for x in toBeAdded: @@ -324,9 +361,7 @@ def extractPkgData(package, etpBranch = etpConst['branch']): f.close() print_info(yellow(" * ")+red(info_package+"Setting package branch..."),back = True) - # always unstable when created - i = etpConst['branches'].index(etpBranch) - etpData['branch'] = etpConst['branches'][i] + etpData['branch'] = etpBranch print_info(yellow(" * ")+red(info_package+"Getting package description..."),back = True) # Fill description diff --git a/server/reagent b/server/reagent index 095b3d979..889d97bfb 100644 --- a/server/reagent +++ b/server/reagent @@ -42,19 +42,17 @@ def print_help(): print_info(" --nocolor\t\tdisable colorized output") print_info(blue("Tools available: ")) print_info(" \t"+green("update")+yellow("\t\t Update Entropy Database analyzing the system for new installed packages")) - print_info(" \t\t"+red("--branch=")+"\t\t Choose which branch to assign to the packages") + print_info(" \t\t"+red("--branch=")+"\t\t Choose which branch assign to the packages") print_info(" \t\t"+red("--seekstore")+"\t\t\t Skip differential COUNTERS scanning and analyze STORE directory") + print_info(" \t\t"+red("--repackage")+"\t\t\t Repackage specified atoms") print_info(" \t"+green("database")+yellow("\t Entropy database tool manager")) print_info(" \t\t"+red("--initialize")+"\t\t\t (Re)Initialize the Entropy packages database [DO NOT USE THIS]") - print_info(" \t\t"+red("statistics")+"\t\t\t Show Entropy database statistics.") print_info(" \t\t"+green("search")+"\t\t\t\t Search a package inside the Entropy packages database") print_info(" \t\t"+green("remove")+"\t\t\t\t Remove a package or a list of packages") print_info(" \t\t\t"+red("--branch=")+"\t Choose which branch of the package to remove") print_info(" \t\t"+green("create-empty-database")+"\t\t Create an empty Entropy database file in the specified ") - print_info(" \t\t"+green("stabilize")+"\t\t\t Mark as stable a package, a list of packages, world") - print_info(" \t\t"+green("unstabilize")+"\t\t\t Mark as unstable a package, a list of packages, world") + print_info(" \t\t"+green("switchbranch")+"\t\t\t Switch to the specified branch, a package, a list of packages, world") print_info(" \t\t"+green("md5check")+"\t\t\t Check digest of a package, a list of packages, world") - #print_info(" \t\t"+green(bold("orphans"))+"\t\t\t Dump the list of files that don't belong on any installed package") FIXME can be done using sets print_info(" \t"+green("deptest")+yellow("\t\t Look for unsatisfied dependencies inside database")) print_info(" \t"+green("depends")+yellow("\t\t Regenerate depends table (plus database lock and bump)")) print_info(" \t\t"+red("--quiet")+"\t\t\t\t just print the dependencies list")