sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Entropy/Server Interface:

Browse Source 
- daily work on the code migration


git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@1560 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
lxnay
2008-04-05 17:06:20 +00:00
parent ceb50912f7
commit 18baa11a8d
5 changed files with 259 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/activatorTools.py
+1 -34
View File
@@ -24,12 +24,11 @@ import time
import shutil
from entropyConstants import *
from outputTools import *
from entropy import FtpInterface, EquoInterface, rssFeed, LogFile
from entropy import FtpInterface, EquoInterface, rssFeed
import exceptionTools
Entropy = EquoInterface(noclientdb = 2)
# Logging initialization
activatorLog = LogFile(level=etpConst['activatorloglevel'],filename = etpConst['activatorlogfile'], header = "[Activator]")
def sync(options, justTidy = False):
@@ -42,7 +41,6 @@ def sync(options, justTidy = False):
myopts.append(i)
options = myopts
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: called with justTidy -> "+str(justTidy))
print_info(green(" * ")+red("Starting to sync data across mirrors (packages/database) ..."))
if (not justTidy):
@@ -105,7 +103,6 @@ def sync(options, justTidy = False):
# not expired.
if (not removeList):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: no packages to remove from the lirrors.")
print_info(green(" * ")+red("No packages to remove from the mirrors."))
print_info(green(" * ")+red("Syncronization across mirrors completed."))
continue
@@ -121,15 +118,12 @@ def sync(options, justTidy = False):
sys.exit(0)
# remove them!
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: starting to remove packages from mirrors.")
for uri in etpConst['activatoruploaduris']:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"sync: connecting to mirror "+Entropy.entropyTools.extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Connecting to: ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri)))
ftp = FtpInterface(uri, Entropy)
ftp.setCWD(etpConst['binaryurirelativepath']+"/"+mybranch)
for xfile in removeList:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (remote) file "+xfile)
print_info(green(" * ")+red("Removing file: ")+bold(xfile), back = True)
# remove remotely
if (ftp.isFileAvailable(xfile)):
@@ -147,7 +141,6 @@ def sync(options, justTidy = False):
print_warning(brown(" * ")+red("ATTENTION: remote checksum file ")+bold(xfile)+red(" cannot be removed."))
# remove locally
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile)
print_info(green(" * ")+red("Package file: ")+bold(xfile)+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile)
try:
@@ -156,7 +149,6 @@ def sync(options, justTidy = False):
pass
# checksum
if os.path.isfile(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext']):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"sync: removing (local) file "+xfile+etpConst['packageshashfileext'])
print_info(green(" * ")+red("Checksum file: ")+bold(xfile+etpConst['packageshashfileext'])+red(" removed successfully from ")+bold(etpConst['packagesbindir']+"/"+mybranch))
os.remove(etpConst['packagesbindir']+"/"+mybranch+"/"+xfile+etpConst['packageshashfileext'])
ftp.closeConnection()
@@ -166,8 +158,6 @@ def sync(options, justTidy = False):
def packages(options):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called with options -> "+str(options))
# Options available for all the packages submodules
myopts = options[1:]
activatorRequestPackagesCheck = False
@@ -187,8 +177,6 @@ def packages(options):
totalSuccessfulUri = 0
mirrorsTainted = False
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: called sync.")
for uri in etpConst['activatoruploaduris']:
uriSuccessfulSync = 0
@@ -213,7 +201,6 @@ def packages(options):
if tbz2.endswith(".tbz2"):
uploadCounter += 1
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: upload directory stats -> files: "+str(uploadCounter)+" | upload packages list: "+str(toBeUploaded))
print_info(green(" * ")+red("Upload directory:\t\t")+bold(str(uploadCounter))+red(" files ready."))
localPackagesRepository = set() # parse etpConst['packagesbindir']
@@ -225,7 +212,6 @@ def packages(options):
if tbz2.endswith(".tbz2"):
packageCounter += 1
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: packages directory stats -> files: "+str(packageCounter)+" | download packages list (including md5): "+str(localPackagesRepository))
print_info(green(" * ")+red("Packages directory:\t")+bold(str(packageCounter))+red(" files ready."))
@@ -270,7 +256,6 @@ def packages(options):
downloadQueue = set()
removalQueue = set()
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"packages: starting packages calculation...")
finePackages = set()
# Fill uploadQueue and if something weird is found, add the packages to downloadQueue
@@ -645,8 +630,6 @@ def packages(options):
if (str(e) == "100"):
sys.exit(0)
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: Exception caught: "+str(e)+" . Trying to continue if possible.")
activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: cannot properly syncronize "+Entropy.entropyTools.extractFTPHostFromUri(uri)+". Trying to continue if possible.")
# print warning cannot sync uri
print_warning(brown(" * ")+red("ATTENTION: cannot properly syncronize ")+bold(Entropy.entropyTools.extractFTPHostFromUri(uri))+red(". Continuing if possible..."))
@@ -654,7 +637,6 @@ def packages(options):
# decide what to do
if (totalSuccessfulUri > 0) or (etpUi['pretend']):
# we're safe
activatorLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_NORMAL,"packages: at least one mirror has been synced properly. I'm fine.")
print_info(green(" * ")+red("At least one mirror has been synced properly. I'm fine."))
continue
else:
@@ -664,7 +646,6 @@ def packages(options):
else:
# no mirrors were synced properly
# show error and return, do not move files from the upload dir
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"packages: no mirrors have been properly syncronized. Check network status and retry. Cannot continue.")
print_error(brown(" * ")+red("ERROR: no mirrors have been properly syncronized. Check network status and retry. Cannot continue."))
return False
@@ -673,7 +654,6 @@ def packages(options):
if (totalSuccessfulUri > 0):
if etpUi['pretend']:
return False
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"packages: all done. Now it's time to move packages to "+etpConst['packagesbindir'])
pkgbranches = os.listdir(etpConst['packagessuploaddir'])
pkgbranches = [x for x in pkgbranches if os.path.isdir(etpConst['packagessuploaddir']+"/"+x)]
for branch in pkgbranches:
@@ -701,17 +681,14 @@ def packages(options):
def database(options):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: called with -> "+str(options))
# lock tool
if (options[0] == "lock"):
print_info(green(" * ")+green("Starting to lock mirrors' databases..."))
rc = lockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (lock) A problem occured on at least one mirror !")
print_info(green(" * ")+red("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases lock complete"))
# unlock tool
@@ -719,10 +696,8 @@ def database(options):
print_info(green(" * ")+green("Starting to unlock mirrors' databases..."))
rc = lockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Databases lock complete.")
print_info(green(" * ")+green("Databases unlock complete"))
# download lock tool
@@ -730,10 +705,8 @@ def database(options):
print_info(green(" * ")+green("Starting to lock download mirrors' databases..."))
rc = downloadLockDatabases(lock = True)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-lock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror !"))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors lock complete.")
print_info(green(" * ")+green("Download mirrors lock complete"))
# download unlock tool
@@ -741,10 +714,8 @@ def database(options):
print_info(green(" * ")+green("Starting to unlock download mirrors' databases..."))
rc = downloadLockDatabases(lock = False)
if (rc):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database: (download-unlock) A problem occured on at least one mirror !")
print_info(green(" * ")+green("A problem occured on at least one mirror..."))
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: Download mirrors unlock complete.")
print_info(green(" * ")+green("Download mirrors unlock complete"))
# lock status tool
@@ -765,7 +736,6 @@ def database(options):
# database sync tool
elif (options[0] == "sync"):
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database: database sync called.")
print_info(green(" * ")+red("Checking database status ..."), back = True)
@@ -813,15 +783,12 @@ def database(options):
os.remove(etpConst['etpdatabasedir']+"/"+etpConst['etpdatabasetaintfile'])
else:
print
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): At the moment, mirrors are locked, someone is working on their databases, try again later...")
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: At the moment, mirrors are locked, someone is working on their databases, try again later...")
else:
if (dbLockFile):
activatorLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_NORMAL,"database (sync inside activatorTools): Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
raise exceptionTools.OnlineMirrorError("OnlineMirrorError: Mirrors are not locked remotely but the local database is. It is a non-sense. Please remove the lock file "+etpConst['etpdatabasedir']+"/"+etpConst['etpdatabaselockfile'])
else:
activatorLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"database (sync inside activatorTools): Mirrors are not locked. Fetching data...")
print_info(green(" * ")+red("Mirrors are not locked. Fetching data..."))
syncRemoteDatabases()
libraries/databaseTools.py
+5 -10
View File
@@ -3072,16 +3072,10 @@ class etpDatabase:
columns.append(row[1])
return columns
def tablesChecksum(self):
# NOTE: if you will add dependstable to the validation
# please have a look at EquoInterface.__install_package_into_database world calculation cache stuff
self.cursor.execute('select * from baseinfo')
c_hash = 0
for row in self.cursor:
c_hash += hash(str(row))
self.cursor.execute('select * from dependenciesreference')
for row in self.cursor:
c_hash += hash(str(row))
def database_checksum(self):
# primary keys are now autoincrement
self.cursor.execute('select idpackage from baseinfo')
c_hash = hash(tuple(self.cursor.fetchall()))
return str(c_hash)
@@ -3217,6 +3211,7 @@ class etpDatabase:
def createNeededIndex(self):
if (self.dbname != etpConst['serverdbid']) and self.indexing:
self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex ON neededreference ( library )')
self.cursor.execute('CREATE INDEX IF NOT EXISTS neededindex_idneeded ON needed ( idneeded )')
self.commitChanges()
def createUseflagsIndex(self):
libraries/entropy.py
+192 -57
View File
@@ -25,7 +25,7 @@ import commands
import urllib2
import time
from entropyConstants import *
from outputTools import *
from outputTools import TextInterface, print_info, print_warning, print_error, red, brown, blue, green, darkgreen, darkred, bold
import exceptionTools
class matchContainer:
@@ -50,9 +50,7 @@ class EquoInterface(TextInterface):
def __init__(self, indexing = True, noclientdb = 0, xcache = True, user_xcache = False, repo_validation = True):
# Logging initialization
self.equoLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[Equo]")
self.clientLog = LogFile(level = etpConst['equologlevel'],filename = etpConst['equologfile'], header = "[client]")
import dumpTools
self.dumpTools = dumpTools
import databaseTools
@@ -1323,7 +1321,8 @@ class EquoInterface(TextInterface):
if self.xcache:
c_data = list(dependencies)
c_data.sort()
c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps))
client_checksum = self.clientDbconn.database_checksum()
c_hash = str(hash(tuple(c_data)))+str(hash(deep_deps))+client_checksum
c_hash = str(hash(c_hash))
del c_data
cached = self.dumpTools.loadobj(etpCache['filter_satisfied_deps']+c_hash)
@@ -1610,7 +1609,8 @@ class EquoInterface(TextInterface):
if self.xcache:
c_data = list(matched_atoms)
c_data.sort()
c_hash = str(hash(tuple(c_data)))+str(hash(empty_deps))+str(hash(deep_deps))
client_checksum = self.clientDbconn.database_checksum()
c_hash = str(hash(tuple(c_data)))+str(hash(empty_deps))+str(hash(deep_deps))+client_checksum
c_hash = str(hash(c_hash))
del c_data
cached = self.dumpTools.loadobj(etpCache['dep_tree']+c_hash)
@@ -1805,7 +1805,7 @@ class EquoInterface(TextInterface):
dbconn = self.openRepositoryDatabase(repo)
except exceptionTools.RepositoryError:
continue # repo not available
sum_hashes += dbconn.tablesChecksum()
sum_hashes += dbconn.database_checksum()
return sum_hashes
def get_available_packages_chash(self, branch):
@@ -2114,6 +2114,9 @@ class EquoInterface(TextInterface):
if install and removal:
myremmatch = {}
for x in removal:
# XXX check if stupid users removed idpackage while this whole instance is running
if not self.clientDbconn.isIDPackageAvailable(x):
continue
myremmatch.update({(self.entropyTools.dep_getkey(self.clientDbconn.retrieveAtom(x)),self.clientDbconn.retrieveSlot(x)): x})
for packageInfo in install:
dbconn = self.openRepositoryDatabase(packageInfo[1])
@@ -2411,7 +2414,7 @@ class EquoInterface(TextInterface):
path = shiftpath+path
try:
exist = os.lstat(path)
except OSError, e:
except OSError:
continue # skip file
arcname = path[len(shiftpath):] # remove shiftpath
if arcname.startswith("/"):
@@ -2455,8 +2458,7 @@ class EquoInterface(TextInterface):
tbz2.recompose(dbdir)
if edb:
# appending entropy metadata
dbpath = self.inject_entropy_database_into_package(dirpath, pkgdata)
self.inject_entropy_database_into_package(dirpath, pkgdata)
if os.path.isfile(dirpath):
return dirpath
@@ -2510,7 +2512,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package name/version..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
tbz2File = package
@@ -2546,7 +2548,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package md5..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# .tbz2 md5
@@ -2557,7 +2559,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package mtime..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# .tbz2 md5
@@ -2568,7 +2570,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package size..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# .tbz2 byte size
@@ -2579,7 +2581,7 @@ class EquoInterface(TextInterface):
red(info_package+"Unpacking package data..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# unpack file
@@ -2595,7 +2597,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package CHOST..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill chost
@@ -2608,7 +2610,7 @@ class EquoInterface(TextInterface):
red(info_package+"Setting package branch..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
data['branch'] = etpBranch
@@ -2618,7 +2620,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package description..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill description
@@ -2635,7 +2637,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package homepage..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill homepage
@@ -2652,7 +2654,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package slot information..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# fill slot, if it is
@@ -2669,7 +2671,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package injection information..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# fill slot, if it is
@@ -2683,7 +2685,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package eclasses information..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# fill eclasses list
@@ -2700,7 +2702,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package needed libraries information..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# fill needed list
@@ -2727,7 +2729,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package content..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
data['content'] = {}
@@ -2833,7 +2835,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package category..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill category
@@ -2846,7 +2848,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package download URL..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill download relative URI
@@ -2866,7 +2868,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package counter..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill counter
@@ -2883,7 +2885,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package external trigger availability..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
if os.path.isfile(etpConst['triggersdir']+"/"+data['category']+"/"+data['name']+"/"+etpConst['triggername']):
@@ -2896,7 +2898,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package CFLAGS..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill CFLAGS
@@ -2913,7 +2915,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package CXXFLAGS..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# Fill CXXFLAGS
@@ -2930,7 +2932,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting source package supported ARCHs..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# fill KEYWORDS
@@ -2953,7 +2955,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package dependencies..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
@@ -3007,7 +3009,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package metadata information..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
portage_metadata = Spm.calculate_dependencies(iuse, use, lics, depend, rdepend, pdepend, provide, sources)
@@ -3052,7 +3054,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting package mirrors list..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# manage data['sources'] to create data['mirrorlinks']
@@ -3070,7 +3072,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting System Packages List..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# write only if it's a systempackage
@@ -3089,7 +3091,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting CONFIG_PROTECT/CONFIG_PROTECT_MASK List..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# write only if it's a systempackage
@@ -3128,7 +3130,7 @@ class EquoInterface(TextInterface):
red(etpConst['logdir']+"/elog")+" not set, have you configured make.conf properly?",
importance = 1,
type = "warning",
header = yellow(" * ")
header = brown(" * ")
)
if not silent:
@@ -3136,7 +3138,7 @@ class EquoInterface(TextInterface):
red(info_package+"Getting Entropy API version..."),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
# write API info
@@ -3155,7 +3157,7 @@ class EquoInterface(TextInterface):
red(info_package+"Done"),
importance = 0,
type = "info",
header = yellow(" * "),
header = brown(" * "),
back = True
)
return data
@@ -3911,8 +3913,7 @@ class PackageInterface:
try:
exist = os.lstat(path)
except OSError, e:
#print e
except OSError:
continue # skip file
ftype = package_content[encoded_path]
if str(ftype) == '0': ftype = 'dir' # force match below, '0' means databases without ftype
@@ -5490,7 +5491,7 @@ class RepoInterface:
header = darkred(" @@ ")
)
self.syncErrors = True
self._resources_run_remove_lock()
self.Entropy._resources_run_remove_lock()
return 128
rc = False
@@ -5552,7 +5553,7 @@ class FtpInterface:
# this must be run before calling the other functions
def __init__(self, ftpuri, EntropyInterface):
if not isinstance(EntropyInterface, (EquoInterface, TextInterface)):
if not isinstance(EntropyInterface, (EquoInterface, TextInterface, ServerInterface)):
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid TextInterface based instance is needed")
self.Entropy = EntropyInterface
@@ -5745,7 +5746,7 @@ class FtpInterface:
myUploadPercentage = str(myUploadPercentage)+"%"
# create text
currentText = brown(" <-> Upload status: ")+green(str(myUploadSize))+"/"+red(str(self.myFileSize))+" kB "+yellow("[")+str(myUploadPercentage)+yellow("]")
currentText = brown(" <-> Upload status: ")+green(str(myUploadSize))+"/"+red(str(self.myFileSize))+" kB "+brown("[")+str(myUploadPercentage)+brown("]")
# print !
print_info(currentText, back = True)
# XXX too slow, reimplement self.updateProgress and do whatever you want
@@ -5871,9 +5872,6 @@ class FtpInterface:
self.ftpconn.quit()
'''
Entropy FTP/HTTP download interface
'''
class urlFetcher:
def __init__(self, url, pathToSave, checksum = True, showSpeed = True, resume = True):
@@ -6043,9 +6041,10 @@ class urlFetcher:
curbarsize = 1
if self.average > self.oldaverage+self.updatestep:
averagesize = (self.average*barsize)/100
for y in range(averagesize):
while averagesize > 0:
curbarsize += 1
bartext += "="
averagesize -= 1
bartext += ">"
diffbarsize = barsize-curbarsize
for y in range(diffbarsize):
@@ -6058,7 +6057,6 @@ class urlFetcher:
if len(average) < 2:
average = " "+average
currentText += " <-> "+average+"% "+bartext
# print !
print_info(currentText,back = True)
@@ -6158,7 +6156,7 @@ class rssFeed:
self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data.strip()
self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data.strip()
def addItem(self, title, link = '', description = '', light = False):
def addItem(self, title, link = '', description = ''):
self.itemscounter += 1
self.items[self.itemscounter] = {}
self.items[self.itemscounter]['title'] = title
@@ -6294,7 +6292,7 @@ class TriggerInterface:
raise exceptionTools.IncorrectParameter("IncorrectParameter: a valid Entropy Instance is needed")
self.Entropy = EquoInstance
self.equoLog = self.Entropy.equoLog
self.clientLog = self.Entropy.equoLog
self.validPhases = ("preinstall","postinstall","preremove","postremove")
self.pkgdata = pkgdata
self.prepared = False
@@ -6789,13 +6787,13 @@ class TriggerInterface:
item = etpConst['systemroot']+item
if item.startswith(etpConst['systemroot']+"/etc/init.d/") and os.path.isfile(item):
# running?
running = os.path.isfile(etpConst['systemroot']+self.INITSERVICES_DIR+'/started/'+os.path.basename(item))
#running = os.path.isfile(etpConst['systemroot']+self.INITSERVICES_DIR+'/started/'+os.path.basename(item))
if not etpConst['systemroot']:
myroot = "/"
else:
myroot = etpConst['systemroot']+"/"
scheduled = not os.system('ROOT="'+myroot+'" rc-update show | grep '+os.path.basename(item)+'&> /dev/null')
self.trigger_initdeactivate(item, running, scheduled)
self.trigger_initdeactivate(item, scheduled)
def trigger_initinform(self):
for item in self.pkgdata['content']:
@@ -7604,7 +7602,7 @@ class TriggerInterface:
@description: shuts down selected init script, and remove from runlevel
@output: returns int() as exit status
'''
def trigger_initdeactivate(self, item, running, scheduled):
def trigger_initdeactivate(self, item, scheduled):
if not etpConst['systemroot']:
myroot = "/"
'''
@@ -7685,7 +7683,7 @@ timeout=10
kernel, initramfs = (unicode(kernel,'raw_unicode_escape'),unicode(initramfs,'raw_unicode_escape'))
except TypeError:
pass
kernelname = os.path.basename(kernel)
#kernelname = os.path.basename(kernel)
new_conf = []
skip = False
for line in content:
@@ -10768,10 +10766,13 @@ class ServerInterface(TextInterface):
if etpConst['uid'] != 0:
raise exceptionTools.PermissionDenied("PermissionDenied: Entropy ServerInterface must be run as root")
self.serverLog = LogFile(level = etpConst['entropyloglevel'],filename = etpConst['entropylogfile'], header = "[server]")
# settings
etpSys['serverside'] = True
self.indexing = False
self.xcache = False
self.FtpInterface = FtpInterface
self.serverDbCache = {}
self.settings_to_backup = []
self.do_save_repository = save_repository
@@ -10796,6 +10797,7 @@ class ServerInterface(TextInterface):
self.setup_entropy_settings()
self.backup_entropy_settings()
self.ClientService = EquoInterface(indexing = self.indexing, xcache = self.xcache, repo_validation = False)
self.ClientService.FtpInterface = self.FtpInterface
self.databaseTools = self.ClientService.databaseTools
self.entropyTools = self.ClientService.entropyTools
self.SpmService = self.ClientService.Spm()
@@ -11169,6 +11171,12 @@ class ServerInterface(TextInterface):
return self.SpmService.quickpkg(atom,storedir)
def remove_package(self, idpackage):
dbconn = self.openServerDatabase(read_only = False, no_upload = True)
dbconn.removePackage(idpackage)
self.close_server_database(dbconn)
def bump_database(self):
dbconn = self.openServerDatabase(read_only = False, no_upload = True)
dbconn.taintDatabase()
@@ -11330,7 +11338,7 @@ class ServerInterface(TextInterface):
f.flush()
f.close()
rc = Entropy.askQuestion("Would you like to sync packages first (important if you don't have them synced) ?")
rc = self.askQuestion("Would you like to sync packages first (important if you don't have them synced) ?")
if rc == "Yes":
activatorTools.packages(["sync","--ask"])
@@ -11819,4 +11827,131 @@ class ServerInterface(TextInterface):
header = brown(" # ")
)
return fine, failed, downloaded_fine, downloaded_errors
self.close_server_database(dbconn)
return fine, failed, downloaded_fine, downloaded_errors
def switch_packages_branch(self, idpackages, to_branch):
self.updateProgress(
red("Switching selected packages..."),
importance = 1,
type = "info",
header = darkgreen(" * ")
)
dbconn = self.openServerDatabase(read_only = False, no_upload = True)
already_switched = set()
not_found = set()
switched = set()
ignored = set()
no_checksum = set()
for idpackage in idpackages:
cur_branch = dbconn.retrieveBranch(idpackage)
atom = dbconn.retrieveAtom(idpackage)
if cur_branch == to_branch:
already_switched.add(idpackage)
self.updateProgress(
red("Ignoring %s, already in branch %s" % (bold(atom),cur_branch,)),
importance = 0,
type = "info",
header = darkgreen(" * ")
)
ignored.add(idpackage)
continue
old_filename = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
# check if file exists
frompath = os.path.join(etpConst['packagesserverbindir'],cur_branch+"/"+old_filename)
if not os.path.isfile(frompath):
self.updateProgress(
red("[%s=>%s] %s, cannot switch, package not found!" % (cur_branch,to_branch,atom,)),
importance = 0,
type = "warning",
header = darkred(" !!! ")
)
not_found.add(idpackage)
continue
self.updateProgress(
red("[%s=>%s] %s, configuring package information..." % (cur_branch,to_branch,atom,)),
importance = 0,
type = "info",
header = darkgreen(" * "),
back = True
)
dbconn.switchBranch(idpackage,to_branch)
dbconn.commitChanges()
# LOCAL
self.updateProgress(
red("[%s -> %s] %s, moving file locally..." % (cur_branch,to_branch,atom,)),
importance = 0,
type = "info",
header = darkgreen(" * "),
back = True
)
new_filename = os.path.basename(dbconn.retrieveDownloadURL(idpackage))
topath = os.path.join(etpConst['packagesserverbindir'],to_branch)
if not os.path.isdir(topath):
os.makedirs(topath)
topath = os.path.join(topath,new_filename)
shutil.move(frompath,topath)
if os.path.isfile(frompath+etpConst['packageshashfileext']):
shutil.move(frompath+etpConst['packageshashfileext'],topath+etpConst['packageshashfileext'])
else:
self.updateProgress(
red("[%s=>%s] %s, cannot find checksum to migrate!" % (cur_branch,to_branch,atom,)),
importance = 0,
type = "warning",
header = darkred(" !!! ")
)
no_checksum.add(idpackage)
# REMOTE
self.updateProgress(
red("[%s=>%s] %s, moving file remotely..." % (cur_branch,to_branch,atom,)),
importance = 0,
type = "info",
header = darkgreen(" * "),
back = True
)
for uri in etpConst['activatoruploaduris']:
crippled_uri = self.entropyTools.extractFTPHostFromUri(uri)
self.updateProgress(
red("[%s=>%s] %s, moving file remotely on: %s" % (cur_branch,to_branch,atom,crippled_uri,)),
importance = 0,
type = "info",
header = darkgreen(" * "),
back = True
)
ftp = self.FtpInterface(uri, self)
ftp.setCWD(etpConst['binaryurirelativepath'])
# create directory if it doesn't exist
if not ftp.isFileAvailable(to_branch):
ftp.mkdir(to_branch)
fromuri = os.path.join(cur_branch,old_filename)
touri = os.path.join(to_branch,new_filename)
ftp.renameFile(fromuri,touri)
ftp.renameFile(fromuri+etpConst['packageshashfileext'],touri+etpConst['packageshashfileext'])
ftp.closeConnection()
switched.add(idpackage)
dbconn.commitChanges()
self.close_server_database(dbconn)
self.updateProgress(
red("[%s=>%s] migration loop completed." % (cur_branch,to_branch,)),
importance = 1,
type = "info",
header = darkgreen(" * ")
)
return switched, already_switched, ignored, not_found, no_checksum
libraries/entropyConstants.py
+1 -21
View File
@@ -581,8 +581,6 @@ def const_defaultSettings(rootdir):
'triggername': "trigger", # name of the trigger file that would be executed by equo inside triggerTools
'proxy': {}, # proxy configuration information, used system wide
'reagentloglevel': 1 , # Reagent log level (default: 1 - see reagent.conf for more info)
'activatorloglevel': 1, # # Activator log level (default: 1 - see activator.conf for more info)
'entropyloglevel': 1, # # Entropy log level (default: 1 - see entropy.conf for more info)
'equologlevel': 1, # # Equo log level (default: 1 - see equo.conf for more info)
'logdir': ETP_LOG_DIR , # Log dir where ebuilds store their stuff
@@ -1209,15 +1207,6 @@ def const_readActivatorSettings():
format = line.split("database-format|")[1]
if format in etpConst['etpdatabasesupportedcformats']:
etpConst['etpdatabasefileformat'] = format
elif line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2):
loglevel = line.split("loglevel|")[1]
try:
loglevel = int(loglevel)
except ValueError:
pass
if (loglevel > -1) and (loglevel < 3):
etpConst['activatorloglevel'] = loglevel
def const_readReagentSettings():
@@ -1226,17 +1215,8 @@ def const_readReagentSettings():
reagentconf = f.readlines()
f.close()
for line in reagentconf:
if line.startswith("loglevel|") and (len(line.split("loglevel|")) == 2):
loglevel = line.split("loglevel|")[1]
try:
loglevel = int(loglevel)
except ValueError:
pass
if (loglevel > -1) and (loglevel < 3):
etpConst['reagentloglevel'] = loglevel
elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2):
if line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2):
feed = line.split("rss-feed|")[1]
if feed in ("enable","enabled","true","1"):
etpConst['rss-feed'] = True
server/server_reagent.py
+60 -143
View File
@@ -224,7 +224,7 @@ def database(options):
_options.append(opt)
options = _options
if len(options) == 0:
if not options:
print_error(brown(" * ")+red("Not enough parameters"))
return 1
@@ -240,7 +240,6 @@ def database(options):
dbpath = None
if myopts:
dbpath = myopts[0]
print_info(darkgreen(" * ")+red("Creating empty database to: ")+dbpath)
if os.path.isfile(dbpath):
print_error(darkgreen(" * ")+red("Cannot overwrite already existing file: ")+dbpath)
@@ -250,170 +249,101 @@ def database(options):
elif (options[0] == "switchbranch"):
if (len(options) < 2):
if (len(options) < 3):
print_error(brown(" * ")+red("Not enough parameters"))
return 6
return 1
switchbranch = options[1]
print_info(green(" * ")+red("Collecting packages that would be marked '"+switchbranch+"' ..."), back = True)
print_info(darkgreen(" * ")+red("Switching branch, be sure to have your packages in sync."))
print_info(darkgreen(" * ")+red("Collecting packages that would be marked %s..." % (switchbranch,) ), back = True)
dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True)
pkglist = set()
myatoms = options[2:]
if not myatoms:
print_error(brown(" * ")+red("Not enough parameters"))
return 7
dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True)
# is world?
if myatoms[0] == "world":
pkglist = dbconn.listAllIdpackages()
if "world" in myatoms:
pkglist |= dbconn.listAllIdpackages()
else:
pkglist = set()
for atom in myatoms:
# validate atom
match = dbconn.atomMatch(atom)
if match == -1:
print_warning(brown(" * ")+red("Cannot match: ")+bold(atom))
else:
pkglist.add(match[0])
# check if atoms were found
if not pkglist:
print
print_error(brown(" * ")+red("No packages found."))
return 8
return 3
# show what would be done
print_info(green(" * ")+red("These are the packages that would be marked '"+switchbranch+"':"))
print_info(darkgreen(" * ")+red("These are the packages that would be marked %s:" % (switchbranch,)))
for idpackage in pkglist:
atom = dbconn.retrieveAtom(idpackage)
print_info(red(" # ")+bold(atom))
for pkg in pkglist:
atom = dbconn.retrieveAtom(pkg)
print_info(red(" (*) ")+bold(atom))
rc = Entropy.askQuestion(" Would you like to continue ?")
rc = Entropy.askQuestion("Would you like to continue ?")
if rc == "No":
return 9
return 4
# sync packages
import activatorTools
ask = etpUi['ask']
etpUi['ask'] = True
activatorTools.packages(["sync"])
etpUi['ask'] = ask
print_info(green(" * ")+red("Switching selected packages ..."))
for pkg in pkglist:
atom = dbconn.retrieveAtom(pkg)
currentbranch = dbconn.retrieveBranch(pkg)
currentdownload = dbconn.retrieveDownloadURL(pkg)
if currentbranch == switchbranch:
print_warning(green(" * ")+red("Ignoring ")+bold(atom)+red(" since it is already in the chosen branch"))
continue
print_info(green(" * ")+darkred(atom+": ")+red("Configuring package information..."), back = True)
# change branch and download URL
dbconn.switchBranch(pkg,switchbranch)
# rename locally
filename = os.path.basename(dbconn.retrieveDownloadURL(pkg))
topath = etpConst['packagesbindir']+"/"+switchbranch
if not os.path.isdir(topath):
os.makedirs(topath)
print_info(green(" * ")+darkred(atom+": ")+red("Moving file locally..."), back = True)
#print etpConst['entropyworkdir']+"/"+currentdownload+" --> "+topath+"/"+newdownload
os.rename(etpConst['entropyworkdir']+"/"+currentdownload,topath+"/"+filename)
# md5
os.rename(etpConst['entropyworkdir']+"/"+currentdownload+etpConst['packageshashfileext'],topath+"/"+filename+etpConst['packageshashfileext'])
# XXX: we can barely ignore branch info injected into .tbz2 since they'll be ignored too
# rename remotely
print_info(green(" * ")+darkred(atom+": ")+red("Moving file remotely..."), back = True)
# change filename remotely
for uri in etpConst['activatoruploaduris']:
print_info(green(" * ")+darkred(atom+": ")+red("Moving file remotely on: ")+Entropy.entropyTools.extractFTPHostFromUri(uri), back = True)
ftp = FtpInterface(uri, Entropy)
ftp.setCWD(etpConst['binaryurirelativepath'])
# create directory if it doesn't exist
if (not ftp.isFileAvailable(switchbranch)):
ftp.mkdir(switchbranch)
# rename tbz2
ftp.renameFile(currentbranch+"/"+filename,switchbranch+"/"+filename)
# rename md5
ftp.renameFile(currentbranch+"/"+filename+etpConst['packageshashfileext'],switchbranch+"/"+filename+etpConst['packageshashfileext'])
ftp.closeConnection()
dbconn.closeDB()
print_info(green(" * ")+red("All the selected packages have been marked as requested. Remember to run activator."))
switched, already_switched, ignored, not_found, no_checksum = Entropy.switch_packages_branch(pkglist, to_branch = switchbranch)
if not_found or no_checksum:
return 1
return 0
elif (options[0] == "remove"):
print_info(green(" * ")+red("Scanning packages that would be removed ..."), back = True)
myopts = options[1:]
_myopts = []
print_info(darkgreen(" * ")+red("Matching packages to remove..."), back = True)
myopts = []
branch = None
for opt in myopts:
for opt in options[1:]:
if (opt.startswith("--branch=")) and (len(opt.split("=")) == 2):
branch = opt.split("=")[1]
else:
_myopts.append(opt)
myopts = _myopts
myopts.append(opt)
if len(myopts) == 0:
if not myopts:
print_error(brown(" * ")+red("Not enough parameters"))
return 10
return 1
dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True)
pkglist = set()
dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True)
for atom in myopts:
if (branch):
pkg = dbconn.atomMatch(atom, matchBranches = (branch,))
if branch:
pkg = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True)
else:
pkg = dbconn.atomMatch(atom)
if pkg[0] != -1:
pkglist.add(pkg[0])
pkg = dbconn.atomMatch(atom, multiMatch = True)
if pkg[1] == 0:
for idpackage in pkg[0]:
pkglist.add(idpackage)
# check if atoms were found
if not pkglist:
print
dbconn.closeDB()
print_error(brown(" * ")+red("No packages found."))
return 11
return 2
print_info(green(" * ")+red("These are the packages that would be removed from the database:"))
print_info(darkgreen(" * ")+red("These are the packages that would be removed from the database:"))
for idpackage in pkglist:
pkgatom = dbconn.retrieveAtom(idpackage)
branch = dbconn.retrieveBranch(idpackage)
print_info(red(" # ")+blue("[")+red(branch)+blue("] ")+bold(pkgatom))
for pkg in pkglist:
pkgatom = dbconn.retrieveAtom(pkg)
branch = dbconn.retrieveBranch(pkg)
print_info(red("\t (*) ")+bold(pkgatom)+blue(" [")+red(branch)+blue("]"))
# ask to continue
rc = Entropy.askQuestion(" Would you like to continue ?")
rc = Entropy.askQuestion("Would you like to continue ?")
if rc == "No":
return 0
# now mark them as stable
print_info(green(" * ")+red("Removing selected packages ..."))
print_info(darkgreen(" * ")+red("Removing selected packages..."))
for idpackage in pkglist:
pkgatom = dbconn.retrieveAtom(idpackage)
print_info(darkgreen(" * ")+red("Removing package: ")+bold(pkgatom)+red("..."), back = True)
Entropy.remove_package(idpackage)
Entropy.close_server_database(dbconn)
print_info(darkgreen(" * ")+red("Packages removed. To remove binary packages, run activator."))
# open db
for pkg in pkglist:
pkgatom = dbconn.retrieveAtom(pkg)
print_info(green(" * ")+red("Removing package: ")+bold(pkgatom)+red(" ..."), back = True)
dbconn.removePackage(pkg)
print_info(green(" * ")+red("All the selected packages have been removed as requested. To remove online binary packages, just run Activator."))
dbconn.closeDB()
return 0
elif (options[0] == "multiremove"):
print_info(green(" * ")+red("Scanning packages that would be removed ..."), back = True)
print_info(darkgreen(" * ")+red("Searching injected packages to remove..."), back = True)
branch = etpConst['branch']
atoms = []
@@ -423,58 +353,46 @@ def database(options):
else:
atoms.append(opt)
pkglist = set()
dbconn = Entropy.openServerDatabase(read_only = True, no_upload = True)
allidpackages = dbconn.listAllIdpackages()
idpackages = set()
if not atoms:
# choose all
allidpackages = dbconn.listAllIdpackages()
for idpackage in allidpackages:
if dbconn.isInjected(idpackage):
idpackages.add(idpackage)
else:
for atom in atoms:
match = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True, packagesFilter = False)
if match[1] != 0:
print_warning(red("Attention, no match for: ")+bold(atom))
else:
match = dbconn.atomMatch(atom, matchBranches = (branch,), multiMatch = True)
if match[1] == 0:
for x in match[0]:
if dbconn.isInjected(x):
idpackages.add(x)
# check if atoms were found
if not idpackages:
dbconn.closeDB()
print_error(brown(" * ")+red("No packages found."))
return 11
return 1
print_info(green(" * ")+blue("These are the packages that would be removed from the database:"))
print_info(darkgreen(" * ")+blue("These are the injected packages pulled in for removal:"))
for idpackage in idpackages:
pkgatom = dbconn.retrieveAtom(idpackage)
branch = dbconn.retrieveBranch(idpackage)
print_info(darkred(" (*) ")+blue("[")+red(branch)+blue("] ")+brown(pkgatom))
print_info(darkred(" # ")+blue("[")+red(branch)+blue("] ")+brown(pkgatom))
# ask to continue
rc = Entropy.askQuestion(" Would you like to continue ?")
rc = Entropy.askQuestion("Would you like to continue ?")
if rc == "No":
return 0
dbconn.closeDB()
del dbconn
dbconn = Entropy.openServerDatabase(read_only = False, no_upload = True)
print_info(green(" * ")+red("Removing selected packages ..."))
# open db
for idpackage in idpackages:
pkgatom = dbconn.retrieveAtom(idpackage)
print_info(green(" * ")+red("Removing package: ")+bold(pkgatom)+red(" ..."))
dbconn.removePackage(idpackage)
print_info(green(" * ")+red("All the selected packages have been removed as requested."))
dbconn.closeDB()
del dbconn
print_info(darkgreen(" * ")+red("Removing package: ")+bold(pkgatom)+red("..."), back = True)
Entropy.remove_package(idpackage)
Entropy.close_server_database(dbconn)
print_info(darkgreen(" * ")+red("Packages removed. To remove binary packages, run activator."))
return 0
# used by reagent
@@ -527,4 +445,3 @@ def spm(options):
print ' '.join(["="+x for x in packages])
else:
os.system(etpConst['spm']['exec']+" "+etpConst['spm']['ask_cmd']+" "+etpConst['spm']['verbose_cmd']+" "+" ".join(["="+x for x in packages]))