added preliminary RSS logging - need to be extended

git-svn-id: http://svn.sabayonlinux.org/projects/entropy/trunk@782 cd1c1023-2f26-0410-ae45-c471fc1f0318
This commit is contained in:
(no author)
2007-11-30 02:57:07 +00:00
parent 47b6e1824d
commit 4cef0fc7c3
9 changed files with 372 additions and 22 deletions

3
TODO
View File

@@ -1,5 +1,5 @@
TODO list:
- add support for database revisions log (RSS)
- RSS Extension: let developer sumbit a comment along with the revision upload
- Community repositories
- server: handle multiple packages for each scope with these abilities:
- enable/disable it
@@ -8,6 +8,7 @@ TODO list:
- add external trigger for splashutils
- find a way to better handle real smartapps deps
- reduce size of packages.db.bz2
- rethink external triggers?
Project Status:

View File

@@ -29,3 +29,15 @@ ftp-proxy|
#
# FTP Proxy default setting
http-proxy|
#
# syntax for system-name:
#
# system-name: Name of the running Operating System
# system-name|<string with spaces>
#
# example:
# system-name|Sabayon Linux
#
# Default Operating System name
system-name|Sabayon Linux

View File

@@ -6,3 +6,26 @@
# 2: Verbose Logging
loglevel|1
# Online RSS support to track changes of the things happened on the database.
# Here are basic settings, like the ability to enable/disable this feature or set the name of the feed.
# Feed will be uploaded on the same packages.db.[bz2|gz] directory
#
# syntax for rss-feed:
# rss-feed|<enable/disable>
#
rss-feed|enable
# syntax for rss-name:
# rss-name|<name.rss>
#
rss-name|packages.rss
# This option allows to specify a base URL for the <guid> and <link> RSS entry.
# If you have the entropy web portal online, just add its path to index.py which
# will be wrapped to add ?search=...
# syntax for rss-base-url:
# rss-base-url|<URL>
#
rss-base-url|http://packages.sabayonlinux.org/
# This is just the website url that will be added to the RSS
rss-website-url|http://www.sabayonlinux.org/

View File

@@ -28,6 +28,7 @@ from serverConstants import *
from entropyTools import *
from outputTools import *
import mirrorTools
import dumpTools
from sys import exit
import os
@@ -857,8 +858,6 @@ def syncRemoteDatabases(noUpload = False, justStats = False):
uploadLatest = False
uploadList = []
#print str(etpDbLocalRevision)
# if the local DB does not exist, get the latest
if (etpDbLocalRevision == 0):
# seek mirrors
@@ -920,18 +919,12 @@ def syncRemoteDatabases(noUpload = False, justStats = False):
latestRemoteDb = dbstat
break
# now compare downloadLatest with our local db revision
#print "data revisions:"
#print str(latestRemoteDb[1])
#print str(etpDbLocalRevision)
if (etpDbLocalRevision < latestRemoteDb[1]):
# download !
#print "appending a download"
downloadLatest = latestRemoteDb
elif (etpDbLocalRevision > latestRemoteDb[1]):
# upload to all !
#print str(etpDbLocalRevision)
#print str(latestRemoteDb[1])
#print "appending the upload to all"
uploadLatest = True
uploadList = remoteDbsStatus
@@ -983,6 +976,39 @@ def uploadDatabase(uris):
import gzip
import bz2
### PREPARE RSS FEED
if etpConst['rss-feed']:
import rssTools
rssClass = rssTools.rssFeed(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'])
# load dump
db_actions = dumpTools.loadobj(etpConst['rss-dump-name'])
if db_actions:
try:
f = open(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])
revision = f.readline().strip()
f.close()
except:
revision = "N/A"
pass
title = ": "+etpConst['systemname']+" "+etpConst['product'][0].upper()+etpConst['product'][1:]+" "+etpConst['branch']+" :: Revision: "+revision
link = etpConst['rss-base-url']
# create description
added_items = db_actions.get("added")
if added_items:
for atom in added_items:
description = atom+": "+added_items[atom]['description']
rssClass.addItem(title = "Added/Updated"+title, link = link, description = description)
removed_items = db_actions.get("removed")
if removed_items:
for atom in removed_items:
description = atom+": "+removed_items[atom]['description']
rssClass.addItem(title = "Removed"+title, link = link, description = description)
rssClass.writeChanges()
# clean global vars
etpRSSMessages.clear()
dumpTools.removeobj(etpConst['rss-dump-name'])
for uri in uris:
downloadLockDatabases(True,[uri])
@@ -1035,14 +1061,23 @@ def uploadDatabase(uris):
entropyLog.log(ETP_LOGPRI_ERROR,ETP_LOGLEVEL_VERBOSE,"uploadDatabase: uploading to: "+extractFTPHostFromUri(uri)+" UNSUCCESSFUL! ERROR!.")
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
# uploading revision file
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" ..."), back = True)
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'],True)
if (rc == True):
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed. Disconnecting."))
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['etpdatabaserevisionfile'])+red(" completed."))
else:
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# uploading rss file (if enabled)
if etpConst['rss-feed'] and os.path.isfile(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name']):
print_info(green(" * ")+red("Uploading file ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'])+red(" ..."), back = True)
rc = ftp.uploadFile(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'],True)
if (rc == True):
print_info(green(" * ")+red("Upload of ")+bold(etpConst['etpdatabasedir'] + "/" + etpConst['rss-name'])+red(" completed."))
else:
print_warning(yellow(" * ")+red("Cannot properly upload to ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# close connection
ftp.closeConnection()
# unlock database
@@ -1096,13 +1131,31 @@ def downloadDatabase(uri):
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['etpdatabasehashfile'])+red(" ..."), back = True)
rc = ftp.downloadFile(etpConst['etpdatabasehashfile'],os.path.dirname(etpConst['etpdatabasefilepath']),True)
if (rc == True):
print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabasehashfile'])+red(" completed. Disconnecting."))
print_info(green(" * ")+red("Download of ")+bold(etpConst['etpdatabasehashfile'])+red(" completed."))
else:
entropyLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_VERBOSE,"downloadDatabase: Cannot properly download from "+extractFTPHostFromUri(uri)+". Please check.")
print_warning(yellow(" * ")+red("Cannot properly download from ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
# download RSS
if etpConst['rss-feed']:
entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadDatabase: downloading RSS file for "+extractFTPHostFromUri(uri))
print_info(green(" * ")+red("Downloading file to ")+bold(etpConst['rss-name'])+red(" ..."), back = True)
try:
rc = ftp.downloadFile(etpConst['rss-name'],etpConst['etpdatabasedir'],True)
if (rc == True):
print_info(green(" * ")+red("Download of ")+bold(etpConst['rss-name'])+red(" completed."))
else:
entropyLog.log(ETP_LOGPRI_WARNING,ETP_LOGLEVEL_VERBOSE,"downloadDatabase: Cannot properly download from "+extractFTPHostFromUri(uri)+". Please check.")
print_warning(yellow(" * ")+red("Cannot properly download from ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
except:
print_warning(yellow(" * ")+red("Cannot properly download RSS file: "+etpConst['rss-name']+" for: ")+bold(extractFTPHostFromUri(uri))+red(". Please check."))
entropyLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_VERBOSE,"downloadDatabase: do some tidy.")
spawnCommand("rm -f " + etpConst['etpdatabasedir'] + "/" + dbfilename, "&> /dev/null")
try:
os.remove(etpConst['etpdatabasedir'] + "/" + dbfilename)
except OSError:
pass
# close connection
ftp.closeConnection()

View File

@@ -501,11 +501,12 @@ class etpDatabase:
trigger = 1
# baseinfo
pkgatom = etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag
try:
self.cursor.execute(
'INSERT into baseinfo VALUES '
'(NULL,?,?,?,?,?,?,?,?,?,?,?)'
, ( etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag,
, ( pkgatom,
catid,
etpData['name'],
etpData['version'],
@@ -523,7 +524,7 @@ class etpDatabase:
self.cursor.execute(
'INSERT into baseinfo VALUES '
'(NULL,?,?,?,?,?,?,?,?,?,?,?)'
, ( etpData['category']+"/"+etpData['name']+"-"+etpData['version']+versiontag,
, ( pkgatom,
catid,
etpData['name'],
etpData['version'],
@@ -540,6 +541,23 @@ class etpDatabase:
self.connection.commit()
idpackage = self.cursor.lastrowid
### RSS Atom support
### dictionary will be elaborated by activator
if etpConst['rss-feed'] and not self.clientDatabase:
rssAtom = pkgatom+"~"+str(revision)
# store addPackage action
rssObj = dumpTools.loadobj(etpConst['rss-dump-name'])
if rssObj:
global etpRSSMessages
etpRSSMessages = rssObj.copy()
if rssAtom in etpRSSMessages['removed']:
del etpRSSMessages['removed'][rssAtom]
etpRSSMessages['added'][rssAtom] = {}
etpRSSMessages['added'][rssAtom]['description'] = etpData['description']
etpRSSMessages['added'][rssAtom]['homepage'] = etpData['homepage']
# save
dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages)
# create new idflag if it doesn't exist
idflags = self.areCompileFlagsAvailable(etpData['chost'],etpData['cflags'],etpData['cxxflags'])
if (idflags == -1):
@@ -944,6 +962,25 @@ class etpDatabase:
key = self.retrieveAtom(idpackage)
branch = self.retrieveBranch(idpackage)
### RSS Atom support
### dictionary will be elaborated by activator
if etpConst['rss-feed'] and not self.clientDatabase:
# store addPackage action
rssObj = dumpTools.loadobj(etpConst['rss-dump-name'])
if rssObj:
global etpRSSMessages
etpRSSMessages = rssObj.copy()
rssAtom = self.retrieveAtom(idpackage)
rssRevision = self.retrieveRevision(idpackage)
rssAtom += "~"+str(rssRevision)
if rssAtom in etpRSSMessages['added']:
del etpRSSMessages['added'][rssAtom]
etpRSSMessages['removed'][rssAtom] = {}
etpRSSMessages['removed'][rssAtom]['description'] = self.retrieveDescription(idpackage)
etpRSSMessages['removed'][rssAtom]['homepage'] = self.retrieveHomepage(idpackage)
# save
dumpTools.dumpobj(etpConst['rss-dump-name'],etpRSSMessages)
idpackage = str(idpackage)
dbLog.log(ETP_LOGPRI_INFO,ETP_LOGLEVEL_NORMAL,"removePackage: trying to remove (if exists) -> "+idpackage+":"+str(key)+" | branch: "+branch)
# baseinfo

View File

@@ -28,7 +28,7 @@ from entropyConstants import *
@input: name of the object, object
@output: status code
'''
def dumpobj(name,object):
def dumpobj(name, object, completePath = False):
while 1: # trap ctrl+C
doc = minidom.Document()
structure = doc.createElement("structure")
@@ -39,9 +39,13 @@ def dumpobj(name,object):
data.appendChild(text)
# etpConst['dumpstoragedir']
try:
if not os.path.isdir(etpConst['dumpstoragedir']):
os.makedirs(etpConst['dumpstoragedir'])
f = open(etpConst['dumpstoragedir']+"/"+name+".dmp","w") #FIXME add check
if completePath:
dmpfile = name
else:
if not os.path.isdir(etpConst['dumpstoragedir']):
os.makedirs(etpConst['dumpstoragedir'])
dmpfile = etpConst['dumpstoragedir']+"/"+name+".dmp"
f = open(dmpfile,"w")
f.writelines(doc.toprettyxml(indent=" "))
f.flush()
f.close()
@@ -55,8 +59,11 @@ def dumpobj(name,object):
@input: name of the object
@output: object or, if error -1
'''
def loadobj(name):
dmpfile = etpConst['dumpstoragedir']+"/"+name+".dmp"
def loadobj(name, completePath = False):
if completePath:
dmpfile = name
else:
dmpfile = etpConst['dumpstoragedir']+"/"+name+".dmp"
if os.path.isfile(dmpfile):
try:
xmldoc = minidom.parse(dmpfile)
@@ -67,3 +74,10 @@ def loadobj(name):
except:
os.remove(dmpfile)
raise SyntaxError,"cannot load object"
def removeobj(name):
if os.path.isfile(etpConst['dumpstoragedir']+"/"+name+".dmp"):
try:
os.remove(etpConst['dumpstoragedir']+"/"+name+".dmp")
except OSError:
pass

View File

@@ -368,6 +368,12 @@ etpConst = {
"bz2": ("bz2.BZ2File","unpackBzip2","etpdatabasefilebzip2",),
"gz": ("gzip.GzipFile","unpackGzip","etpdatabasefilegzip",)
},
'rss-feed': True, # enable/disable packages RSS feed feature
'rss-name': "packages.rss", # default name of the RSS feed
'rss-base-url': "http://packages.sabayonlinux.org/", # default URL to the entropy web interface (overridden in reagent.conf)
'rss-website-url': "http://www.sabayonlinux.org/", # default URL to the Operating System website (overridden in reagent.conf)
'rss-dump-name': "rss_database_actions", # xml file where will be dumped etpRSSMessages dictionary
'packageshashfileext': ".md5", # Extension of the file that contains the checksum of its releated package file
'packagesexpirationfileext': ".expired", # Extension of the file that "contains" expiration mtime
'packagesexpirationdays': 15, # number of days after a package will be removed from mirrors
@@ -427,7 +433,7 @@ etpConst = {
'dbconfigprotectmask': [], # installed database CONFIG_PROTECT_MASK directories
'configprotectcounter': 0, # this will be used to show the number of updated files at the end of the processes
'entropyversion': "1.0", # default Entropy release version
'systemname': "Sabayon Linux", # default system name
'systemname': "Sabayon Linux", # default system name (overidden by entropy.conf settings)
'product': "standard", # Product identificator (standard, professional...)
'errorstatus': ETP_CONF_DIR+"/code",
@@ -472,6 +478,13 @@ etpExitMessages = {
7: "Go to hell."
}
# information about what has been done on the database,
# those dicts will be dumped to a file and used by activator to update and updload .rss
etpRSSMessages = {
'added': {}, # packages that has been added
'removed': {} # packages that has been removed
}
### Application disk cache
dbCacheStore = {}
atomMatchCache = {}
@@ -597,6 +610,8 @@ if os.path.isfile(etpConst['entropyconf']):
httpproxy = line.split("|")[1].strip()
for x in httpproxy.split():
etpConst['proxy']['http'] = httpproxy
elif line.startswith("system-name|") and (len(line.split("|")) == 2):
etpConst['systemname'] = line.split("|")[1].strip()

181
libraries/rssTools.py Normal file
View File

@@ -0,0 +1,181 @@
#!/usr/bin/python
'''
# DESCRIPTION:
# Entropy feeds handling functions
Copyright (C) 2007 Fabio Erculiani
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
'''
from entropyConstants import *
from xml.dom import minidom
import time, datetime
feed_title = etpConst['systemname']+" Online Repository Status"
feed_description = "Keep you updated on what's going on in the Official "+etpConst['systemname']+" Repository."
feed_language = "en-EN"
feed_editor = etpConst['systemname']+" development team"
feed_copyright = etpConst['systemname']+" (C) 2007-2009"
class rssFeed:
def __init__(self, filename):
self.file = filename
self.items = {}
self.itemscounter = 0
if not os.path.isfile(self.file):
self.title = feed_title
self.description = feed_description
self.language = feed_language
self.copyright = feed_copyright
self.editor = feed_editor
self.link = etpConst['rss-website-url']
f = open(self.file,"w")
f.write('')
f.close()
else:
# parse file
self.xmldoc = minidom.parse(self.file)
self.rssdoc = self.xmldoc.getElementsByTagName("rss")[0]
self.channel = self.rssdoc.getElementsByTagName("channel")[0]
self.title = self.channel.getElementsByTagName("title")[0].firstChild.data
self.link = self.channel.getElementsByTagName("link")[0].firstChild.data
self.description = self.channel.getElementsByTagName("description")[0].firstChild.data
self.language = self.channel.getElementsByTagName("language")[0].firstChild.data
self.copyright = self.channel.getElementsByTagName("copyright")[0].firstChild.data
self.editor = self.channel.getElementsByTagName("managingEditor")[0].firstChild.data
entries = self.channel.getElementsByTagName("item")
self.itemscounter = len(entries)
mycounter = self.itemscounter
for item in entries:
mycounter -= 1
self.items[mycounter] = {}
self.items[mycounter]['title'] = item.getElementsByTagName("title")[0].firstChild.data
description = item.getElementsByTagName("description")[0].firstChild
if description:
self.items[mycounter]['description'] = description.data
else:
self.items[mycounter]['description'] = ""
link = item.getElementsByTagName("link")[0].firstChild
if link:
self.items[mycounter]['link'] = link.data
else:
self.items[mycounter]['link'] = ""
self.items[mycounter]['guid'] = item.getElementsByTagName("guid")[0].firstChild.data
self.items[mycounter]['pubDate'] = item.getElementsByTagName("pubDate")[0].firstChild.data
def addItem(self, title, link = '', description = ''):
self.itemscounter += 1
self.items[self.itemscounter] = {}
self.items[self.itemscounter]['title'] = title
self.items[self.itemscounter]['pubDate'] = time.strftime("%a, %d %b %Y %X +0000")
self.items[self.itemscounter]['description'] = description
self.items[self.itemscounter]['link'] = link
self.items[self.itemscounter]['guid'] = str(self.itemscounter)
return self.itemscounter
def removeEntry(self, id):
del self.items[id]
self.itemscounter -= 1
return len(self.itemscounter)
def getEntries(self):
return self.items, self.itemscounter
def writeChanges(self):
doc = minidom.Document()
rss = doc.createElement("rss")
rss.setAttribute("version","2.0")
channel = doc.createElement("channel")
# title
title = doc.createElement("title")
title_text = doc.createTextNode(unicode(self.title))
title.appendChild(title_text)
channel.appendChild(title)
# link
link = doc.createElement("link")
link_text = doc.createTextNode(unicode(self.link))
link.appendChild(link_text)
channel.appendChild(link)
# description
description = doc.createElement("description")
desc_text = doc.createTextNode(unicode(self.description))
description.appendChild(desc_text)
channel.appendChild(description)
# language
language = doc.createElement("language")
lang_text = doc.createTextNode(unicode(self.language))
language.appendChild(lang_text)
channel.appendChild(language)
# copyright
copyright = doc.createElement("copyright")
cr_text = doc.createTextNode(unicode(self.copyright))
copyright.appendChild(cr_text)
channel.appendChild(copyright)
# managingEditor
managingEditor = doc.createElement("managingEditor")
ed_text = doc.createTextNode(unicode(self.editor))
managingEditor.appendChild(ed_text)
channel.appendChild(managingEditor)
keys = self.items.keys()
keys.reverse()
for key in keys:
# item
item = doc.createElement("item")
# title
item_title = doc.createElement("title")
item_title_text = doc.createTextNode(unicode(self.items[key]['title']))
item_title.appendChild(item_title_text)
item.appendChild(item_title)
# link
item_link = doc.createElement("link")
item_link_text = doc.createTextNode(unicode(self.items[key]['link']))
item_link.appendChild(item_link_text)
item.appendChild(item_link)
# guid
item_guid = doc.createElement("guid")
item_guid.setAttribute("isPermaLink","false")
item_guid_text = doc.createTextNode(unicode(self.items[key]['guid']))
item_guid.appendChild(item_guid_text)
item.appendChild(item_guid)
# description
item_desc = doc.createElement("description")
item_desc_text = doc.createTextNode(unicode(self.items[key]['description']))
item_desc.appendChild(item_desc_text)
item.appendChild(item_desc)
# pubdate
item_date = doc.createElement("pubDate")
item_date_text = doc.createTextNode(unicode(self.items[key]['pubDate']))
item_date.appendChild(item_date_text)
item.appendChild(item_date)
# add item to channel
channel.appendChild(item)
# add channel to rss
rss.appendChild(channel)
doc.appendChild(rss)
f = open(self.file,"w")
f.writelines(doc.toprettyxml(indent="\t"))
f.flush()
f.close()

View File

@@ -84,6 +84,20 @@ if (os.path.isfile(etpConst['reagentconf'])):
print "WARNING: invalid loglevel in: "+etpConst['reagentconf']
import time
time.sleep(5)
elif line.startswith("rss-feed|") and (len(line.split("rss-feed|")) == 2):
feed = line.split("rss-feed|")[1]
if feed in ("enable","enabled","true","1"):
etpConst['rss-feed'] = True
elif feed in ("disable","disabled","false","0"):
etpConst['rss-feed'] = False
elif line.startswith("rss-name|") and (len(line.split("rss-name|")) == 2):
feedname = line.split("rss-name|")[1]
etpConst['rss-name'] = feedname
elif line.startswith("rss-base-url|") and (len(line.split("rss-base-url|")) == 2):
etpConst['rss-base-url'] = line.split("rss-base-url|")[1]
elif line.startswith("rss-website-url|") and (len(line.split("rss-website-url|")) == 2):
etpConst['rss-website-url'] = line.split("rss-website-url|")[1]
# mirrors section
if (os.path.isfile(etpConst['mirrorsconf'])):