entropy.misc

1 # -*- coding: utf-8 -*- 2 """ 3 # DESCRIPTION: 4 # Entropy Object Oriented Interface 5 6 Copyright (C) 2007-2009 Fabio Erculiani 7 8 This program is free software; you can redistribute it and/or modify 9 it under the terms of the GNU General Public License as published by 10 the Free Software Foundation; either version 2 of the License, or 11 (at your option) any later version. 12 13 This program is distributed in the hope that it will be useful, 14 but WITHOUT ANY WARRANTY; without even the implied warranty of 15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 GNU General Public License for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with this program; if not, write to the Free Software 20 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 """ 22 # pylink ok - doc 23 from __future__ import with_statement 24 import os 25 import sys 26 import time 27 import urllib2 28 import threading 29 from entropy.const import etpConst, etpUi 30 from entropy.core import SystemSettings 31

32 -class Lifo:

33

34 - def __init__(self):

35 self.__buf = {}

36

37 - def push(self, item):

38 try: 39 idx = max(self.__buf)+1 40 except ValueError: 41 idx = 0 42 self.__buf[idx] = item

43

44 - def clear(self):

45 self.__buf.clear()

46

47 - def is_filled(self):

48 if self.__buf: 49 return True 50 return False

51

52 - def discard(self, entry):

53 for key, buf_entry in self.__buf.items(): 54 # identity is generally faster, so try 55 # this first 56 if entry is buf_entry: 57 self.__buf.pop(key) 58 continue 59 if entry == buf_entry: 60 self.__buf.pop(key) 61 continue

62

63 - def pop(self):

64 try: 65 idx = max(self.__buf) 66 except (ValueError, TypeError,): 67 return None 68 return self.__buf.pop(idx)

69

70 -class TimeScheduled(threading.Thread):

71

72 - def __init__(self, delay, *args, **kwargs):

73 threading.Thread.__init__(self) 74 self.__f = args[0] 75 self.__delay = delay 76 self.__args = args[1:][:] 77 self.__kwargs = kwargs.copy() 78 # never enable this by default 79 # otherwise kill() and thread 80 # check will hang until 81 # time.sleep() is done 82 self.__accurate = False 83 self.__delay_before = False 84 self.__alive = 0

85

86 - def set_delay(self, delay):

87 self.__delay = delay

88

89 - def set_delay_before(self, bool_do):

90 self.__delay_before = bool(bool_do)

91

92 - def set_accuracy(self, bool_do):

93 self.__accurate = bool(bool_do)

94

95 - def run(self):

96 self.__alive = 1 97 while self.__alive: 98 99 if self.__delay_before: 100 do_break = self.__do_delay() 101 if do_break: 102 break 103 104 if self.__f == None: 105 break 106 self.__f(*self.__args, **self.__kwargs) 107 108 if not self.__delay_before: 109 do_break = self.__do_delay() 110 if do_break: 111 break

112 113

114 - def __do_delay(self):

115 116 if not self.__accurate: 117 118 if float == None: 119 return True 120 mydelay = float(self.__delay) 121 t_frac = 0.3 122 while mydelay > 0.0: 123 if not self.__alive: 124 return True 125 if time == None: 126 return True # shut down? 127 time.sleep(t_frac) 128 mydelay -= t_frac 129 130 else: 131 132 if time == None: 133 return True # shut down? 134 time.sleep(self.__delay) 135 136 return False

137

138 - def kill(self):

139 self.__alive = 0

140

141 -class ParallelTask(threading.Thread):

142

143 - def __init__(self, *args, **kwargs):

144 threading.Thread.__init__(self) 145 self.__function = args[0] 146 self.__args = args[1:][:] 147 self.__kwargs = kwargs.copy() 148 self.__rc = None

149

150 - def run(self):

151 self.__rc = self.__function(*self.__args, **self.__kwargs)

152

153 - def get_function(self):

154 return self.__function

155

156 - def get_rc(self):

157 return self.__rc

158

159 -class EmailSender:

160

161 - def __init__(self):

162 import smtplib 163 self.smtplib = smtplib 164 from email.mime.audio import MIMEAudio 165 from email.mime.image import MIMEImage 166 from email.mime.text import MIMEText 167 from email.mime.base import MIMEBase 168 from email.mime.multipart import MIMEMultipart 169 from email import encoders 170 from email.message import Message 171 import mimetypes 172 self.smtpuser = None 173 self.smtppassword = None 174 self.smtphost = 'localhost' 175 self.smtpport = 25 176 self.text = MIMEText 177 self.mimefile = MIMEBase 178 self.audio = MIMEAudio 179 self.image = MIMEImage 180 self.multipart = MIMEMultipart 181 self.default_sender = self.smtp_send 182 self.mimetypes = mimetypes 183 self.encoders = encoders 184 self.message = Message

185

186 - def smtp_send(self, sender, destinations, message):

187 s_srv = self.smtplib.SMTP(self.smtphost, self.smtpport) 188 if self.smtpuser and self.smtppassword: 189 s_srv.login(self.smtpuser, self.smtppassword) 190 s_srv.sendmail(sender, destinations, message) 191 s_srv.quit()

192

193 - def send_text_email(self, sender_email, destination_emails, subject, 194 content):

195 196 # Create a text/plain message 197 if isinstance(content, unicode): 198 content = content.encode('utf-8') 199 if isinstance(subject, unicode): 200 subject = subject.encode('utf-8') 201 202 msg = self.text(content) 203 msg['Subject'] = subject 204 msg['From'] = sender_email 205 msg['To'] = ', '.join(destination_emails) 206 return self.default_sender(sender_email, destination_emails, 207 msg.as_string())

208

209 - def send_mime_email(self, sender_email, destination_emails, subject, 210 content, files):

211 212 outer = self.multipart() 213 outer['Subject'] = subject 214 outer['From'] = sender_email 215 outer['To'] = ', '.join(destination_emails) 216 outer.preamble = subject 217 218 mymsg = self.text(content) 219 outer.attach(mymsg) 220 221 # attach files 222 for myfile in files: 223 if not (os.path.isfile(myfile) and os.access(myfile, os.R_OK)): 224 continue 225 226 ctype, encoding = self.mimetypes.guess_type(myfile) 227 if ctype is None or encoding is not None: 228 ctype = 'application/octet-stream' 229 maintype, subtype = ctype.split('/', 1) 230 231 if maintype == 'image': 232 img_f = open(myfile, 'rb') 233 msg = self.image(img_f.read(), _subtype = subtype) 234 img_f.close() 235 elif maintype == 'audio': 236 audio_f = open(myfile, 'rb') 237 msg = self.audio(audio_f.read(), _subtype = subtype) 238 audio_f.close() 239 else: 240 gen_f = open(myfile, 'rb') 241 msg = self.mimefile(maintype, subtype) 242 msg.set_payload(gen_f.read()) 243 gen_f.close() 244 self.encoders.encode_base64(msg) 245 246 msg.add_header('Content-Disposition', 'attachment', 247 filename = os.path.basename(myfile)) 248 outer.attach(msg) 249 250 composed = outer.as_string() 251 return self.default_sender(sender_email, destination_emails, composed)

252

253 -class EntropyGeoIP:

254 255 """ 256 Entropy geo-tagging interface containing useful 257 methods to ease metadata management and transfor- 258 mation. 259 It's a wrapper over GeoIP at the moment 260 dev-python/geoip-python required 261 """ 262

263 - def __init__(self, geoip_dbfile):

264 265 """ 266 @param1: valid GeoIP (Maxmind) database file (.dat) 267 """ 268 269 import GeoIP 270 self.__geoip = GeoIP 271 # http://www.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz 272 if not (os.path.isfile(geoip_dbfile) and \ 273 os.access(geoip_dbfile, os.R_OK)): 274 raise AttributeError( 275 "expecting a valid filepath for geoip_dbfile, got: %s" % ( 276 repr(geoip_dbfile), 277 ) 278 ) 279 self.__geoip_dbfile = geoip_dbfile

280

281 - def __get_geo_ip_generic(self):

282 return self.__geoip.new(self.__geoip.GEOIP_MEMORY_CACHE)

283

284 - def __get_geo_ip_open(self):

285 return self.__geoip.open(self.__geoip_dbfile, 286 self.__geoip.GEOIP_STANDARD)

287

288 - def get_geoip_country_name_from_ip(self, ip_address):

289 """ 290 @return: string or None 291 @param1: ip address string 292 """ 293 gi_a = self.__get_geo_ip_generic() 294 return gi_a.country_name_by_addr(ip_address)

295

296 - def get_geoip_country_code_from_ip(self, ip_address):

297 """ 298 @return: string or None 299 @param1: ip address string 300 """ 301 gi_a = self.__get_geo_ip_generic() 302 return gi_a.country_code_by_addr(ip_address)

303

304 - def get_geoip_record_from_ip(self, ip_address):

305 """ 306 @return: dict() or None 307 @param1: ip address string 308 dict data: 309 { 310 'city': 'Treviso', 311 'region': '20', 312 'area_code': 0, 313 'longitude': 12.244999885559082, 314 'country_code3': 'ITA', 315 'latitude': 45.666698455810547, 316 'postal_code': None, 317 'dma_code': 0, 318 'country_code': 'IT', 319 'country_name': 'Italy' 320 } 321 """ 322 go_a = self.__get_geo_ip_open() 323 return go_a.record_by_addr(ip_address)

324

325 - def get_geoip_record_from_hostname(self, hostname):

326 """ 327 @return: dict() or None 328 @param1: hostname 329 """ 330 go_a = self.__get_geo_ip_open() 331 return go_a.record_by_name(hostname)

332 333

334 -class RSS:

335 336 # this is a relative import to avoid circular deps 337 import tools as entropyTools

338 - def __init__(self, filename, title, description, maxentries = 100):

339 340 self.__system_settings = SystemSettings() 341 self.__feed_title = title 342 self.__feed_title = self.__feed_title.strip() 343 self.__feed_description = description 344 self.__feed_language = "en-EN" 345 self.__srv_settings_plugin_id = \ 346 etpConst['system_settings_plugins_ids']['server_plugin'] 347 srv_settings = self.__system_settings.get(self.__srv_settings_plugin_id) 348 if srv_settings is None: 349 self.__feed_editor = "N/A" 350 else: 351 self.__feed_editor = srv_settings['server']['rss']['editor'] 352 self.__feed_copyright = "%s - (C) %s" % ( 353 self.__system_settings['system']['name'], 354 self.entropyTools.get_year(), 355 ) 356 357 self.__file = filename 358 self.__items = {} 359 self.__itemscounter = 0 360 self.__maxentries = maxentries 361 from xml.dom import minidom 362 self.minidom = minidom 363 364 # sanity check 365 broken = False 366 if os.path.isfile(self.__file): 367 try: 368 self.xmldoc = self.minidom.parse(self.__file) 369 except: 370 broken = True 371 372 if not os.path.isfile(self.__file) or broken: 373 374 self.__title = self.__feed_title 375 self.__description = self.__feed_description 376 self.__language = self.__feed_language 377 self.__cright = self.__feed_copyright 378 self.__editor = self.__feed_editor 379 sys_set = self.__system_settings[self.__srv_settings_plugin_id] 380 srv_set = sys_set['server'] 381 self.__link = srv_set['rss']['website_url'] 382 rss_f = open(self.__file, "w") 383 rss_f.write('') 384 rss_f.flush() 385 rss_f.close() 386 387 else: 388 389 self.__rssdoc = self.xmldoc.getElementsByTagName("rss")[0] 390 self.__channel = self.__rssdoc.getElementsByTagName("channel")[0] 391 title_obj = self.__channel.getElementsByTagName("title")[0] 392 self.__title = title_obj.firstChild.data.strip() 393 link_obj = self.__channel.getElementsByTagName("link")[0] 394 self.__link = link_obj.firstChild.data.strip() 395 desc_obj = self.__channel.getElementsByTagName("description")[0] 396 description = desc_obj.firstChild 397 if hasattr(description, "data"): 398 self.__description = description.data.strip() 399 else: 400 self.__description = '' 401 try: 402 lang_obj = self.__channel.getElementsByTagName("language")[0] 403 self.__language = lang_obj.firstChild.data.strip() 404 except IndexError: 405 self.__language = 'en' 406 try: 407 cright_obj = self.__channel.getElementsByTagName("copyright")[0] 408 self.__cright = cright_obj.firstChild.data.strip() 409 except IndexError: 410 self.__cright = '' 411 try: 412 e_obj = self.__channel.getElementsByTagName("managingEditor")[0] 413 self.__editor = e_obj.firstChild.data.strip() 414 except IndexError: 415 self.__editor = '' 416 entries = self.__channel.getElementsByTagName("item") 417 self.__itemscounter = len(entries) 418 if self.__itemscounter > self.__maxentries: 419 self.__itemscounter = self.__maxentries 420 mycounter = self.__itemscounter 421 for item in entries: 422 if mycounter == 0: # max entries reached 423 break 424 mycounter -= 1 425 self.__items[mycounter] = {} 426 title_obj = item.getElementsByTagName("title")[0] 427 self.__items[mycounter]['title'] = \ 428 title_obj.firstChild.data.strip() 429 desc_obj = item.getElementsByTagName("description")[0] 430 description = desc_obj.firstChild 431 if description: 432 self.__items[mycounter]['description'] = \ 433 description.data.strip() 434 else: 435 self.__items[mycounter]['description'] = "" 436 437 link = item.getElementsByTagName("link")[0].firstChild 438 if link: 439 self.__items[mycounter]['link'] = link.data.strip() 440 else: 441 self.__items[mycounter]['link'] = "" 442 443 guid_obj = item.getElementsByTagName("guid")[0] 444 self.__items[mycounter]['guid'] = \ 445 guid_obj.firstChild.data.strip() 446 pub_date_obj = item.getElementsByTagName("pubDate")[0] 447 self.__items[mycounter]['pubDate'] = \ 448 pub_date_obj.firstChild.data.strip() 449 dcs = item.getElementsByTagName("dc:creator") 450 if dcs: 451 self.__items[mycounter]['dc:creator'] = \ 452 dcs[0].firstChild.data.strip()

453 454

455 - def add_item(self, title, link = '', description = '', pubDate = ''):

456 457 self.__itemscounter += 1 458 self.__items[self.__itemscounter] = {} 459 self.__items[self.__itemscounter]['title'] = title 460 if pubDate: 461 self.__items[self.__itemscounter]['pubDate'] = pubDate 462 else: 463 self.__items[self.__itemscounter]['pubDate'] = \ 464 time.strftime("%a, %d %b %Y %X +0000") 465 self.__items[self.__itemscounter]['description'] = description 466 self.__items[self.__itemscounter]['link'] = link 467 if link: 468 self.__items[self.__itemscounter]['guid'] = link 469 else: 470 myguid = self.__system_settings['system']['name'].lower() 471 myguid = myguid.replace(" ", "") 472 self.__items[self.__itemscounter]['guid'] = myguid+"~" + \ 473 description + str(self.__itemscounter) 474 return self.__itemscounter

475

476 - def remove_entry(self, key):

477 if key in self.__items: 478 del self.__items[key] 479 self.__itemscounter -= 1 480 return self.__itemscounter

481

482 - def get_entries(self):

483 return self.__items, self.__itemscounter

484

485 - def write_changes(self, reverse = True):

486 487 # filter entries to fit in maxentries 488 if self.__itemscounter > self.__maxentries: 489 tobefiltered = self.__itemscounter - self.__maxentries 490 for index in range(tobefiltered): 491 try: 492 del self.__items[index] 493 except KeyError: 494 pass 495 496 doc = self.minidom.Document() 497 498 rss = doc.createElement("rss") 499 rss.setAttribute("version", "2.0") 500 rss.setAttribute("xmlns:atom", "http://www.w3.org/2005/Atom") 501 502 channel = doc.createElement("channel") 503 504 # title 505 title = doc.createElement("title") 506 title_text = doc.createTextNode(unicode(self.__title)) 507 title.appendChild(title_text) 508 channel.appendChild(title) 509 # link 510 link = doc.createElement("link") 511 link_text = doc.createTextNode(unicode(self.__link)) 512 link.appendChild(link_text) 513 channel.appendChild(link) 514 # description 515 description = doc.createElement("description") 516 desc_text = doc.createTextNode(unicode(self.__description)) 517 description.appendChild(desc_text) 518 channel.appendChild(description) 519 # language 520 language = doc.createElement("language") 521 lang_text = doc.createTextNode(unicode(self.__language)) 522 language.appendChild(lang_text) 523 channel.appendChild(language) 524 # copyright 525 cright = doc.createElement("copyright") 526 cr_text = doc.createTextNode(unicode(self.__cright)) 527 cright.appendChild(cr_text) 528 channel.appendChild(cright) 529 # managingEditor 530 managing_editor = doc.createElement("managingEditor") 531 ed_text = doc.createTextNode(unicode(self.__editor)) 532 managing_editor.appendChild(ed_text) 533 channel.appendChild(managing_editor) 534 535 keys = self.__items.keys() 536 if reverse: 537 keys.reverse() 538 for key in keys: 539 540 # sanity check, you never know 541 if not self.__items.has_key(key): 542 self.remove_entry(key) 543 continue 544 k_error = False 545 for item in ('title', 'link', 'guid', 'description', 'pubDate',): 546 if not self.__items[key].has_key(item): 547 k_error = True 548 break 549 if k_error: 550 self.remove_entry(key) 551 continue 552 553 # item 554 item = doc.createElement("item") 555 # title 556 item_title = doc.createElement("title") 557 item_title_text = doc.createTextNode( 558 unicode(self.__items[key]['title'])) 559 item_title.appendChild(item_title_text) 560 item.appendChild(item_title) 561 # link 562 item_link = doc.createElement("link") 563 item_link_text = doc.createTextNode( 564 unicode(self.__items[key]['link'])) 565 item_link.appendChild(item_link_text) 566 item.appendChild(item_link) 567 # guid 568 item_guid = doc.createElement("guid") 569 item_guid.setAttribute("isPermaLink", "true") 570 item_guid_text = doc.createTextNode( 571 unicode(self.__items[key]['guid'])) 572 item_guid.appendChild(item_guid_text) 573 item.appendChild(item_guid) 574 # description 575 item_desc = doc.createElement("description") 576 item_desc_text = doc.createTextNode( 577 unicode(self.__items[key]['description'])) 578 item_desc.appendChild(item_desc_text) 579 item.appendChild(item_desc) 580 # pubdate 581 item_date = doc.createElement("pubDate") 582 item_date_text = doc.createTextNode( 583 unicode(self.__items[key]['pubDate'])) 584 item_date.appendChild(item_date_text) 585 item.appendChild(item_date) 586 587 # add item to channel 588 channel.appendChild(item) 589 590 # add channel to rss 591 rss.appendChild(channel) 592 doc.appendChild(rss) 593 rss_f = open(self.__file, "w") 594 rss_f.writelines(doc.toprettyxml(indent=" ").encode('utf-8')) 595 rss_f.flush() 596 rss_f.close()

597

598 -class LogFile:

599

600 - def __init__(self, level = 0, filename = None, header = "[LOG]"):

601 self.handler = self.default_handler 602 self.level = level 603 self.header = header 604 self.__logfile = None 605 self.open(filename) 606 self.__filename = filename

607

608 - def __del__(self):

609 self.close()

610

611 - def close(self):

612 try: 613 self.__logfile.close() 614 except (IOError, OSError,): 615 pass

616

617 - def get_fpath(self):

618 return self.__filename

619

620 - def flush(self):

621 self.__logfile.flush()

622

623 - def fileno(self):

624 return self.__get_file()

625

626 - def isatty(self):

627 return False

628

629 - def read(self, *args):

630 return ''

631

632 - def readline(self):

633 return ''

634

635 - def readlines(self):

636 return []

637

638 - def seek(self, offset):

639 return self.__logfile.seek(offset)

640

641 - def tell(self):

642 return self.__logfile.tell()

643

644 - def truncate(self):

645 return self.__logfile.truncate()

646

647 - def open(self, file_path = None):

648 649 if isinstance(file_path, basestring): 650 if not os.access(file_path, os.F_OK) and os.access( 651 os.path.dirname(file_path), os.W_OK): 652 self.__logfile = open(file_path, "aw") 653 else: 654 if os.access(file_path, os.W_OK | os.F_OK): 655 self.__logfile = open(file_path, "aw") 656 else: 657 self.__logfile = open("/dev/null", "aw") 658 elif hasattr(file_path, 'write'): 659 self.__logfile = file_path 660 else: 661 self.__logfile = sys.stderr

662

663 - def __get_file(self):

664 return self.__logfile.fileno()

665

666 - def __call__(self, format, *args):

667 self.handler (format % args)

668

669 - def default_handler (self, mystr):

670 try: 671 self.__logfile.write ("* %s\n" % (mystr)) 672 except UnicodeEncodeError: 673 self.__logfile.write ("* %s\n" % (mystr.encode('utf-8'),)) 674 self.__logfile.flush()

675

676 - def set_loglevel(self, level):

677 self.level = level

678

679 - def log(self, messagetype, level, message):

680 if self.level >= level and not etpUi['nolog']: 681 self.handler("%s %s %s %s" % (self.__get_header(), 682 messagetype, self.header, message,))

683

684 - def write(self, line):

685 self.handler(line)

686

687 - def writelines(self, lst):

688 for line in lst: 689 self.write(line)

690

691 - def __get_header(self):

692 return time.strftime('[%H:%M:%S %d/%m/%Y %Z]')

693

694 -class Callable:

695 - def __init__(self, anycallable):

696 self.__call__ = anycallable

697

698 -class MultipartPostHandler(urllib2.BaseHandler):

699 700 handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first 701

702 - def __init__(self):

703 pass

704

705 - def http_request(self, request):

706 707 import urllib 708 doseq = 1 709 710 data = request.get_data() 711 if data is not None and type(data) != str: 712 v_files = [] 713 v_vars = [] 714 try: 715 for (key, value) in data.items(): 716 if type(value) == file: 717 v_files.append((key, value)) 718 else: 719 v_vars.append((key, value)) 720 except TypeError: 721 raise TypeError, "not a valid non-string sequence" \ 722 " or mapping object" 723 724 if len(v_files) == 0: 725 data = urllib.urlencode(v_vars, doseq) 726 else: 727 boundary, data = self.multipart_encode(v_vars, v_files) 728 contenttype = 'multipart/form-data; boundary=%s' % boundary 729 request.add_unredirected_header('Content-Type', contenttype) 730 731 request.add_data(data) 732 return request

733

734 - def multipart_encode(self, myvars, files, boundary = None, buf = None):

735 736 from cStringIO import StringIO 737 import mimetools, mimetypes 738 #import stat 739 740 if boundary is None: 741 boundary = mimetools.choose_boundary() 742 if buf is None: 743 buf = StringIO() 744 for(key, value) in myvars: 745 buf.write('--%s\r\n' % boundary) 746 buf.write('Content-Disposition: form-data; name="%s"' % key) 747 buf.write('\r\n\r\n' + value + '\r\n') 748 for(key, fdesc) in files: 749 #file_size = os.fstat(fdesc.fileno())[stat.ST_SIZE] 750 filename = fdesc.name.split('/')[-1] 751 contenttype = mimetypes.guess_type(filename)[0] or \ 752 'application/octet-stream' 753 buf.write('--%s\r\n' % boundary) 754 buf.write('Content-Disposition: form-data; name="%s"; ' \ 755 'filename="%s"\r\n' % (key, filename)) 756 buf.write('Content-Type: %s\r\n' % contenttype) 757 # buffer += 'Content-Length: %s\r\n' % file_size 758 fdesc.seek(0) 759 buf.write('\r\n' + fdesc.read() + '\r\n') 760 buf.write('--' + boundary + '--\r\n\r\n') 761 buf = buf.getvalue() 762 return boundary, buf

763 764 multipart_encode = Callable(multipart_encode) 765 766 https_request = http_request

767

Source Code for Module entropy.misc