entropy.misc

1 # -*- coding: utf-8 -*- 2 """ 3 4 @author: Fabio Erculiani <lxnay@sabayonlinux.org> 5 @contact: lxnay@sabayonlinux.org 6 @copyright: Fabio Erculiani 7 @license: GPL-2 8 9 B{Entropy Framework miscellaneous module}. 10 11 12 13 """ 14 # pylink ok - doc 15 from __future__ import with_statement 16 import os 17 import sys 18 import time 19 import urllib2 20 import threading 21 from entropy.const import etpConst, etpUi 22 from entropy.core import SystemSettings 23

24 -class Lifo:

25

26 - def __init__(self):

27 self.__buf = {}

28

29 - def push(self, item):

30 try: 31 idx = max(self.__buf)+1 32 except ValueError: 33 idx = 0 34 self.__buf[idx] = item

35

36 - def clear(self):

37 self.__buf.clear()

38

39 - def is_filled(self):

40 if self.__buf: 41 return True 42 return False

43

44 - def discard(self, entry):

45 for key, buf_entry in self.__buf.items(): 46 # identity is generally faster, so try 47 # this first 48 if entry is buf_entry: 49 self.__buf.pop(key) 50 continue 51 if entry == buf_entry: 52 self.__buf.pop(key) 53 continue

54

55 - def pop(self):

56 try: 57 idx = max(self.__buf) 58 except (ValueError, TypeError,): 59 return None 60 return self.__buf.pop(idx)

61

62 -class TimeScheduled(threading.Thread):

63

64 - def __init__(self, delay, *args, **kwargs):

65 threading.Thread.__init__(self) 66 self.__f = args[0] 67 self.__delay = delay 68 self.__args = args[1:][:] 69 self.__kwargs = kwargs.copy() 70 # never enable this by default 71 # otherwise kill() and thread 72 # check will hang until 73 # time.sleep() is done 74 self.__accurate = False 75 self.__delay_before = False 76 self.__alive = 0

77

78 - def set_delay(self, delay):

79 self.__delay = delay

80

81 - def set_delay_before(self, bool_do):

82 self.__delay_before = bool(bool_do)

83

84 - def set_accuracy(self, bool_do):

85 self.__accurate = bool(bool_do)

86

87 - def run(self):

88 self.__alive = 1 89 while self.__alive: 90 91 if self.__delay_before: 92 do_break = self.__do_delay() 93 if do_break: 94 break 95 96 if self.__f == None: 97 break 98 self.__f(*self.__args, **self.__kwargs) 99 100 if not self.__delay_before: 101 do_break = self.__do_delay() 102 if do_break: 103 break

104 105

106 - def __do_delay(self):

107 108 if not self.__accurate: 109 110 if float == None: 111 return True 112 mydelay = float(self.__delay) 113 t_frac = 0.3 114 while mydelay > 0.0: 115 if not self.__alive: 116 return True 117 if time == None: 118 return True # shut down? 119 time.sleep(t_frac) 120 mydelay -= t_frac 121 122 else: 123 124 if time == None: 125 return True # shut down? 126 time.sleep(self.__delay) 127 128 return False

129

130 - def kill(self):

131 self.__alive = 0

132

133 -class ParallelTask(threading.Thread):

134

135 - def __init__(self, *args, **kwargs):

136 threading.Thread.__init__(self) 137 self.__function = args[0] 138 self.__args = args[1:][:] 139 self.__kwargs = kwargs.copy() 140 self.__rc = None

141

142 - def run(self):

143 self.__rc = self.__function(*self.__args, **self.__kwargs)

144

145 - def get_function(self):

146 return self.__function

147

148 - def get_rc(self):

149 return self.__rc

150

151 -class EmailSender:

152

153 - def __init__(self):

154 import smtplib 155 self.smtplib = smtplib 156 from email.mime.audio import MIMEAudio 157 from email.mime.image import MIMEImage 158 from email.mime.text import MIMEText 159 from email.mime.base import MIMEBase 160 from email.mime.multipart import MIMEMultipart 161 from email import encoders 162 from email.message import Message 163 import mimetypes 164 self.smtpuser = None 165 self.smtppassword = None 166 self.smtphost = 'localhost' 167 self.smtpport = 25 168 self.text = MIMEText 169 self.mimefile = MIMEBase 170 self.audio = MIMEAudio 171 self.image = MIMEImage 172 self.multipart = MIMEMultipart 173 self.default_sender = self.smtp_send 174 self.mimetypes = mimetypes 175 self.encoders = encoders 176 self.message = Message

177

178 - def smtp_send(self, sender, destinations, message):

179 s_srv = self.smtplib.SMTP(self.smtphost, self.smtpport) 180 if self.smtpuser and self.smtppassword: 181 s_srv.login(self.smtpuser, self.smtppassword) 182 s_srv.sendmail(sender, destinations, message) 183 s_srv.quit()

184

185 - def send_text_email(self, sender_email, destination_emails, subject, 186 content):

187 188 # Create a text/plain message 189 if isinstance(content, unicode): 190 content = content.encode('utf-8') 191 if isinstance(subject, unicode): 192 subject = subject.encode('utf-8') 193 194 msg = self.text(content) 195 msg['Subject'] = subject 196 msg['From'] = sender_email 197 msg['To'] = ', '.join(destination_emails) 198 return self.default_sender(sender_email, destination_emails, 199 msg.as_string())

200

201 - def send_mime_email(self, sender_email, destination_emails, subject, 202 content, files):

203 204 outer = self.multipart() 205 outer['Subject'] = subject 206 outer['From'] = sender_email 207 outer['To'] = ', '.join(destination_emails) 208 outer.preamble = subject 209 210 mymsg = self.text(content) 211 outer.attach(mymsg) 212 213 # attach files 214 for myfile in files: 215 if not (os.path.isfile(myfile) and os.access(myfile, os.R_OK)): 216 continue 217 218 ctype, encoding = self.mimetypes.guess_type(myfile) 219 if ctype is None or encoding is not None: 220 ctype = 'application/octet-stream' 221 maintype, subtype = ctype.split('/', 1) 222 223 if maintype == 'image': 224 img_f = open(myfile, 'rb') 225 msg = self.image(img_f.read(), _subtype = subtype) 226 img_f.close() 227 elif maintype == 'audio': 228 audio_f = open(myfile, 'rb') 229 msg = self.audio(audio_f.read(), _subtype = subtype) 230 audio_f.close() 231 else: 232 gen_f = open(myfile, 'rb') 233 msg = self.mimefile(maintype, subtype) 234 msg.set_payload(gen_f.read()) 235 gen_f.close() 236 self.encoders.encode_base64(msg) 237 238 msg.add_header('Content-Disposition', 'attachment', 239 filename = os.path.basename(myfile)) 240 outer.attach(msg) 241 242 composed = outer.as_string() 243 return self.default_sender(sender_email, destination_emails, composed)

244

245 -class EntropyGeoIP:

246 247 """ 248 Entropy geo-tagging interface containing useful 249 methods to ease metadata management and transfor- 250 mation. 251 It's a wrapper over GeoIP at the moment 252 dev-python/geoip-python required 253 """ 254

255 - def __init__(self, geoip_dbfile):

256 257 """ 258 @param1: valid GeoIP (Maxmind) database file (.dat) 259 """ 260 261 import GeoIP 262 self.__geoip = GeoIP 263 # http://www.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz 264 if not (os.path.isfile(geoip_dbfile) and \ 265 os.access(geoip_dbfile, os.R_OK)): 266 raise AttributeError( 267 "expecting a valid filepath for geoip_dbfile, got: %s" % ( 268 repr(geoip_dbfile), 269 ) 270 ) 271 self.__geoip_dbfile = geoip_dbfile

272

273 - def __get_geo_ip_generic(self):

274 return self.__geoip.new(self.__geoip.GEOIP_MEMORY_CACHE)

275

276 - def __get_geo_ip_open(self):

277 return self.__geoip.open(self.__geoip_dbfile, 278 self.__geoip.GEOIP_STANDARD)

279

280 - def get_geoip_country_name_from_ip(self, ip_address):

281 """ 282 @return: string or None 283 @param1: ip address string 284 """ 285 gi_a = self.__get_geo_ip_generic() 286 return gi_a.country_name_by_addr(ip_address)

287

288 - def get_geoip_country_code_from_ip(self, ip_address):

289 """ 290 @return: string or None 291 @param1: ip address string 292 """ 293 gi_a = self.__get_geo_ip_generic() 294 return gi_a.country_code_by_addr(ip_address)

295

296 - def get_geoip_record_from_ip(self, ip_address):

297 """ 298 @return: dict() or None 299 @param1: ip address string 300 dict data: 301 { 302 'city': 'Treviso', 303 'region': '20', 304 'area_code': 0, 305 'longitude': 12.244999885559082, 306 'country_code3': 'ITA', 307 'latitude': 45.666698455810547, 308 'postal_code': None, 309 'dma_code': 0, 310 'country_code': 'IT', 311 'country_name': 'Italy' 312 } 313 """ 314 go_a = self.__get_geo_ip_open() 315 return go_a.record_by_addr(ip_address)

316

317 - def get_geoip_record_from_hostname(self, hostname):

318 """ 319 @return: dict() or None 320 @param1: hostname 321 """ 322 go_a = self.__get_geo_ip_open() 323 return go_a.record_by_name(hostname)

324 325

326 -class RSS:

327 328 # this is a relative import to avoid circular deps 329 import tools as entropyTools

330 - def __init__(self, filename, title, description, maxentries = 100):

331 332 self.__system_settings = SystemSettings() 333 self.__feed_title = title 334 self.__feed_title = self.__feed_title.strip() 335 self.__feed_description = description 336 self.__feed_language = "en-EN" 337 self.__srv_settings_plugin_id = \ 338 etpConst['system_settings_plugins_ids']['server_plugin'] 339 srv_settings = self.__system_settings.get(self.__srv_settings_plugin_id) 340 if srv_settings is None: 341 self.__feed_editor = "N/A" 342 else: 343 self.__feed_editor = srv_settings['server']['rss']['editor'] 344 self.__feed_copyright = "%s - (C) %s" % ( 345 self.__system_settings['system']['name'], 346 self.entropyTools.get_year(), 347 ) 348 349 self.__file = filename 350 self.__items = {} 351 self.__itemscounter = 0 352 self.__maxentries = maxentries 353 from xml.dom import minidom 354 self.minidom = minidom 355 356 # sanity check 357 broken = False 358 if os.path.isfile(self.__file): 359 try: 360 self.xmldoc = self.minidom.parse(self.__file) 361 except: 362 broken = True 363 364 if not os.path.isfile(self.__file) or broken: 365 366 self.__title = self.__feed_title 367 self.__description = self.__feed_description 368 self.__language = self.__feed_language 369 self.__cright = self.__feed_copyright 370 self.__editor = self.__feed_editor 371 sys_set = self.__system_settings.get(self.__srv_settings_plugin_id) 372 if sys_set is None: 373 self.__link = etpConst['rss-website-url'] 374 else: 375 srv_set = sys_set['server'] 376 self.__link = srv_set['rss']['website_url'] 377 rss_f = open(self.__file, "w") 378 rss_f.write('') 379 rss_f.flush() 380 rss_f.close() 381 382 else: 383 384 self.__rssdoc = self.xmldoc.getElementsByTagName("rss")[0] 385 self.__channel = self.__rssdoc.getElementsByTagName("channel")[0] 386 title_obj = self.__channel.getElementsByTagName("title")[0] 387 self.__title = title_obj.firstChild.data.strip() 388 link_obj = self.__channel.getElementsByTagName("link")[0] 389 self.__link = link_obj.firstChild.data.strip() 390 desc_obj = self.__channel.getElementsByTagName("description")[0] 391 description = desc_obj.firstChild 392 if hasattr(description, "data"): 393 self.__description = description.data.strip() 394 else: 395 self.__description = '' 396 try: 397 lang_obj = self.__channel.getElementsByTagName("language")[0] 398 self.__language = lang_obj.firstChild.data.strip() 399 except IndexError: 400 self.__language = 'en' 401 try: 402 cright_obj = self.__channel.getElementsByTagName("copyright")[0] 403 self.__cright = cright_obj.firstChild.data.strip() 404 except IndexError: 405 self.__cright = '' 406 try: 407 e_obj = self.__channel.getElementsByTagName("managingEditor")[0] 408 self.__editor = e_obj.firstChild.data.strip() 409 except IndexError: 410 self.__editor = '' 411 entries = self.__channel.getElementsByTagName("item") 412 self.__itemscounter = len(entries) 413 if self.__itemscounter > self.__maxentries: 414 self.__itemscounter = self.__maxentries 415 mycounter = self.__itemscounter 416 for item in entries: 417 if mycounter == 0: # max entries reached 418 break 419 mycounter -= 1 420 self.__items[mycounter] = {} 421 title_obj = item.getElementsByTagName("title")[0] 422 self.__items[mycounter]['title'] = \ 423 title_obj.firstChild.data.strip() 424 desc_obj = item.getElementsByTagName("description") 425 description = None 426 if desc_obj: 427 description = desc_obj[0].firstChild 428 if description: 429 self.__items[mycounter]['description'] = \ 430 description.data.strip() 431 else: 432 self.__items[mycounter]['description'] = "" 433 434 link = item.getElementsByTagName("link")[0].firstChild 435 if link: 436 self.__items[mycounter]['link'] = link.data.strip() 437 else: 438 self.__items[mycounter]['link'] = "" 439 440 guid_obj = item.getElementsByTagName("guid")[0] 441 self.__items[mycounter]['guid'] = \ 442 guid_obj.firstChild.data.strip() 443 pub_date_obj = item.getElementsByTagName("pubDate")[0] 444 self.__items[mycounter]['pubDate'] = \ 445 pub_date_obj.firstChild.data.strip() 446 dcs = item.getElementsByTagName("dc:creator") 447 if dcs: 448 self.__items[mycounter]['dc:creator'] = \ 449 dcs[0].firstChild.data.strip()

450 451

452 - def add_item(self, title, link = '', description = '', pubDate = ''):

453 454 self.__itemscounter += 1 455 self.__items[self.__itemscounter] = {} 456 self.__items[self.__itemscounter]['title'] = title 457 if pubDate: 458 self.__items[self.__itemscounter]['pubDate'] = pubDate 459 else: 460 self.__items[self.__itemscounter]['pubDate'] = \ 461 time.strftime("%a, %d %b %Y %X +0000") 462 self.__items[self.__itemscounter]['description'] = description 463 self.__items[self.__itemscounter]['link'] = link 464 if link: 465 self.__items[self.__itemscounter]['guid'] = link 466 else: 467 myguid = self.__system_settings['system']['name'].lower() 468 myguid = myguid.replace(" ", "") 469 self.__items[self.__itemscounter]['guid'] = myguid+"~" + \ 470 description + str(self.__itemscounter) 471 return self.__itemscounter

472

473 - def remove_entry(self, key):

474 if key in self.__items: 475 del self.__items[key] 476 self.__itemscounter -= 1 477 return self.__itemscounter

478

479 - def get_entries(self):

480 return self.__items, self.__itemscounter

481

482 - def write_changes(self, reverse = True):

483 484 # filter entries to fit in maxentries 485 if self.__itemscounter > self.__maxentries: 486 tobefiltered = self.__itemscounter - self.__maxentries 487 for index in range(tobefiltered): 488 try: 489 del self.__items[index] 490 except KeyError: 491 pass 492 493 doc = self.minidom.Document() 494 495 rss = doc.createElement("rss") 496 rss.setAttribute("version", "2.0") 497 rss.setAttribute("xmlns:atom", "http://www.w3.org/2005/Atom") 498 499 channel = doc.createElement("channel") 500 501 # title 502 title = doc.createElement("title") 503 title_text = doc.createTextNode(unicode(self.__title)) 504 title.appendChild(title_text) 505 channel.appendChild(title) 506 # link 507 link = doc.createElement("link") 508 link_text = doc.createTextNode(unicode(self.__link)) 509 link.appendChild(link_text) 510 channel.appendChild(link) 511 # description 512 description = doc.createElement("description") 513 desc_text = doc.createTextNode(unicode(self.__description)) 514 description.appendChild(desc_text) 515 channel.appendChild(description) 516 # language 517 language = doc.createElement("language") 518 lang_text = doc.createTextNode(unicode(self.__language)) 519 language.appendChild(lang_text) 520 channel.appendChild(language) 521 # copyright 522 cright = doc.createElement("copyright") 523 cr_text = doc.createTextNode(unicode(self.__cright)) 524 cright.appendChild(cr_text) 525 channel.appendChild(cright) 526 # managingEditor 527 managing_editor = doc.createElement("managingEditor") 528 ed_text = doc.createTextNode(unicode(self.__editor)) 529 managing_editor.appendChild(ed_text) 530 channel.appendChild(managing_editor) 531 532 keys = self.__items.keys() 533 if reverse: 534 keys.reverse() 535 for key in keys: 536 537 # sanity check, you never know 538 if not self.__items.has_key(key): 539 self.remove_entry(key) 540 continue 541 k_error = False 542 for item in ('title', 'link', 'guid', 'description', 'pubDate',): 543 if not self.__items[key].has_key(item): 544 k_error = True 545 break 546 if k_error: 547 self.remove_entry(key) 548 continue 549 550 # item 551 item = doc.createElement("item") 552 # title 553 item_title = doc.createElement("title") 554 item_title_text = doc.createTextNode( 555 unicode(self.__items[key]['title'])) 556 item_title.appendChild(item_title_text) 557 item.appendChild(item_title) 558 # link 559 item_link = doc.createElement("link") 560 item_link_text = doc.createTextNode( 561 unicode(self.__items[key]['link'])) 562 item_link.appendChild(item_link_text) 563 item.appendChild(item_link) 564 # guid 565 item_guid = doc.createElement("guid") 566 item_guid.setAttribute("isPermaLink", "true") 567 item_guid_text = doc.createTextNode( 568 unicode(self.__items[key]['guid'])) 569 item_guid.appendChild(item_guid_text) 570 item.appendChild(item_guid) 571 # description 572 item_desc = doc.createElement("description") 573 item_desc_text = doc.createTextNode( 574 unicode(self.__items[key]['description'])) 575 item_desc.appendChild(item_desc_text) 576 item.appendChild(item_desc) 577 # pubdate 578 item_date = doc.createElement("pubDate") 579 item_date_text = doc.createTextNode( 580 unicode(self.__items[key]['pubDate'])) 581 item_date.appendChild(item_date_text) 582 item.appendChild(item_date) 583 584 # add item to channel 585 channel.appendChild(item) 586 587 # add channel to rss 588 rss.appendChild(channel) 589 doc.appendChild(rss) 590 rss_f = open(self.__file, "w") 591 rss_f.writelines(doc.toprettyxml(indent=" ").encode('utf-8')) 592 rss_f.flush() 593 rss_f.close()

594

595 -class LogFile:

596

597 - def __init__(self, level = 0, filename = None, header = "[LOG]"):

598 self.handler = self.default_handler 599 self.level = level 600 self.header = header 601 self.__logfile = None 602 self.open(filename) 603 self.__filename = filename

604

605 - def __del__(self):

606 self.close()

607

608 - def close(self):

609 try: 610 self.__logfile.close() 611 except (IOError, OSError,): 612 pass

613

614 - def get_fpath(self):

615 return self.__filename

616

617 - def flush(self):

618 self.__logfile.flush()

619

620 - def fileno(self):

621 return self.__get_file()

622

623 - def isatty(self):

624 return False

625

626 - def read(self, *args):

627 return ''

628

629 - def readline(self):

630 return ''

631

632 - def readlines(self):

633 return []

634

635 - def seek(self, offset):

636 return self.__logfile.seek(offset)

637

638 - def tell(self):

639 return self.__logfile.tell()

640

641 - def truncate(self):

642 return self.__logfile.truncate()

643

644 - def open(self, file_path = None):

645 646 if isinstance(file_path, basestring): 647 if not os.access(file_path, os.F_OK) and os.access( 648 os.path.dirname(file_path), os.W_OK): 649 self.__logfile = open(file_path, "aw") 650 else: 651 if os.access(file_path, os.W_OK | os.F_OK): 652 self.__logfile = open(file_path, "aw") 653 else: 654 self.__logfile = open("/dev/null", "aw") 655 elif hasattr(file_path, 'write'): 656 self.__logfile = file_path 657 else: 658 self.__logfile = sys.stderr

659

660 - def __get_file(self):

661 return self.__logfile.fileno()

662

663 - def __call__(self, format, *args):

664 self.handler (format % args)

665

666 - def default_handler(self, mystr):

667 try: 668 self.__logfile.write ("* %s\n" % (mystr)) 669 except UnicodeEncodeError: 670 self.__logfile.write ("* %s\n" % (mystr.encode('utf-8'),)) 671 self.__logfile.flush()

672

673 - def set_loglevel(self, level):

674 self.level = level

675

676 - def log(self, messagetype, level, message):

677 if self.level >= level and not etpUi['nolog']: 678 self.handler("%s %s %s %s" % (self.__get_header(), 679 messagetype, self.header, message,))

680

681 - def write(self, line):

682 self.handler(line)

683

684 - def writelines(self, lst):

685 for line in lst: 686 self.write(line)

687

688 - def __get_header(self):

689 return time.strftime('[%H:%M:%S %d/%m/%Y %Z]')

690

691 -class Callable:

692 - def __init__(self, anycallable):

693 self.__call__ = anycallable

694

695 -class MultipartPostHandler(urllib2.BaseHandler):

696 697 handler_order = urllib2.HTTPHandler.handler_order - 10 # needs to run first 698

699 - def __init__(self):

700 pass

701

702 - def http_request(self, request):

703 704 import urllib 705 doseq = 1 706 707 data = request.get_data() 708 if data is not None and type(data) != str: 709 v_files = [] 710 v_vars = [] 711 try: 712 for (key, value) in data.items(): 713 if type(value) == file: 714 v_files.append((key, value)) 715 else: 716 v_vars.append((key, value)) 717 except TypeError: 718 raise TypeError, "not a valid non-string sequence" \ 719 " or mapping object" 720 721 if len(v_files) == 0: 722 data = urllib.urlencode(v_vars, doseq) 723 else: 724 boundary, data = self.multipart_encode(v_vars, v_files) 725 contenttype = 'multipart/form-data; boundary=%s' % boundary 726 request.add_unredirected_header('Content-Type', contenttype) 727 728 request.add_data(data) 729 return request

730

731 - def multipart_encode(self, myvars, files, boundary = None, buf = None):

732 733 from cStringIO import StringIO 734 import mimetools, mimetypes 735 #import stat 736 737 if boundary is None: 738 boundary = mimetools.choose_boundary() 739 if buf is None: 740 buf = StringIO() 741 for(key, value) in myvars: 742 buf.write('--%s\r\n' % boundary) 743 buf.write('Content-Disposition: form-data; name="%s"' % key) 744 buf.write('\r\n\r\n' + value + '\r\n') 745 for(key, fdesc) in files: 746 #file_size = os.fstat(fdesc.fileno())[stat.ST_SIZE] 747 filename = fdesc.name.split('/')[-1] 748 contenttype = mimetypes.guess_type(filename)[0] or \ 749 'application/octet-stream' 750 buf.write('--%s\r\n' % boundary) 751 buf.write('Content-Disposition: form-data; name="%s"; ' \ 752 'filename="%s"\r\n' % (key, filename)) 753 buf.write('Content-Type: %s\r\n' % contenttype) 754 # buffer += 'Content-Length: %s\r\n' % file_size 755 fdesc.seek(0) 756 buf.write('\r\n' + fdesc.read() + '\r\n') 757 buf.write('--' + boundary + '--\r\n\r\n') 758 buf = buf.getvalue() 759 return boundary, buf

760 761 multipart_encode = Callable(multipart_encode) 762 763 https_request = http_request

764

Source Code for Module entropy.misc