1
2
3 '''
4 # DESCRIPTION:
5 # generic tools for all the handlers applications
6
7 Copyright (C) 2007-2008 Fabio Erculiani
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 '''
23
24 from __future__ import with_statement
25 import random
26 import stat
27 import errno
28 import re
29 import os
30 import time
31 import shutil
32 import tarfile
33 import subprocess
34 import grp
35 import pwd
36 import hashlib
37 from entropy.output import *
38 from entropy.const import *
39 from entropy.exceptions import *
40
43
45
46 if uid == None:
47 uid = os.getuid()
48 if uid == 0:
49 return True
50
51 try:
52 username = pwd.getpwuid(uid)[0]
53 except KeyError:
54 return False
55
56 try:
57 data = grp.getgrnam(etpConst['sysgroup'])
58 except KeyError:
59 return False
60
61
62 etp_group_users = data[3]
63
64 if not etp_group_users or \
65 username not in etp_group_users:
66 return False
67
68 return True
69
71 try:
72 return pwd.getpwnam(username)[2]
73 except (KeyError, IndexError,):
74 return -1
75
77 try:
78 return grp.getgrnam(groupname)[2]
79 except (KeyError, IndexError,):
80 return -1
81
83 try:
84 return pwd.getpwuid(uid)[0]
85 except KeyError:
86 return None
87
89 try:
90 return grp.getgrgid(gid)[0]
91 except (KeyError, IndexError,):
92 return -1
93
96
98 import traceback
99 traceback.print_exc(file = f)
100
102 import traceback
103 from cStringIO import StringIO
104 buf = StringIO()
105 traceback.print_exc(file = buf)
106 return buf.getvalue()
107
109 import traceback
110 if not returndata: traceback.print_exc()
111 data = []
112 tb = sys.exc_info()[2]
113 while 1:
114 if not tb.tb_next:
115 break
116 tb = tb.tb_next
117 stack = []
118 stack.append(tb.tb_frame)
119
120 for frame in stack:
121 if not returndata:
122 print
123 print "Frame %s in %s at line %s" % (frame.f_code.co_name,
124 frame.f_code.co_filename,
125 frame.f_lineno)
126 else:
127 data.append("Frame %s in %s at line %s" % (frame.f_code.co_name,
128 frame.f_code.co_filename,
129 frame.f_lineno))
130 for key, value in frame.f_locals.items():
131 if not returndata:
132 print "\t%20s = " % key,
133 else:
134 data.append("\t%20s = " % key,)
135 try:
136 if not returndata:
137 print value
138 else:
139 data.append(value)
140 except:
141 if not returndata: print "<ERROR WHILE PRINTING VALUE>"
142 return data
143
144
145
146
148
149 import socket
150 import urllib2
151 socket.setdefaulttimeout(60)
152
153 from entropy.core import SystemSettings
154 sys_settings = SystemSettings()
155 proxy_settings = sys_settings['system']['proxy']
156 try:
157 mydict = {}
158 if proxy_settings['ftp']:
159 mydict['ftp'] = proxy_settings['ftp']
160 if proxy_settings['http']:
161 mydict['http'] = proxy_settings['http']
162 if mydict:
163 mydict['username'] = proxy_settings['username']
164 mydict['password'] = proxy_settings['password']
165 add_proxy_opener(urllib2, mydict)
166 else:
167
168 urllib2._opener = None
169 item = urllib2.urlopen(url)
170 result = item.readlines()
171 item.close()
172 del item
173 if (not result):
174 socket.setdefaulttimeout(2)
175 return False
176 socket.setdefaulttimeout(2)
177 return result
178 except:
179 socket.setdefaulttimeout(2)
180 return False
181
183 f = open(path,"r")
184 x = f.read(4)
185 if x == '\x89PNG':
186 return True
187 return False
188
190 f = open(path,"r")
191 x = f.read(10)
192 if x == '\xff\xd8\xff\xe0\x00\x10JFIF':
193 return True
194 return False
195
197 f = open(path,"r")
198 x = f.read(2)
199 if x == 'BM':
200 return True
201 return False
202
204 f = open(path,"r")
205 x = f.read(5)
206 if x == 'GIF89':
207 return True
208 return False
209
215
217 april_first = "01-04"
218 cur_time = time.strftime("%d-%m")
219 if april_first == cur_time:
220 return True
221 return False
222
224 import types
225 if type(module) != types.ModuleType:
226 raise InvalidDataType("InvalidDataType: not a module")
227 if not data:
228 return
229
230 username = None
231 password = None
232 authinfo = None
233 if data.has_key('password'):
234 username = data.pop('username')
235 if data.has_key('password'):
236 username = data.pop('password')
237 if username == None or password == None:
238 username = None
239 password = None
240 else:
241 passmgr = module.HTTPPasswordMgrWithDefaultRealm()
242 if data['http']:
243 passmgr.add_password(None, data['http'], username, password)
244 if data['ftp']:
245 passmgr.add_password(None, data['ftp'], username, password)
246 authinfo = module.ProxyBasicAuthHandler(passmgr)
247
248 proxy_support = module.ProxyHandler(data)
249 if authinfo:
250 opener = module.build_opener(proxy_support, authinfo)
251 else:
252 opener = module.build_opener(proxy_support)
253 module.install_opener(opener)
254
256 try:
257 mystring = str(string)
258 del mystring
259 except:
260 return False
261 return True
262
264 try:
265 unicode(string)
266 except:
267 return False
268 return True
269
271 monster = "(?:[a-z0-9!#$%&'*+/=?^_{|}~-]+(?:.[a-z0-9!#$%" + \
272 "&'*+/=?^_{|}~-]+)*|\"(?:" + \
273 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]" + \
274 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*\")@(?:(?:[a-z0-9]" + \
275 "(?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" + \
276 "|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.)" + \
277 "{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?" + \
278 "|[a-z0-9-]*[a-z0-9]:(?:" + \
279 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]" + \
280 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"
281 evil = re.compile(monster)
282 if evil.match(email):
283 return True
284 return False
285
288
290 mystat = os.lstat(file_path)
291 return int(mystat.st_size)
292
294 size = 0
295 for myfile in file_list:
296 try:
297 size += get_file_size(myfile)
298 except (OSError,IOError,):
299 continue
300 return size
301
303 import statvfs
304 st = os.statvfs(mountpoint)
305 freeblocks = st[statvfs.F_BFREE]
306 blocksize = st[statvfs.F_BSIZE]
307 freespace = freeblocks*blocksize
308 if bytes_required > freespace:
309
310 return False
311 return True
312
314 """Return (status, output) of executing cmd in a shell."""
315 pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
316 text = pipe.read()
317 sts = pipe.close()
318 if sts is None: sts = 0
319 if text[-1:] == '\n': text = text[:-1]
320 return sts, text
321
322
323
324
325
326
327 -def movefile(src, dest, src_basedir = None):
328
329 sstat = os.lstat(src)
330 destexists = 1
331 try:
332 dstat = os.lstat(dest)
333 except (OSError, IOError,):
334 dstat = os.lstat(os.path.dirname(dest))
335 destexists = 0
336
337 if destexists:
338 if stat.S_ISLNK(dstat[stat.ST_MODE]):
339 try:
340 os.unlink(dest)
341 destexists = 0
342 except (OSError, IOError,):
343 pass
344
345 if stat.S_ISLNK(sstat[stat.ST_MODE]):
346 try:
347 target = os.readlink(src)
348 if src_basedir != None:
349 if target.find(src_basedir) == 0:
350 target = target[len(src_basedir):]
351 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
352 os.unlink(dest)
353 os.symlink(target,dest)
354 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
355 return True
356 except SystemExit:
357 raise
358 except Exception, e:
359 print "!!! failed to properly create symlink:"
360 print "!!!",dest,"->",target
361 print "!!!",e
362 return False
363
364 renamefailed = True
365 if sstat.st_dev == dstat.st_dev:
366 try:
367 os.rename(src,dest)
368 renamefailed = False
369 except Exception, e:
370 if e[0] != errno.EXDEV:
371
372 print "!!! Failed to move",src,"to",dest
373 print "!!!",e
374 return False
375
376
377 if renamefailed:
378 didcopy = True
379 if stat.S_ISREG(sstat[stat.ST_MODE]):
380 try:
381 while 1:
382 tmp_dest = "%s#entropy_new_%s" % (dest,get_random_number(),)
383 if not os.path.lexists(tmp_dest): break
384 shutil.copyfile(src,tmp_dest)
385 os.rename(tmp_dest,dest)
386 didcopy = True
387 except SystemExit, e:
388 raise
389 except Exception, e:
390 print '!!! copy',src,'->',dest,'failed.'
391 print "!!!",e
392 return False
393 else:
394
395 a = getstatusoutput("mv -f '%s' '%s'" % (src,dest,))
396 if a[0]!=0:
397 print "!!! Failed to move special file:"
398 print "!!! '"+src+"' to '"+dest+"'"
399 print "!!!",a
400 return False
401 try:
402 if didcopy:
403 if stat.S_ISLNK(sstat[stat.ST_MODE]):
404 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
405 else:
406 os.chown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
407 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE]))
408 os.unlink(src)
409 except SystemExit, e:
410 raise
411 except Exception, e:
412 print "!!! Failed to chown/chmod/unlink in movefile()"
413 print "!!!",dest
414 print "!!!",e
415 return False
416
417 try:
418 os.utime(dest, (sstat.st_atime, sstat.st_mtime))
419 except OSError:
420
421
422 try:
423 long(os.stat(dest).st_mtime)
424 return True
425 except OSError, e:
426 print "!!! Failed to stat in movefile()\n"
427 print "!!! %s\n" % dest
428 print "!!! %s\n" % str(e)
429 return False
430
431 return True
432
434 mycount = count
435 while mycount > 0:
436 os.system("sleep 0.35; echo -ne \"\a\"; sleep 0.35")
437 mycount -= 1
438
440 if etpConst['applicationlock']:
441 if not gentle:
442 raise SystemExit(10)
443 return True
444 return False
445
447 try:
448 return abs(hash(os.urandom(2)))%99999
449 except NotImplementedError:
450 random.seed()
451 return random.randint(10000,99999)
452
453 -def countdown(secs=5, what="Counting...", back = False):
454 if secs:
455 if back:
456 try:
457 print red(">>"), what,
458 except UnicodeEncodeError:
459 print red(">>"),what.encode('utf-8'),
460 else:
461 try:
462 print what
463 except UnicodeEncodeError:
464 print what.encode('utf-8')
465 for i in range(secs)[::-1]:
466 sys.stdout.write(red(str(i+1)+" "))
467 sys.stdout.flush()
468 time.sleep(1)
469
471 m = hashlib.md5()
472 readfile = open(filepath)
473 block = readfile.read(1024)
474 while block:
475 m.update(block)
476 block = readfile.read(1024)
477 readfile.close()
478 return m.hexdigest()
479
481 m = hashlib.sha512()
482 readfile = open(filepath)
483 block = readfile.read(1024)
484 while block:
485 m.update(block)
486 block = readfile.read(1024)
487 readfile.close()
488 return m.hexdigest()
489
491 m = hashlib.sha256()
492 readfile = open(filepath)
493 block = readfile.read(1024)
494 while block:
495 m.update(block)
496 block = readfile.read(1024)
497 readfile.close()
498 return m.hexdigest()
499
501 m = hashlib.sha1()
502 readfile = open(filepath)
503 block = readfile.read(1024)
504 while block:
505 m.update(block)
506 block = readfile.read(1024)
507 readfile.close()
508 return m.hexdigest()
509
511 if not os.path.isdir(directory):
512 raise DirectoryNotFound("DirectoryNotFound: directory just does not exist.")
513 myfiles = os.listdir(directory)
514 m = hashlib.md5()
515 if not myfiles:
516 if get_obj:
517 return m
518 return "0"
519
520 for currentdir,subdirs,files in os.walk(directory):
521 for myfile in files:
522 myfile = os.path.join(currentdir,myfile)
523 readfile = open(myfile)
524 block = readfile.read(1024)
525 while block:
526 m.update(block)
527 block = readfile.read(1024)
528 readfile.close()
529 if get_obj:
530 return m
531 return m.hexdigest()
532
533
534
535 -def getfd(filespec, readOnly = 0):
536 import types
537 if type(filespec) == types.IntType:
538 return filespec
539 if filespec == None:
540 filespec = "/dev/null"
541
542 flags = os.O_RDWR | os.O_CREAT
543 if (readOnly):
544 flags = os.O_RDONLY
545 return os.open(filespec, flags)
546
548 f_out = open(destination_path,"wb")
549 f_in = opener(file_path,"rb")
550 data = f_in.read(8192)
551 while data:
552 f_out.write(data)
553 data = f_in.read(8192)
554 f_out.flush()
555 f_out.close()
556 f_in.close()
557
558 -def compress_file(file_path, destination_path, opener, compress_level = None):
559 f_in = open(file_path,"rb")
560 if compress_level != None:
561 f_out = opener(destination_path,"wb",compresslevel = compress_level)
562 else:
563 f_out = opener(destination_path,"wb")
564 data = f_in.read(8192)
565 while data:
566 f_out.write(data)
567 data = f_in.read(8192)
568 if hasattr(f_out,'flush'):
569 f_out.flush()
570 f_out.close()
571 f_in.close()
572
573
575
576 if compressor not in ("bz2","gz",):
577 raise AttributeError("invalid compressor specified")
578
579 id_strings = {}
580 tar = tarfile.open(dest_file,"w:%s" % (compressor,))
581 try:
582 for path in files_to_compress:
583 exist = os.lstat(path)
584 tarinfo = tar.gettarinfo(path, os.path.basename(path))
585 tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
586 tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))
587 if not stat.S_ISREG(exist.st_mode): continue
588 tarinfo.type = tarfile.REGTYPE
589 with open(path) as f:
590 tar.addfile(tarinfo, f)
591 finally:
592 tar.close()
593
595
596 try:
597 tar = tarfile.open(compressed_file,"r")
598 except tarfile.ReadError:
599 if catch_empty:
600 return True
601 return False
602 except EOFError:
603 return False
604
605 try:
606
607 dest_path = dest_path.encode('utf-8')
608 def mymf(tarinfo):
609 if tarinfo.isdir():
610
611
612 try:
613 os.makedirs(os.path.join(dest_path, tarinfo.name), 0777)
614 except EnvironmentError:
615 pass
616 return tarinfo
617 tar.extract(tarinfo, dest_path)
618 del tar.members[:]
619 return tarinfo
620
621 def mycmp(a,b):
622 return cmp(a.name, b.name)
623
624 directories = sorted(map(mymf, tar), mycmp, reverse = True)
625
626
627 def mymf2(tarinfo):
628 epath = os.path.join(dest_path, tarinfo.name)
629 try:
630 tar.chown(tarinfo, epath)
631
632
633
634 uname = tarinfo.uname
635 gname = tarinfo.gname
636 ugdata_valid = False
637 try:
638 int(gname)
639 int(uname)
640 except ValueError:
641 ugdata_valid = True
642
643 try:
644 if ugdata_valid:
645
646
647 uid, gid = get_uid_from_user(uname), \
648 get_gid_from_group(gname)
649 os.lchown(epath, uid, gid)
650 except OSError:
651 pass
652
653 tar.utime(tarinfo, epath)
654 tar.chmod(tarinfo, epath)
655 except tarfile.ExtractError:
656 if tar.errorlevel > 1:
657 return False
658 done = map(mymf2, directories)
659 del done
660
661 except EOFError:
662 return False
663
664 finally:
665 tar.close()
666
667 return True
668
670 import gzip
671 filepath = gzipfilepath[:-3]
672 item = open(filepath,"wb")
673 filegz = gzip.GzipFile(gzipfilepath,"rb")
674 chunk = filegz.read(8192)
675 while chunk:
676 item.write(chunk)
677 chunk = filegz.read(8192)
678 filegz.close()
679 item.flush()
680 item.close()
681 return filepath
682
684 import bz2
685 filepath = bzip2filepath[:-4]
686 item = open(filepath,"wb")
687 filebz2 = bz2.BZ2File(bzip2filepath,"rb")
688 chunk = filebz2.read(8192)
689 while chunk:
690 item.write(chunk)
691 chunk = filebz2.read(8192)
692 filebz2.close()
693 item.flush()
694 item.close()
695 return filepath
696
698 if os.path.isfile(etpConst['etpdatabaseclientfilepath']):
699 rnd = get_random_number()
700 source = etpConst['etpdatabaseclientfilepath']
701 dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd)
702 shutil.copy2(source,dest)
703 user = os.stat(source)[4]
704 group = os.stat(source)[5]
705 os.chown(dest,user,group)
706 shutil.copystat(source,dest)
707 return dest
708 return ""
709
714
716 xpakpath = suck_xpak(tbz2file, etpConst['entropyunpackdir'])
717 f = open(xpakpath,"rb")
718 data = f.read()
719 f.close()
720 os.remove(xpakpath)
721 return data
722
724 try:
725 import entropy.xpak as xpak
726 if tmpdir is None:
727 tmpdir = etpConst['packagestmpdir']+"/"+os.path.basename(xpakfile)[:-5]+"/"
728 if os.path.isdir(tmpdir):
729 shutil.rmtree(tmpdir,True)
730 os.makedirs(tmpdir)
731 xpakdata = xpak.getboth(xpakfile)
732 xpak.xpand(xpakdata,tmpdir)
733 del xpakdata
734 try:
735 os.remove(xpakfile)
736 except OSError:
737 pass
738 except:
739 return None
740 return tmpdir
741
743
744 dest_filename = os.path.basename(tbz2file)[:-5]+".xpak"
745 xpakpath = os.path.join(outputpath, dest_filename)
746 old = open(tbz2file,"rb")
747
748
749 old.seek(0, os.SEEK_END)
750
751 bytes = old.tell()
752 counter = bytes - 1
753 xpak_end = "XPAKSTOP"
754 xpak_start = "XPAKPACK"
755 xpak_entry_point = "X"
756 xpak_tag_len = len(xpak_start)
757 chunk_len = 3
758 data_start_position = None
759 data_end_position = None
760
761 while counter >= (0 - chunk_len):
762
763 old.seek(counter - bytes, os.SEEK_END)
764 if (bytes - (abs(counter - bytes))) < chunk_len:
765 chunk_len = 1
766 read_bytes = old.read(chunk_len)
767 read_len = len(read_bytes)
768
769 entry_idx = read_bytes.rfind(xpak_entry_point)
770 if entry_idx != -1:
771
772 cut_gotten = read_bytes[entry_idx:]
773 offset = xpak_tag_len - len(cut_gotten)
774 chunk = cut_gotten + old.read(offset)
775
776 if (chunk == xpak_end) and (data_start_position is None):
777 data_end_position = old.tell()
778
779 elif (chunk == xpak_start) and (data_end_position is not None):
780 data_start_position = old.tell() - xpak_tag_len
781 break
782
783 counter -= read_len
784
785 if data_start_position is None:
786 return None
787 if data_end_position is None:
788 return None
789
790
791
792
793 db = open(xpakpath,"wb")
794 old.seek(data_start_position)
795 to_read = data_end_position - data_start_position
796 while to_read > 0:
797 data = old.read(to_read)
798 db.write(data)
799 to_read -= len(data)
800
801 db.flush()
802 db.close()
803 old.close()
804 return xpakpath
805
817
829
831
832 old = open(tbz2file, "rb")
833 if not dbpath:
834 dbpath = tbz2file[:-5] + ".db"
835
836 start_position = locate_edb(old)
837 if not start_position:
838 old.close()
839 try:
840 os.remove(dbpath)
841 except OSError:
842 return None
843 return None
844
845 db = open(dbpath, "wb")
846 data = old.read(1024)
847 while data:
848 db.write(data)
849 data = old.read(1024)
850 db.flush()
851 db.close()
852
853 return dbpath
854
856
857
858 fileobj.seek(0, os.SEEK_END)
859
860 bytes = fileobj.tell()
861 counter = bytes - 1
862
863 db_tag = etpConst['databasestarttag']
864 db_tag_len = len(db_tag)
865 give_up_threshold = 1024000 * 30
866 entry_point = db_tag[::-1][0]
867 max_read_len = 8
868 start_position = None
869
870 while counter >= 0:
871 cur_threshold = abs((counter-bytes))
872 if cur_threshold >= give_up_threshold:
873 start_position = None
874 break
875 fileobj.seek(counter-bytes, os.SEEK_END)
876 read_bytes = fileobj.read(max_read_len)
877 read_len = len(read_bytes)
878 entry_idx = read_bytes.rfind(entry_point)
879 if entry_idx != -1:
880 rollback = (read_len - entry_idx) * -1
881 fileobj.seek(rollback, os.SEEK_CUR)
882 chunk = fileobj.read(db_tag_len)
883 if chunk == db_tag:
884 start_position = fileobj.tell()
885 break
886 counter -= read_len
887
888 return start_position
889
891 old = open(tbz2file, "rb")
892
893 start_position = locate_edb(old)
894 if not start_position:
895 old.close()
896 return None
897
898 new_path = os.path.join(savedir, os.path.basename(tbz2file))
899 new = open(new_path, "wb")
900
901 old.seek(0)
902 counter = 0
903 max_read_len = 1024
904 db_tag = etpConst['databasestarttag']
905 db_tag_len = len(db_tag)
906 start_position -= db_tag_len
907
908 while counter < start_position:
909 delta = start_position - counter
910 if delta < max_read_len:
911 max_read_len = delta
912 bytes = old.read(max_read_len)
913 read_bytes = len(bytes)
914 new.write(bytes)
915 counter += read_bytes
916
917 new.flush()
918 new.close()
919 old.close()
920 return savedir+"/"+os.path.basename(tbz2file)
921
922
924 md5hash = md5sum(filepath)
925 hashfile = filepath+etpConst['packagesmd5fileext']
926 f = open(hashfile,"w")
927 name = os.path.basename(filepath)
928 f.write(md5hash+" "+name+"\n")
929 f.flush()
930 f.close()
931 return hashfile
932
934 sha512hash = sha512(filepath)
935 hashfile = filepath+etpConst['packagessha512fileext']
936 f = open(hashfile,"w")
937 tbz2name = os.path.basename(filepath)
938 f.write(sha512hash+" "+tbz2name+"\n")
939 f.flush()
940 f.close()
941 return hashfile
942
944 sha256hash = sha256(filepath)
945 hashfile = filepath+etpConst['packagessha256fileext']
946 f = open(hashfile,"w")
947 tbz2name = os.path.basename(filepath)
948 f.write(sha256hash+" "+tbz2name+"\n")
949 f.flush()
950 f.close()
951 return hashfile
952
954 sha1hash = sha1(filepath)
955 hashfile = filepath+etpConst['packagessha1fileext']
956 f = open(hashfile,"w")
957 tbz2name = os.path.basename(filepath)
958 f.write(sha1hash+" "+tbz2name+"\n")
959 f.flush()
960 f.close()
961 return hashfile
962
964 checksum = str(checksum)
965 result = md5sum(filepath)
966 result = str(result)
967 if checksum == result:
968 return True
969 return False
970
972 checksum = str(checksum)
973 result = sha512(filepath)
974 result = str(result)
975 if checksum == result:
976 return True
977 return False
978
980 checksum = str(checksum)
981 result = sha256(filepath)
982 result = str(result)
983 if checksum == result:
984 return True
985 return False
986
988 checksum = str(checksum)
989 result = sha1(filepath)
990 result = str(result)
991 if checksum == result:
992 return True
993 return False
994
996 m = hashlib.md5()
997 m.update(string)
998 return m.hexdigest()
999
1000
1002 sort_dict = {}
1003
1004 for item in update_list:
1005
1006 year = item.split("-")[1]
1007 if sort_dict.has_key(year):
1008 sort_dict[year].append(item)
1009 else:
1010 sort_dict[year] = []
1011 sort_dict[year].append(item)
1012 new_list = []
1013 keys = sort_dict.keys()
1014 keys.sort()
1015 for key in keys:
1016 sort_dict[key].sort()
1017 new_list += sort_dict[key]
1018 del sort_dict
1019 return new_list
1020
1022 data = []
1023 if os.access(filepath, os.R_OK | os.F_OK):
1024 gen_f = open(filepath,"r")
1025 content = gen_f.readlines()
1026 gen_f.close()
1027
1028 content = [x.strip().rsplit("#", 1)[0].strip() for x in content \
1029 if not x.startswith("#") and x.strip()]
1030 for line in content:
1031 if line in data:
1032 continue
1033 data.append(line)
1034 return data
1035
1036
1038
1039
1040 if os.path.isfile(file) and os.path.isfile(fromfile):
1041 old = md5sum(fromfile)
1042 new = md5sum(file)
1043 if old == new:
1044 return file, False
1045
1046 counter = -1
1047 newfile = ""
1048 previousfile = ""
1049
1050 while 1:
1051 counter += 1
1052 txtcounter = str(counter)
1053 oldtxtcounter = str(counter-1)
1054 txtcounter_len = 4-len(txtcounter)
1055 cnt = 0
1056 while cnt < txtcounter_len:
1057 txtcounter = "0"+txtcounter
1058 oldtxtcounter = "0"+oldtxtcounter
1059 cnt += 1
1060 newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file)
1061 if counter > 0:
1062 previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file)
1063 else:
1064 previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1065 if not os.path.exists(newfile):
1066 break
1067 if not newfile:
1068 newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1069 else:
1070
1071 if os.path.exists(previousfile):
1072
1073
1074 new = md5sum(fromfile)
1075 old = md5sum(previousfile)
1076 if new == old:
1077 return previousfile, False
1078
1079
1080 new = md5sum(file)
1081 old = md5sum(previousfile)
1082 if (new == old):
1083 return previousfile, False
1084
1085 return newfile, True
1086
1088
1089 logline = False
1090 logoutput = []
1091 f = open(file,"r")
1092 reallog = f.readlines()
1093 f.close()
1094
1095 for line in reallog:
1096 if line.startswith("INFO: postinst") or line.startswith("LOG: postinst"):
1097 logline = True
1098 continue
1099
1100 elif line.startswith("LOG:"):
1101 logline = False
1102 continue
1103 if (logline) and (line.strip()):
1104
1105 logoutput.append(line.strip())
1106 return logoutput
1107
1108
1109
1110
1111
1112 ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\\d*)*)(-r(\\d+))?$")
1113 suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
1114 suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
1115 endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
1116
1117
1119 myparts = mypkg.split('-')
1120 for x in myparts:
1121 if ververify(x):
1122 return 0
1123 return 1
1124
1126
1127 myver = myverx[:]
1128 if myver.endswith("*"):
1129 myver = myver[:-1]
1130 if ver_regexp.match(myver):
1131 return 1
1132 else:
1133 if not silent:
1134 print "!!! syntax error in version: %s" % myver
1135 return 0
1136
1137
1138
1139
1140
1141 valid_category = re.compile("^\w[\w-]*")
1142 invalid_atom_chars_regexp = re.compile("[()|@]")
1143
1145 """
1146 Check to see if a depend atom is valid
1147
1148 Example usage:
1149 >>> isvalidatom('media-libs/test-3.0')
1150 0
1151 >>> isvalidatom('>=media-libs/test-3.0')
1152 1
1153
1154 @param atom: The depend atom to check against
1155 @type atom: String
1156 @rtype: Integer
1157 @return: One of the following:
1158 1) 0 if the atom is invalid
1159 2) 1 if the atom is valid
1160 """
1161 atom = remove_tag(myatom)
1162 atom = remove_usedeps(atom)
1163 if invalid_atom_chars_regexp.search(atom):
1164 return 0
1165 if allow_blockers and atom[:1] == "!":
1166 if atom[1:2] == "!":
1167 atom = atom[2:]
1168 else:
1169 atom = atom[1:]
1170
1171
1172 if atom.count("/") > 1:
1173 return 0
1174
1175 cpv = dep_getcpv(atom)
1176 cpv_catsplit = catsplit(cpv)
1177 mycpv_cps = None
1178 if cpv:
1179 if len(cpv_catsplit) == 2:
1180 if valid_category.match(cpv_catsplit[0]) is None:
1181 return 0
1182 if cpv_catsplit[0] == "null":
1183
1184 mycpv_cps = catpkgsplit(cpv.replace("null/", "cat/", 1))
1185 if mycpv_cps:
1186 mycpv_cps = list(mycpv_cps)
1187 mycpv_cps[0] = "null"
1188 if not mycpv_cps:
1189 mycpv_cps = catpkgsplit(cpv)
1190
1191 operator = get_operator(atom)
1192 if operator:
1193 if operator[0] in "<>" and remove_slot(atom).endswith("*"):
1194 return 0
1195 if mycpv_cps:
1196 if len(cpv_catsplit) == 2:
1197
1198 return 1
1199 else:
1200 return 0
1201 else:
1202
1203 return 0
1204 if mycpv_cps:
1205
1206 return 0
1207
1208 if len(cpv_catsplit) == 2:
1209
1210 return 1
1211 return 0
1212
1214 return mydep.split("/", 1)
1215
1217 """
1218 Return the operator used in a depstring.
1219
1220 Example usage:
1221 >>> from portage.dep import *
1222 >>> get_operator(">=test-1.0")
1223 '>='
1224
1225 @param mydep: The dep string to check
1226 @type mydep: String
1227 @rtype: String
1228 @return: The operator. One of:
1229 '~', '=', '>', '<', '=*', '>=', or '<='
1230 """
1231 if mydep:
1232 mydep = remove_slot(mydep)
1233 if not mydep:
1234 return None
1235 if mydep[0] == "~":
1236 operator = "~"
1237 elif mydep[0] == "=":
1238 if mydep[-1] == "*":
1239 operator = "=*"
1240 else:
1241 operator = "="
1242 elif mydep[0] in "><":
1243 if len(mydep) > 1 and mydep[1] == "=":
1244 operator = mydep[0:2]
1245 else:
1246 operator = mydep[0]
1247 else:
1248 operator = None
1249
1250 return operator
1251
1253 """
1254 Checks to see if the depstring is only the package name (no version parts)
1255
1256 Example usage:
1257 >>> isjustname('media-libs/test-3.0')
1258 0
1259 >>> isjustname('test')
1260 1
1261 >>> isjustname('media-libs/test')
1262 1
1263
1264 @param mypkg: The package atom to check
1265 @param mypkg: String
1266 @rtype: Integer
1267 @return: One of the following:
1268 1) 0 if the package string is not just the package name
1269 2) 1 if it is
1270 """
1271
1272 myparts = mypkg.split('-')
1273 for x in myparts:
1274 if ververify(x):
1275 return 0
1276 return 1
1277
1279 """
1280 Checks to see if a package is in category/package-version or package-version format,
1281 possibly returning a cached result.
1282
1283 Example usage:
1284 >>> isspecific('media-libs/test')
1285 0
1286 >>> isspecific('media-libs/test-3.0')
1287 1
1288
1289 @param mypkg: The package depstring to check against
1290 @type mypkg: String
1291 @rtype: Integer
1292 @return: One of the following:
1293 1) 0 if the package string is not specific
1294 2) 1 if it is
1295 """
1296 mysplit = mypkg.split("/")
1297 if not isjustname(mysplit[-1]):
1298 return 1
1299 return 0
1300
1301
1303 """
1304 Takes a Category/Package-Version-Rev and returns a list of each.
1305
1306 @param mydata: Data to split
1307 @type mydata: string
1308 @param silent: suppress error messages
1309 @type silent: Boolean (integer)
1310 @rype: list
1311 @return:
1312 1. If each exists, it returns [cat, pkgname, version, rev]
1313 2. If cat is not specificed in mydata, cat will be "null"
1314 3. if rev does not exist it will be '-r0'
1315 4. If cat is invalid (specified but has incorrect syntax)
1316 an InvalidData Exception will be thrown
1317 """
1318
1319
1320 mysplit=mydata.split("/")
1321 p_split=None
1322 if len(mysplit)==1:
1323 retval=["null"]
1324 p_split=pkgsplit(mydata,silent=silent)
1325 elif len(mysplit)==2:
1326 retval=[mysplit[0]]
1327 p_split=pkgsplit(mysplit[1],silent=silent)
1328 if not p_split:
1329 return None
1330 retval.extend(p_split)
1331 return retval
1332
1334 myparts=mypkg.split("-")
1335
1336 if len(myparts)<2:
1337 if not silent:
1338 print "!!! Name error in",mypkg+": missing a version or name part."
1339 return None
1340 for x in myparts:
1341 if len(x)==0:
1342 if not silent:
1343 print "!!! Name error in",mypkg+": empty \"-\" part."
1344 return None
1345
1346
1347 revok=0
1348 myrev=myparts[-1]
1349
1350 if len(myrev) and myrev[0]=="r":
1351 try:
1352 int(myrev[1:])
1353 revok=1
1354 except ValueError:
1355 pass
1356 if revok:
1357 verPos = -2
1358 revision = myparts[-1]
1359 else:
1360 verPos = -1
1361 revision = "r0"
1362
1363 if ververify(myparts[verPos]):
1364 if len(myparts)== (-1*verPos):
1365 return None
1366 else:
1367 for x in myparts[:verPos]:
1368 if ververify(x):
1369 return None
1370
1371 myval=["-".join(myparts[:verPos]),myparts[verPos],revision]
1372 return myval
1373 else:
1374 return None
1375
1377 """
1378 Return the category/package-name of a depstring.
1379
1380 Example usage:
1381 >>> dep_getkey('media-libs/test-3.0')
1382 'media-libs/test'
1383
1384 @param mydep: The depstring to retrieve the category/package-name of
1385 @type mydep: String
1386 @rtype: String
1387 @return: The package category/package-version
1388 """
1389 if not mydepx: return mydepx
1390 mydep = mydepx[:]
1391 mydep = remove_tag(mydep)
1392 mydep = remove_usedeps(mydep)
1393
1394 mydep = dep_getcpv(mydep)
1395 if mydep and isspecific(mydep):
1396 mysplit = catpkgsplit(mydep)
1397 if not mysplit:
1398 return mydep
1399 return mysplit[0] + "/" + mysplit[1]
1400
1401 return mydep
1402
1403
1405 """
1406 Return the category-package-version with any operators/slot specifications stripped off
1407
1408 Example usage:
1409 >>> dep_getcpv('>=media-libs/test-3.0')
1410 'media-libs/test-3.0'
1411
1412 @param mydep: The depstring
1413 @type mydep: String
1414 @rtype: String
1415 @return: The depstring with the operator removed
1416 """
1417
1418 if mydep and mydep[0] == "*":
1419 mydep = mydep[1:]
1420 if mydep and mydep[-1] == "*":
1421 mydep = mydep[:-1]
1422 if mydep and mydep[0] == "!":
1423 mydep = mydep[1:]
1424 if mydep[:2] in [">=", "<="]:
1425 mydep = mydep[2:]
1426 elif mydep[:1] in "=<>~":
1427 mydep = mydep[1:]
1428 colon = mydep.rfind(":")
1429 if colon != -1:
1430 mydep = mydep[:colon]
1431
1432 return mydep
1433
1435 """
1436
1437 # Imported from portage.dep
1438 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1439
1440 Retrieve the slot on a depend.
1441
1442 Example usage:
1443 >>> dep_getslot('app-misc/test:3')
1444 '3'
1445
1446 @param mydep: The depstring to retrieve the slot of
1447 @type mydep: String
1448 @rtype: String
1449 @return: The slot
1450 """
1451 colon = mydep.find(":")
1452 if colon != -1:
1453 bracket = mydep.find("[", colon)
1454 if bracket == -1:
1455 return mydep[colon+1:]
1456 else:
1457 return mydep[colon+1:bracket]
1458 return None
1459
1461
1462 """
1463
1464 # Imported from portage.dep
1465 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1466
1467 Pull a listing of USE Dependencies out of a dep atom.
1468
1469 Example usage:
1470 >>> dep_getusedeps('app-misc/test:3[foo,-bar]')
1471 ('foo','-bar')
1472
1473 @param depend: The depstring to process
1474 @type depend: String
1475 @rtype: List
1476 @return: List of use flags ( or [] if no flags exist )
1477 """
1478
1479 use_list = []
1480 open_bracket = depend.find('[')
1481
1482 comma_separated = False
1483 bracket_count = 0
1484 while( open_bracket != -1 ):
1485 bracket_count += 1
1486 if bracket_count > 1:
1487 raise InvalidAtom("USE Dependency with more " + \
1488 "than one set of brackets: %s" % (depend,))
1489 close_bracket = depend.find(']', open_bracket )
1490 if close_bracket == -1:
1491 raise InvalidAtom("USE Dependency with no closing bracket: %s" % depend )
1492 use = depend[open_bracket + 1: close_bracket]
1493
1494 if not use:
1495 raise InvalidAtom("USE Dependency with " + \
1496 "no use flag ([]): %s" % depend )
1497 if not comma_separated:
1498 comma_separated = "," in use
1499
1500 if comma_separated and bracket_count > 1:
1501 raise InvalidAtom("USE Dependency contains a mixture of " + \
1502 "comma and bracket separators: %s" % depend )
1503
1504 if comma_separated:
1505 for x in use.split(","):
1506 if x:
1507 use_list.append(x)
1508 else:
1509 raise InvalidAtom("USE Dependency with no use " + \
1510 "flag next to comma: %s" % depend )
1511 else:
1512 use_list.append(use)
1513
1514
1515 open_bracket = depend.find( '[', open_bracket+1 )
1516
1517 return tuple(use_list)
1518
1520 mydepend = depend[:]
1521
1522 close_bracket = mydepend.find(']')
1523 after_closebracket = ''
1524 if close_bracket != -1: after_closebracket = mydepend[close_bracket+1:]
1525
1526 open_bracket = mydepend.find('[')
1527 if open_bracket != -1: mydepend = mydepend[:open_bracket]
1528
1529 return mydepend+after_closebracket
1530
1532 """
1533
1534 # Imported from portage.dep
1535 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1536
1537 Removes dep components from the right side of an atom:
1538 * slot
1539 * use
1540 * repo
1541 """
1542 colon = mydep.find(":")
1543 if colon != -1:
1544 mydep = mydep[:colon]
1545 else:
1546 bracket = mydep.find("[")
1547 if bracket != -1:
1548 mydep = mydep[:bracket]
1549 return mydep
1550
1551
1553 myver = ver.split("-")
1554 if myver[-1][0] == "r":
1555 return '-'.join(myver[:-1])
1556 return ver
1557
1559 colon = mydep.rfind("#")
1560 if colon == -1:
1561 return mydep
1562 return mydep[:colon]
1563
1571
1573
1574 colon = mydep.rfind("~")
1575 if colon != -1:
1576 myrev = mydep[colon+1:]
1577 try:
1578 myrev = int(myrev)
1579 except ValueError:
1580 return None
1581 return myrev
1582 return None
1583
1584
1585 dep_revmatch = re.compile('^r[0-9]')
1587 myver = mydep.split("-")
1588 myrev = myver[-1]
1589 if dep_revmatch.match(myrev):
1590 return myrev
1591 else:
1592 return "r0"
1593
1594
1596 colon = mydep.rfind("@")
1597 if colon != -1:
1598 mydata = mydep[colon+1:]
1599 mydata = mydata.split(",")
1600 if not mydata:
1601 mydata = None
1602 return mydep[:colon],mydata
1603 else:
1604 return mydep,None
1605
1607
1608 """
1609 Retrieve the slot on a depend.
1610
1611 Example usage:
1612 >>> dep_gettag('app-misc/test#2.6.23-sabayon-r1')
1613 '2.6.23-sabayon-r1'
1614
1615 """
1616
1617 colon = dep.rfind("#")
1618 if colon != -1:
1619 mydep = dep[colon+1:]
1620 rslt = remove_slot(mydep)
1621 return rslt
1622 return None
1623
1625 try:
1626 while atom:
1627 if atom[0] in ('>','<','=','~',):
1628 atom = atom[1:]
1629 continue
1630 break
1631 except IndexError:
1632 pass
1633 return atom
1634
1635
1636
1637
1639
1640 if ver1 == ver2:
1641 return 0
1642
1643 match1 = None
1644 match2 = None
1645 if ver1:
1646 match1 = ver_regexp.match(ver1)
1647 if ver2:
1648 match2 = ver_regexp.match(ver2)
1649
1650
1651 invalid = False
1652 invalid_rc = 0
1653 if not match1:
1654 invalid = True
1655 elif not match1.groups():
1656 invalid = True
1657 elif not match2:
1658 invalid_rc = 1
1659 invalid = True
1660 elif not match2.groups():
1661 invalid_rc = 1
1662 invalid = True
1663 if invalid: return invalid_rc
1664
1665
1666
1667 list1 = [int(match1.group(2))]
1668 list2 = [int(match2.group(2))]
1669
1670
1671 if len(match1.group(3)) or len(match2.group(3)):
1672 vlist1 = match1.group(3)[1:].split(".")
1673 vlist2 = match2.group(3)[1:].split(".")
1674 for i in range(0, max(len(vlist1), len(vlist2))):
1675
1676
1677
1678 if len(vlist1) <= i or len(vlist1[i]) == 0:
1679 list1.append(-1)
1680 list2.append(int(vlist2[i]))
1681 elif len(vlist2) <= i or len(vlist2[i]) == 0:
1682 list1.append(int(vlist1[i]))
1683 list2.append(-1)
1684
1685 elif (vlist1[i][0] != "0" and vlist2[i][0] != "0"):
1686 list1.append(int(vlist1[i]))
1687 list2.append(int(vlist2[i]))
1688
1689 else:
1690 list1.append(float("0."+vlist1[i]))
1691 list2.append(float("0."+vlist2[i]))
1692
1693
1694 if len(match1.group(5)):
1695 list1.append(ord(match1.group(5)))
1696 if len(match2.group(5)):
1697 list2.append(ord(match2.group(5)))
1698
1699 for i in range(0, max(len(list1), len(list2))):
1700 if len(list1) <= i:
1701 return -1
1702 elif len(list2) <= i:
1703 return 1
1704 elif list1[i] != list2[i]:
1705 return list1[i] - list2[i]
1706
1707
1708 list1 = match1.group(6).split("_")[1:]
1709 list2 = match2.group(6).split("_")[1:]
1710
1711 for i in range(0, max(len(list1), len(list2))):
1712 if len(list1) <= i:
1713 s1 = ("p","0")
1714 else:
1715 s1 = suffix_regexp.match(list1[i]).groups()
1716 if len(list2) <= i:
1717 s2 = ("p","0")
1718 else:
1719 s2 = suffix_regexp.match(list2[i]).groups()
1720 if s1[0] != s2[0]:
1721 return suffix_value[s1[0]] - suffix_value[s2[0]]
1722 if s1[1] != s2[1]:
1723
1724
1725 try:
1726 r1 = int(s1[1])
1727 except ValueError:
1728 r1 = 0
1729 try:
1730 r2 = int(s2[1])
1731 except ValueError:
1732 r2 = 0
1733 return r1 - r2
1734
1735
1736 if match1.group(10):
1737 r1 = int(match1.group(10))
1738 else:
1739 r1 = 0
1740 if match2.group(10):
1741 r2 = int(match2.group(10))
1742 else:
1743 r2 = 0
1744 return r1 - r2
1745
1747 '''
1748 @description: compare two lists composed by [version,tag,revision] and [version,tag,revision]
1749 if listA > listB --> positive number
1750 if listA == listB --> 0
1751 if listA < listB --> negative number
1752 @input package: listA[version,tag,rev] and listB[version,tag,rev]
1753 @output: integer number
1754 '''
1755
1756
1757 rc = 0
1758 if listA[1] and listB[1]:
1759 rc = cmp(listA[1],listB[1])
1760 if rc == 0:
1761 rc = compare_versions(listA[0],listB[0])
1762
1763 if rc == 0:
1764
1765 if listA[1] > listB[1]:
1766 return 1
1767 elif listA[1] < listB[1]:
1768 return -1
1769 else:
1770
1771 if listA[2] > listB[2]:
1772 return 1
1773 elif listA[2] < listB[2]:
1774 return -1
1775 else:
1776 return 0
1777 return rc
1778
1780 '''
1781 @description: reorder a version list
1782 @input versionlist: a list
1783 @output: the ordered list
1784 '''
1785 rc = compare_versions(a,b)
1786 if rc < 0: return -1
1787 elif rc > 0: return 1
1788 else: return 0
1790 return sorted(versions,g_n_w_cmp,reverse = True)
1791
1793
1794 if len(versions) == 1:
1795 return versions
1796
1797 versionlist = versions[:]
1798
1799 rc = False
1800 while not rc:
1801 change = False
1802 for x in range(len(versionlist)):
1803 pkgA = versionlist[x]
1804 try:
1805 pkgB = versionlist[x+1]
1806 except:
1807 pkgB = "0"
1808 result = compare_versions(pkgA,pkgB)
1809 if result < 0:
1810 versionlist[x] = pkgB
1811 versionlist[x+1] = pkgA
1812 change = True
1813 if not change:
1814 rc = True
1815
1816 return versionlist
1817
1818
1820 '''
1821 @description: reorder a version list
1822 @input versionlist: a list
1823 @output: the ordered list
1824 '''
1825 rc = entropy_compare_versions(a,b)
1826 if rc < 0: return -1
1827 elif rc > 0: return 1
1828 else: return 0
1829
1831 return sorted(versions,g_e_n_w_cmp,reverse = True)
1832
1834 '''
1835 descendent order
1836 versions = [(version,tag,revision),(version,tag,revision)]
1837 '''
1838 if len(versions) == 1:
1839 return versions
1840
1841 myversions = versions[:]
1842
1843
1844 rc = False
1845 while not rc:
1846 change = False
1847 for x in range(len(myversions)):
1848 pkgA = myversions[x]
1849 try:
1850 pkgB = myversions[x+1]
1851 except:
1852 pkgB = ("0","",0)
1853 result = entropy_compare_versions(pkgA,pkgB)
1854 if result < 0:
1855 myversions[x] = pkgB
1856 myversions[x+1] = pkgA
1857 change = True
1858 if not change:
1859 rc = True
1860
1861 return myversions
1862
1863
1865 try:
1866 t = int(x)
1867 del t
1868 return True
1869 except:
1870 return False
1871
1872
1873 -def istextfile(filename, blocksize = 512):
1874 f = open(filename)
1875 r = istext(f.read(blocksize))
1876 f.close()
1877 return r
1878
1880 import string
1881 _null_trans = string.maketrans("", "")
1882 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
1883
1884 if "\0" in s:
1885 return False
1886
1887 if not s:
1888 return True
1889
1890
1891
1892 t = s.translate(_null_trans, text_characters)
1893
1894
1895
1896 if len(t)/len(s) > 0.30:
1897 return False
1898 return True
1899
1900
1901
1902
1904 mydata = {}
1905 return [mydata.setdefault(e,e) for e in alist if e not in mydata]
1906
1907
1908
1909 mappings = {
1910 "'":"''",
1911 '"':'""',
1912 ' ':'+'
1913 }
1914
1916 arg_lst = []
1917 if len(args)==1:
1918 return escape_single(args[0])
1919 for x in args:
1920 arg_lst.append(escape_single(x))
1921 return tuple(arg_lst)
1922
1924 if type(x)==type(()) or type(x)==type([]):
1925 return escape(x)
1926 if type(x)==type(""):
1927 tmpstr=''
1928 for d in range(len(x)):
1929 if x[d] in mappings.keys():
1930 if x[d] in ("'", '"'):
1931 if d+1<len(x):
1932 if x[d+1]!=x[d]:
1933 tmpstr+=mappings[x[d]]
1934 else:
1935 tmpstr+=mappings[x[d]]
1936 else:
1937 tmpstr+=mappings[x[d]]
1938 else:
1939 tmpstr+=x[d]
1940 else:
1941 tmpstr=x
1942 return tmpstr
1943
1945 if type(val)==type(""):
1946 tmpstr=''
1947 for key,item in mappings.items():
1948 val=val.replace(item,key)
1949 tmpstr = val
1950 else:
1951 tmpstr=val
1952 return tmpstr
1953
1955 arg_lst = []
1956 for x in args:
1957 arg_lst.append(unescape(x))
1958 return tuple(arg_lst)
1959
1961 myuri = spliturl(uri)[1]
1962
1963 myuri = myuri.split("@")[len(myuri.split("@"))-1]
1964 return myuri
1965
1967 import urlparse
1968 return urlparse.urlsplit(url)
1969
1971 cmd = "cd \""+pathtocompress+"\" && tar cjf \""+storepath+"\" " + \
1972 ". &> /dev/null"
1973 return subprocess.call(cmd, shell = True)
1974
1976
1977 uid = kwds.get('spf_uid')
1978 if uid != None: kwds.pop('spf_uid')
1979
1980 gid = kwds.get('spf_gid')
1981 if gid != None: kwds.pop('spf_gid')
1982
1983 write_pid_func = kwds.get('write_pid_func')
1984 if write_pid_func != None:
1985 kwds.pop('write_pid_func')
1986
1987 try:
1988 import cPickle as pickle
1989 except ImportError:
1990 import pickle
1991 pread, pwrite = os.pipe()
1992 pid = os.fork()
1993 if pid > 0:
1994 if write_pid_func != None:
1995 write_pid_func(pid)
1996 os.close(pwrite)
1997 f = os.fdopen(pread, 'rb')
1998 status, result = pickle.load(f)
1999 os.waitpid(pid, 0)
2000 f.close()
2001 if status == 0:
2002 return result
2003 else:
2004 raise result
2005 else:
2006 os.close(pread)
2007 if gid != None:
2008 os.setgid(gid)
2009 if uid != None:
2010 os.setuid(uid)
2011 try:
2012 result = f(*args, **kwds)
2013 status = 0
2014 except Exception, exc:
2015 result = exc
2016 status = 1
2017 f = os.fdopen(pwrite, 'wb')
2018 try:
2019 pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
2020 except pickle.PicklingError, exc:
2021 pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
2022 f.close()
2023 os._exit(0)
2024
2025
2027
2028 if extractPath == None:
2029 extractPath = os.path.dirname(filepath)
2030 if not os.path.isfile(filepath):
2031 raise FileNotFound('FileNotFound: archive does not exist')
2032
2033 try:
2034 tar = tarfile.open(filepath,"r")
2035 except tarfile.ReadError:
2036 if catchEmpty:
2037 return 0
2038 raise
2039 except EOFError:
2040 return -1
2041
2042 def fix_uid_gid(tarinfo, epath):
2043
2044 uname = tarinfo.uname
2045 gname = tarinfo.gname
2046 ugdata_valid = False
2047 try:
2048 int(gname)
2049 int(uname)
2050 except ValueError:
2051 ugdata_valid = True
2052 try:
2053 if ugdata_valid:
2054
2055
2056 uid, gid = get_uid_from_user(uname), \
2057 get_gid_from_group(gname)
2058 os.lchown(epath, uid, gid)
2059 except OSError:
2060 pass
2061
2062 def mycmp(a,b):
2063 return cmp(a[0].name, b[0].name)
2064
2065 try:
2066
2067 encoded_path = extractPath.encode('utf-8')
2068 def mymf(tarinfo):
2069 epath = os.path.join(encoded_path, tarinfo.name)
2070 if tarinfo.isdir():
2071
2072
2073 try:
2074 os.makedirs(epath, 0777)
2075 except EnvironmentError:
2076 pass
2077 return tarinfo, epath
2078
2079 tar.extract(tarinfo, encoded_path)
2080
2081 del tar.members[:]
2082 return tarinfo, epath
2083
2084 entries = sorted(map(mymf, tar), mycmp, reverse = True)
2085
2086
2087 def mymf2(tardata):
2088 tarinfo, epath = tardata
2089 try:
2090
2091 tar.chown(tarinfo, epath)
2092 fix_uid_gid(tarinfo, epath)
2093 tar.utime(tarinfo, epath)
2094 mode = tarinfo.mode
2095
2096
2097 if not os.path.islink(epath):
2098 tar.chmod(tarinfo, epath)
2099
2100 except tarfile.ExtractError:
2101 if tar.errorlevel > 1:
2102 raise
2103
2104 done = map(mymf2, entries)
2105 del done
2106
2107 except EOFError:
2108 return -1
2109
2110 finally:
2111 tar.close()
2112
2113 if os.listdir(extractPath):
2114 return 0
2115 return -1
2116
2118 size = str(round(float(bytes)/1024,1))
2119 if bytes < 1024:
2120 size = str(round(float(bytes)))+"b"
2121 elif bytes < 1023999:
2122 size += "kB"
2123 elif bytes > 1023999:
2124 size = str(round(float(size)/1024,1))
2125 size += "MB"
2126 return size
2127
2129 ftppassword = uri.split("@")[:-1]
2130 if not ftppassword: return uri
2131 ftppassword = '@'.join(ftppassword)
2132 ftppassword = ftppassword.split(":")[-1]
2133 if not ftppassword:
2134 return uri
2135 newuri = uri.replace(ftppassword,"xxxxxxxx")
2136 return newuri
2137
2139 ftpuser = ftpuri.split("ftp://")[-1].split(":")[0]
2140 if (ftpuser == ""):
2141 ftpuser = "anonymous@"
2142 ftppassword = "anonymous"
2143 else:
2144 ftppassword = ftpuri.split("@")[:-1]
2145 if len(ftppassword) > 1:
2146 ftppassword = '@'.join(ftppassword)
2147 ftppassword = ftppassword.split(":")[-1]
2148 if (ftppassword == ""):
2149 ftppassword = "anonymous"
2150 else:
2151 ftppassword = ftppassword[0]
2152 ftppassword = ftppassword.split(":")[-1]
2153 if (ftppassword == ""):
2154 ftppassword = "anonymous"
2155
2156 ftpport = ftpuri.split(":")[-1]
2157 try:
2158 ftpport = int(ftpport)
2159 except ValueError:
2160 ftpport = 21
2161
2162 ftpdir = '/'
2163 if ftpuri.count("/") > 2:
2164 ftpdir = ftpuri.split("ftp://")[-1]
2165 ftpdir = ftpdir.split("/")[-1]
2166 ftpdir = ftpdir.split(":")[0]
2167 if ftpdir.endswith("/"):
2168 ftpdir = ftpdir[:len(ftpdir)-1]
2169 if not ftpdir: ftpdir = "/"
2170
2171 return ftpuser, ftppassword, ftpport, ftpdir
2172
2174 return os.path.getmtime(path)
2175
2183
2185 from datetime import datetime
2186
2187 unixtime = os.path.getmtime(path)
2188 humantime = datetime.fromtimestamp(unixtime)
2189
2190 humantime = str(humantime)
2191 outputtime = ""
2192 for char in humantime:
2193 if char != "-" and char != " " and char != ":":
2194 outputtime += char
2195 return outputtime
2196
2198 from datetime import datetime
2199 humantime = str(datetime.fromtimestamp(unixtime))
2200 return humantime
2201
2203 from datetime import datetime
2204 return datetime.fromtimestamp(unixtime)
2205
2208
2210 return time.strftime("%Y")
2211
2213
2214 mysecs = seconds
2215 myminutes = 0
2216 myhours = 0
2217 mydays = 0
2218
2219 while mysecs >= 60:
2220 mysecs -= 60
2221 myminutes += 1
2222
2223 while myminutes >= 60:
2224 myminutes -= 60
2225 myhours += 1
2226
2227 while myhours >= 24:
2228 myhours -= 24
2229 mydays += 1
2230
2231 output = []
2232 output.append(str(mysecs)+"s")
2233 if myminutes > 0 or myhours > 0:
2234 output.append(str(myminutes)+"m")
2235 if myhours > 0 or mydays > 0:
2236 output.append(str(myhours)+"h")
2237 if mydays > 0:
2238 output.append(str(mydays)+"d")
2239 output.reverse()
2240 return ':'.join(output)
2241
2242
2244
2245 if not toCleanDirs:
2246 toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ]
2247 counter = 0
2248
2249 for xdir in toCleanDirs:
2250 print_info(red(" * ")+"Cleaning "+darkgreen(xdir)+" directory...", back = True)
2251 if os.path.isdir(xdir):
2252 dircontent = os.listdir(xdir)
2253 if dircontent != []:
2254 for data in dircontent:
2255 subprocess.call(["rm","-rf",os.path.join(xdir,data)])
2256 counter += 1
2257
2258 print_info(green(" * ")+"Cleaned: "+str(counter)+" files and directories")
2259 return 0
2260
2261 -def flatten(l, ltypes=(list, tuple)):
2262 i = 0
2263 while i < len(l):
2264 while isinstance(l[i], ltypes):
2265 if not l[i]:
2266 l.pop(i)
2267 if not len(l):
2268 break
2269 else:
2270 l[i:i+1] = list(l[i])
2271 i += 1
2272 return l
2273
2275 content = []
2276 if os.path.isfile(etpConst['repositoriesconf']):
2277 f = open(etpConst['repositoriesconf'])
2278 content = f.readlines()
2279 f.close()
2280 return content
2281
2283 content = read_repositories_conf()
2284 content = [x.strip() for x in content]
2285 repolines = [x for x in content if x.startswith("repository|") and (len(x.split("|")) == 5)]
2286 content = [x for x in content if x not in repolines]
2287 for repoid in ordered_repository_list:
2288
2289 for x in repolines:
2290 repoidline = x.split("|")[1]
2291 if repoid == repoidline:
2292 content.append(x)
2293 _save_repositories_content(content)
2294
2296
2297 if repodata['repoid'].endswith(".tbz2"):
2298 return
2299
2300 content = read_repositories_conf()
2301 content = [x.strip() for x in content]
2302 if not disable and not enable:
2303 content = [x for x in content if not x.startswith("repository|"+repodata['repoid'])]
2304 if remove:
2305
2306 content = [x for x in content if not (x.startswith("#") and not x.startswith("##") and (x.find("repository|"+repodata['repoid']) != -1))]
2307 if not remove:
2308
2309 repolines = [x for x in content if x.startswith("repository|") or (x.startswith("#") and not x.startswith("##") and (x.find("repository|") != -1))]
2310 content = [x for x in content if x not in repolines]
2311
2312 repolines = [x for x in repolines if (len(x.split("|")) == 5)]
2313 repolines_data = {}
2314 repocount = 0
2315 for x in repolines:
2316 repolines_data[repocount] = {}
2317 repolines_data[repocount]['repoid'] = x.split("|")[1]
2318 repolines_data[repocount]['line'] = x
2319 if disable and x.split("|")[1] == repodata['repoid']:
2320 if not x.startswith("#"):
2321 x = "#"+x
2322 repolines_data[repocount]['line'] = x
2323 elif enable and x.split("|")[1] == repodata['repoid'] and x.startswith("#"):
2324 repolines_data[repocount]['line'] = x[1:]
2325 repocount += 1
2326
2327 if not disable and not enable:
2328
2329 line = "repository|%s|%s|%s|%s#%s#%s,%s" % ( repodata['repoid'],
2330 repodata['description'],
2331 ' '.join(repodata['plain_packages']),
2332 repodata['plain_database'],
2333 repodata['dbcformat'],
2334 repodata['service_port'],
2335 repodata['ssl_service_port'],
2336 )
2337
2338
2339 to_remove = set()
2340 for cc in repolines_data:
2341 if repolines_data[cc]['line'].startswith("#") and \
2342 (repolines_data[cc]['line'].find("repository|"+repodata['repoid']) != -1):
2343
2344 to_remove.add(cc)
2345 for x in to_remove:
2346 del repolines_data[x]
2347
2348 repolines_data[repocount] = {}
2349 repolines_data[repocount]['repoid'] = repodata['repoid']
2350 repolines_data[repocount]['line'] = line
2351
2352
2353 keys = repolines_data.keys()
2354 keys.sort()
2355 for cc in keys:
2356
2357
2358 line = repolines_data[cc]['line']
2359 content.append(line)
2360
2361 _save_repositories_content(content)
2362
2364 if os.path.isfile(etpConst['repositoriesconf']):
2365 if os.path.isfile(etpConst['repositoriesconf']+".old"):
2366 os.remove(etpConst['repositoriesconf']+".old")
2367 shutil.copy2(etpConst['repositoriesconf'],etpConst['repositoriesconf']+".old")
2368 f = open(etpConst['repositoriesconf'],"w")
2369 for x in content:
2370 f.write(x+"\n")
2371 f.flush()
2372 f.close()
2373
2375
2376
2377 if not os.access(os.path.dirname(config_file),os.W_OK):
2378 return False
2379
2380 content = []
2381 if os.path.isfile(config_file):
2382 f = open(config_file,"r")
2383 content = [x.strip() for x in f.readlines()]
2384 f.close()
2385
2386
2387 config_file_tmp = config_file+".tmp"
2388 f = open(config_file_tmp,"w")
2389 param_found = False
2390 if data:
2391 proposed_line = "%s|%s" % (name,data,)
2392 myreg = re.compile('^(%s)?[|].*$' % (name,))
2393 else:
2394 proposed_line = "# %s|" % (name,)
2395 myreg_rem = re.compile('^(%s)?[|].*$' % (name,))
2396 myreg = re.compile('^#([ \t]+?)?(%s)?[|].*$' % (name,))
2397 new_content = []
2398 for line in content:
2399 if myreg_rem.match(line):
2400 continue
2401 new_content.append(line)
2402 content = new_content
2403
2404 for line in content:
2405 if myreg.match(line):
2406 param_found = True
2407 line = proposed_line
2408 f.write(line+"\n")
2409 if not param_found:
2410 f.write(proposed_line+"\n")
2411 f.flush()
2412 f.close()
2413 shutil.move(config_file_tmp,config_file)
2414 return True
2415
2418
2420 if not os.path.exists(tbz2file):
2421 return False
2422 try:
2423 obj = open(tbz2file, "r")
2424 entry_point = locate_edb(obj)
2425 if entry_point is None:
2426 obj.close()
2427 return False
2428 obj.close()
2429 return True
2430 except (IOError, OSError,):
2431 return False
2432
2434 invalid = [ord(x) for x in string if ord(x) not in xrange(32,127)]
2435 if invalid: return False
2436 return True
2437
2439 try:
2440 os.stat(path)
2441 except OSError:
2442 return False
2443 return True
2444
2446 if re.findall(r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])', myhash):
2447 return True
2448 return False
2449
2450
2452 try:
2453 import cStringIO as stringio
2454 except ImportError:
2455 import StringIO as stringio
2456 return stringio.StringIO()
2457
2459 count = 0
2460 f.seek(count, os.SEEK_END)
2461 size = f.tell()
2462 while count > (size*-1):
2463 count -= 1
2464 f.seek(count, os.SEEK_END)
2465 myc = f.read(1)
2466 if myc == "\n":
2467 break
2468 f.seek(count+1, os.SEEK_END)
2469 pos = f.tell()
2470 f.truncate(pos)
2471
2473 import struct
2474 f = open(elf_file,"rb")
2475 f.seek(4)
2476 elf_class = f.read(1)
2477 f.close()
2478 elf_class = struct.unpack('B',elf_class)[0]
2479 return elf_class
2480
2482 import struct
2483 f = open(elf_file,"rb")
2484 data = f.read(4)
2485 f.close()
2486 try:
2487 data = struct.unpack('BBBB',data)
2488 except struct.error:
2489 return False
2490 if data == (127, 69, 76, 70):
2491 return True
2492 return False
2493
2494 readelf_avail_check = False
2495 ldd_avail_check = False
2496
2497
2505
2513
2514
2515
2517 global readelf_avail_check
2518 if not readelf_avail_check:
2519 if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
2520 raise FileNotFound('FileNotFound: no readelf')
2521 readelf_avail_check = True
2522 data = [x.strip().split()[-1][1:-1].split(":") for x in getstatusoutput('readelf -d %s' % (elf_file,))[1].split("\n") if not ((x.find("(RPATH)") == -1) and (x.find("(RUNPATH)") == -1))]
2523 mypaths = []
2524 for mypath in data:
2525 for xpath in mypath:
2526 xpath = xpath.replace("$ORIGIN",os.path.dirname(elf_file))
2527 mypaths.append(xpath)
2528 return mypaths
2529
2531 from xml.dom import minidom
2532 doc = minidom.Document()
2533 ugc = doc.createElement("entropy")
2534 for key, value in dictionary.items():
2535 item = doc.createElement('item')
2536 item.setAttribute('value',key)
2537 if isinstance(value,str):
2538 mytype = "str"
2539 elif isinstance(value,unicode):
2540 mytype = "unicode"
2541 elif isinstance(value,list):
2542 mytype = "list"
2543 elif isinstance(value,set):
2544 mytype = "set"
2545 elif isinstance(value,frozenset):
2546 mytype = "frozenset"
2547 elif isinstance(value,dict):
2548 mytype = "dict"
2549 elif isinstance(value,tuple):
2550 mytype = "tuple"
2551 elif isinstance(value,int):
2552 mytype = "int"
2553 elif isinstance(value,float):
2554 mytype = "float"
2555 elif value == None:
2556 mytype = "None"
2557 value = "None"
2558 else: raise TypeError
2559 item.setAttribute('type',mytype)
2560 item_value = doc.createTextNode("%s" % (value,))
2561 item.appendChild(item_value)
2562 ugc.appendChild(item)
2563 doc.appendChild(ugc)
2564 return doc.toxml()
2565
2567 from xml.dom import minidom
2568 doc = minidom.parseString(xml_string)
2569 entropies = doc.getElementsByTagName("entropy")
2570 if not entropies: return {}
2571 entropy = entropies[0]
2572 items = entropy.getElementsByTagName('item')
2573
2574 my_map = {
2575 "str": str,
2576 "unicode": unicode,
2577 "list": list,
2578 "set": set,
2579 "frozenset": frozenset,
2580 "dict": dict,
2581 "tuple": tuple,
2582 "int": int,
2583 "float": float,
2584 "None": None,
2585 }
2586
2587 mydict = {}
2588 for item in items:
2589 key = item.getAttribute('value')
2590 if not key: continue
2591 mytype = item.getAttribute('type')
2592 mytype_m = my_map.get(mytype)
2593 if mytype_m == None: raise TypeError
2594 try:
2595 data = item.firstChild.data
2596 except AttributeError:
2597 data = ''
2598 if mytype in ("list","set","frozenset","dict","tuple",):
2599 if data:
2600 if data[0] not in ("(","[","s","{",): data = ''
2601 mydict[key] = eval(data)
2602 elif mytype == "None":
2603 mydict[key] = None
2604 else:
2605 mydict[key] = mytype_m(data)
2606 return mydict
2607
2609 from xml.dom import minidom
2610 doc = minidom.Document()
2611 ugc = doc.createElement("entropy")
2612 for key, value in dictionary.items():
2613 item = doc.createElement('item')
2614 item.setAttribute('value',key)
2615 item_value = doc.createTextNode(value)
2616 item.appendChild(item_value)
2617 ugc.appendChild(item)
2618 doc.appendChild(ugc)
2619 return doc.toxml()
2620
2622 from xml.dom import minidom
2623 doc = minidom.parseString(xml_string)
2624 entropies = doc.getElementsByTagName("entropy")
2625 if not entropies:
2626 return {}
2627 entropy = entropies[0]
2628 items = entropy.getElementsByTagName('item')
2629 mydict = {}
2630 for item in items:
2631 key = item.getAttribute('value')
2632 if not key: continue
2633 try:
2634 data = item.firstChild.data
2635 except AttributeError:
2636 data = ''
2637 mydict[key] = data
2638 return mydict
2639
2641 if package_tag:
2642 package_tag = "#%s" % (package_tag,)
2643 else:
2644 package_tag = ''
2645
2646 package_name = "%s:%s-%s" % (category, name, version,)
2647 package_name += package_tag
2648 package_name += etpConst['packagesext']
2649 return package_name
2650
2652 if package_tag:
2653 package_tag = "#%s" % (package_tag,)
2654 else:
2655 package_tag = ''
2656 package_name = "%s/%s-%s" % (category,name,version,)
2657 package_name += package_tag
2658 return package_name
2659
2661 f = open(filepath,"r")
2662 items = set()
2663 line = f.readline()
2664 while line:
2665 x = line.strip().rsplit("#",1)[0]
2666 if x and (not x.startswith('#')):
2667 items.add(x)
2668 line = f.readline()
2669 f.close()
2670 return items
2671
2673
2674 ldpaths = []
2675 try:
2676 f = open(etpConst['systemroot']+"/etc/ld.so.conf","r")
2677 paths = f.readlines()
2678 for path in paths:
2679 path = path.strip()
2680 if path:
2681 if path[0] == "/":
2682 ldpaths.append(os.path.normpath(path))
2683 f.close()
2684 except (IOError,OSError,TypeError,ValueError,IndexError,):
2685 pass
2686
2687
2688 if "/lib" not in ldpaths:
2689 ldpaths.append("/lib")
2690 if "/usr/lib" not in ldpaths:
2691 ldpaths.append("/usr/lib")
2692
2693 return ldpaths
2694
2696 path = set()
2697 paths = os.getenv("PATH")
2698 if paths != None:
2699 paths = set(paths.split(":"))
2700 path |= paths
2701 return path
2702
2704 mynewlist = []
2705 for item in mylist:
2706 try:
2707 mynewlist.append(item.decode("utf-8"))
2708 except UnicodeDecodeError:
2709 try:
2710 mynewlist.append(item.decode("latin1").decode("utf-8"))
2711 except:
2712 raise
2713 return mynewlist
2714