1
2
3 '''
4 # DESCRIPTION:
5 # generic tools for all the handlers applications
6
7 Copyright (C) 2007-2008 Fabio Erculiani
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 '''
23
24 from __future__ import with_statement
25 import random
26 import stat
27 import errno
28 import re
29 import os
30 import time
31 import shutil
32 import tarfile
33 import subprocess
34 import grp
35 import pwd
36 import hashlib
37 from entropy.output import *
38 from entropy.const import *
39 from entropy.exceptions import *
40
43
45
46 if uid == None:
47 uid = os.getuid()
48 if uid == 0:
49 return True
50
51 try:
52 username = pwd.getpwuid(uid)[0]
53 except KeyError:
54 return False
55
56 try:
57 data = grp.getgrnam(etpConst['sysgroup'])
58 except KeyError:
59 return False
60
61
62 etp_group_users = data[3]
63
64 if not etp_group_users or \
65 username not in etp_group_users:
66 return False
67
68 return True
69
71 try:
72 return pwd.getpwnam(username)[2]
73 except (KeyError, IndexError,):
74 return -1
75
77 try:
78 return grp.getgrnam(groupname)[2]
79 except (KeyError, IndexError,):
80 return -1
81
83 try:
84 return pwd.getpwuid(uid)[0]
85 except KeyError:
86 return None
87
89 try:
90 return grp.getgrgid(gid)[0]
91 except (KeyError, IndexError,):
92 return -1
93
96
98 import traceback
99 traceback.print_exc(file = f)
100
102 import traceback
103 from cStringIO import StringIO
104 buf = StringIO()
105 traceback.print_exc(file = buf)
106 return buf.getvalue()
107
109 import traceback
110 if not returndata: traceback.print_exc()
111 data = []
112 tb = sys.exc_info()[2]
113 while 1:
114 if not tb.tb_next:
115 break
116 tb = tb.tb_next
117 stack = []
118 stack.append(tb.tb_frame)
119
120 for frame in stack:
121 if not returndata:
122 print
123 print "Frame %s in %s at line %s" % (frame.f_code.co_name,
124 frame.f_code.co_filename,
125 frame.f_lineno)
126 else:
127 data.append("Frame %s in %s at line %s" % (frame.f_code.co_name,
128 frame.f_code.co_filename,
129 frame.f_lineno))
130 for key, value in frame.f_locals.items():
131 if not returndata:
132 print "\t%20s = " % key,
133 else:
134 data.append("\t%20s = " % key,)
135 try:
136 if not returndata:
137 print value
138 else:
139 data.append(value)
140 except:
141 if not returndata: print "<ERROR WHILE PRINTING VALUE>"
142 return data
143
144
145
146
148
149 import socket
150 import urllib2
151 socket.setdefaulttimeout(60)
152
153 from entropy.core import SystemSettings
154 sys_settings = SystemSettings()
155 proxy_settings = sys_settings['system']['proxy']
156 try:
157 mydict = {}
158 if proxy_settings['ftp']:
159 mydict['ftp'] = proxy_settings['ftp']
160 if proxy_settings['http']:
161 mydict['http'] = proxy_settings['http']
162 if mydict:
163 mydict['username'] = proxy_settings['username']
164 mydict['password'] = proxy_settings['password']
165 add_proxy_opener(urllib2, mydict)
166 else:
167
168 urllib2._opener = None
169 item = urllib2.urlopen(url)
170 result = item.readlines()
171 item.close()
172 del item
173 if (not result):
174 socket.setdefaulttimeout(2)
175 return False
176 socket.setdefaulttimeout(2)
177 return result
178 except:
179 socket.setdefaulttimeout(2)
180 return False
181
183 f = open(path,"r")
184 x = f.read(4)
185 if x == '\x89PNG':
186 return True
187 return False
188
190 f = open(path,"r")
191 x = f.read(10)
192 if x == '\xff\xd8\xff\xe0\x00\x10JFIF':
193 return True
194 return False
195
197 f = open(path,"r")
198 x = f.read(2)
199 if x == 'BM':
200 return True
201 return False
202
204 f = open(path,"r")
205 x = f.read(5)
206 if x == 'GIF89':
207 return True
208 return False
209
215
217 import types
218 if type(module) != types.ModuleType:
219 raise InvalidDataType("InvalidDataType: not a module")
220 if not data:
221 return
222
223 username = None
224 password = None
225 authinfo = None
226 if data.has_key('password'):
227 username = data.pop('username')
228 if data.has_key('password'):
229 username = data.pop('password')
230 if username == None or password == None:
231 username = None
232 password = None
233 else:
234 passmgr = module.HTTPPasswordMgrWithDefaultRealm()
235 if data['http']:
236 passmgr.add_password(None, data['http'], username, password)
237 if data['ftp']:
238 passmgr.add_password(None, data['ftp'], username, password)
239 authinfo = module.ProxyBasicAuthHandler(passmgr)
240
241 proxy_support = module.ProxyHandler(data)
242 if authinfo:
243 opener = module.build_opener(proxy_support, authinfo)
244 else:
245 opener = module.build_opener(proxy_support)
246 module.install_opener(opener)
247
249 try:
250 mystring = str(string)
251 del mystring
252 except:
253 return False
254 return True
255
257 try:
258 unicode(string)
259 except:
260 return False
261 return True
262
264 monster = "(?:[a-z0-9!#$%&'*+/=?^_{|}~-]+(?:.[a-z0-9!#$%" + \
265 "&'*+/=?^_{|}~-]+)*|\"(?:" + \
266 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]" + \
267 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*\")@(?:(?:[a-z0-9]" + \
268 "(?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" + \
269 "|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.)" + \
270 "{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?" + \
271 "|[a-z0-9-]*[a-z0-9]:(?:" + \
272 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]" + \
273 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"
274 evil = re.compile(monster)
275 if evil.match(email):
276 return True
277 return False
278
281
283 mystat = os.lstat(file_path)
284 return int(mystat.st_size)
285
287 size = 0
288 for myfile in file_list:
289 try:
290 size += get_file_size(myfile)
291 except (OSError,IOError,):
292 continue
293 return size
294
296 import statvfs
297 st = os.statvfs(mountpoint)
298 freeblocks = st[statvfs.F_BFREE]
299 blocksize = st[statvfs.F_BSIZE]
300 freespace = freeblocks*blocksize
301 if bytes_required > freespace:
302
303 return False
304 return True
305
307 """Return (status, output) of executing cmd in a shell."""
308 pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
309 text = pipe.read()
310 sts = pipe.close()
311 if sts is None: sts = 0
312 if text[-1:] == '\n': text = text[:-1]
313 return sts, text
314
315
316
317
318
319
320 -def movefile(src, dest, src_basedir = None):
321
322 sstat = os.lstat(src)
323 destexists = 1
324 try:
325 dstat = os.lstat(dest)
326 except (OSError, IOError,):
327 dstat = os.lstat(os.path.dirname(dest))
328 destexists = 0
329
330 if destexists:
331 if stat.S_ISLNK(dstat[stat.ST_MODE]):
332 try:
333 os.unlink(dest)
334 destexists = 0
335 except (OSError, IOError,):
336 pass
337
338 if stat.S_ISLNK(sstat[stat.ST_MODE]):
339 try:
340 target = os.readlink(src)
341 if src_basedir != None:
342 if target.find(src_basedir) == 0:
343 target = target[len(src_basedir):]
344 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
345 os.unlink(dest)
346 os.symlink(target,dest)
347 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
348 return True
349 except SystemExit:
350 raise
351 except Exception, e:
352 print "!!! failed to properly create symlink:"
353 print "!!!",dest,"->",target
354 print "!!!",e
355 return False
356
357 renamefailed = True
358 if sstat.st_dev == dstat.st_dev:
359 try:
360 os.rename(src,dest)
361 renamefailed = False
362 except Exception, e:
363 if e[0] != errno.EXDEV:
364
365 print "!!! Failed to move",src,"to",dest
366 print "!!!",e
367 return False
368
369
370 if renamefailed:
371 didcopy = True
372 if stat.S_ISREG(sstat[stat.ST_MODE]):
373 try:
374 while 1:
375 tmp_dest = "%s#entropy_new_%s" % (dest,get_random_number(),)
376 if not os.path.lexists(tmp_dest): break
377 shutil.copyfile(src,tmp_dest)
378 os.rename(tmp_dest,dest)
379 didcopy = True
380 except SystemExit, e:
381 raise
382 except Exception, e:
383 print '!!! copy',src,'->',dest,'failed.'
384 print "!!!",e
385 return False
386 else:
387
388 a = getstatusoutput("mv -f '%s' '%s'" % (src,dest,))
389 if a[0]!=0:
390 print "!!! Failed to move special file:"
391 print "!!! '"+src+"' to '"+dest+"'"
392 print "!!!",a
393 return False
394 try:
395 if didcopy:
396 if stat.S_ISLNK(sstat[stat.ST_MODE]):
397 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
398 else:
399 os.chown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
400 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE]))
401 os.unlink(src)
402 except SystemExit, e:
403 raise
404 except Exception, e:
405 print "!!! Failed to chown/chmod/unlink in movefile()"
406 print "!!!",dest
407 print "!!!",e
408 return False
409
410 try:
411 os.utime(dest, (sstat.st_atime, sstat.st_mtime))
412 except OSError:
413
414
415 try:
416 long(os.stat(dest).st_mtime)
417 return True
418 except OSError, e:
419 print "!!! Failed to stat in movefile()\n"
420 print "!!! %s\n" % dest
421 print "!!! %s\n" % str(e)
422 return False
423
424 return True
425
427 mycount = count
428 while mycount > 0:
429 os.system("sleep 0.35; echo -ne \"\a\"; sleep 0.35")
430 mycount -= 1
431
433 if etpConst['applicationlock']:
434 if not gentle:
435 raise SystemExit(10)
436 return True
437 return False
438
440 try:
441 return abs(hash(os.urandom(2)))%99999
442 except NotImplementedError:
443 random.seed()
444 return random.randint(10000,99999)
445
446 -def countdown(secs=5, what="Counting...", back = False):
447 if secs:
448 if back:
449 try:
450 print red(">>"), what,
451 except UnicodeEncodeError:
452 print red(">>"),what.encode('utf-8'),
453 else:
454 try:
455 print what
456 except UnicodeEncodeError:
457 print what.encode('utf-8')
458 for i in range(secs)[::-1]:
459 sys.stdout.write(red(str(i+1)+" "))
460 sys.stdout.flush()
461 time.sleep(1)
462
464 m = hashlib.md5()
465 readfile = open(filepath)
466 block = readfile.read(1024)
467 while block:
468 m.update(block)
469 block = readfile.read(1024)
470 readfile.close()
471 return m.hexdigest()
472
474 m = hashlib.sha512()
475 readfile = open(filepath)
476 block = readfile.read(1024)
477 while block:
478 m.update(block)
479 block = readfile.read(1024)
480 readfile.close()
481 return m.hexdigest()
482
484 m = hashlib.sha256()
485 readfile = open(filepath)
486 block = readfile.read(1024)
487 while block:
488 m.update(block)
489 block = readfile.read(1024)
490 readfile.close()
491 return m.hexdigest()
492
494 m = hashlib.sha1()
495 readfile = open(filepath)
496 block = readfile.read(1024)
497 while block:
498 m.update(block)
499 block = readfile.read(1024)
500 readfile.close()
501 return m.hexdigest()
502
504 if not os.path.isdir(directory):
505 raise DirectoryNotFound("DirectoryNotFound: directory just does not exist.")
506 myfiles = os.listdir(directory)
507 m = hashlib.md5()
508 if not myfiles:
509 if get_obj:
510 return m
511 return "0"
512
513 for currentdir,subdirs,files in os.walk(directory):
514 for myfile in files:
515 myfile = os.path.join(currentdir,myfile)
516 readfile = open(myfile)
517 block = readfile.read(1024)
518 while block:
519 m.update(block)
520 block = readfile.read(1024)
521 readfile.close()
522 if get_obj:
523 return m
524 return m.hexdigest()
525
526
527
528 -def getfd(filespec, readOnly = 0):
529 import types
530 if type(filespec) == types.IntType:
531 return filespec
532 if filespec == None:
533 filespec = "/dev/null"
534
535 flags = os.O_RDWR | os.O_CREAT
536 if (readOnly):
537 flags = os.O_RDONLY
538 return os.open(filespec, flags)
539
541 f_out = open(destination_path,"wb")
542 f_in = opener(file_path,"rb")
543 data = f_in.read(8192)
544 while data:
545 f_out.write(data)
546 data = f_in.read(8192)
547 f_out.flush()
548 f_out.close()
549 f_in.close()
550
551 -def compress_file(file_path, destination_path, opener, compress_level = None):
552 f_in = open(file_path,"rb")
553 if compress_level != None:
554 f_out = opener(destination_path,"wb",compresslevel = compress_level)
555 else:
556 f_out = opener(destination_path,"wb")
557 data = f_in.read(8192)
558 while data:
559 f_out.write(data)
560 data = f_in.read(8192)
561 if hasattr(f_out,'flush'):
562 f_out.flush()
563 f_out.close()
564 f_in.close()
565
566
568
569 if compressor not in ("bz2","gz",):
570 raise AttributeError("invalid compressor specified")
571
572 id_strings = {}
573 tar = tarfile.open(dest_file,"w:%s" % (compressor,))
574 try:
575 for path in files_to_compress:
576 exist = os.lstat(path)
577 tarinfo = tar.gettarinfo(path, os.path.basename(path))
578 tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
579 tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))
580 if not stat.S_ISREG(exist.st_mode): continue
581 tarinfo.type = tarfile.REGTYPE
582 with open(path) as f:
583 tar.addfile(tarinfo, f)
584 finally:
585 tar.close()
586
588
589 try:
590 tar = tarfile.open(compressed_file,"r")
591 except tarfile.ReadError:
592 if catch_empty:
593 return True
594 return False
595 except EOFError:
596 return False
597
598 try:
599
600 dest_path = dest_path.encode('utf-8')
601 def mymf(tarinfo):
602 if tarinfo.isdir():
603
604
605 try:
606 os.makedirs(os.path.join(dest_path, tarinfo.name), 0777)
607 except EnvironmentError:
608 pass
609 return tarinfo
610 tar.extract(tarinfo, dest_path)
611 del tar.members[:]
612 return tarinfo
613
614 def mycmp(a,b):
615 return cmp(a.name, b.name)
616
617 directories = sorted(map(mymf, tar), mycmp, reverse = True)
618
619
620 def mymf2(tarinfo):
621 epath = os.path.join(dest_path, tarinfo.name)
622 try:
623 tar.chown(tarinfo, epath)
624
625
626
627 uname = tarinfo.uname
628 gname = tarinfo.gname
629 ugdata_valid = False
630 try:
631 int(gname)
632 int(uname)
633 except ValueError:
634 ugdata_valid = True
635
636 try:
637 if ugdata_valid:
638
639
640 uid, gid = get_uid_from_user(uname), \
641 get_gid_from_group(gname)
642 os.lchown(epath, uid, gid)
643 except OSError:
644 pass
645
646 tar.utime(tarinfo, epath)
647 tar.chmod(tarinfo, epath)
648 except tarfile.ExtractError:
649 if tar.errorlevel > 1:
650 return False
651 done = map(mymf2, directories)
652 del done
653
654 except EOFError:
655 return False
656
657 finally:
658 tar.close()
659
660 return True
661
663 import gzip
664 filepath = gzipfilepath[:-3]
665 item = open(filepath,"wb")
666 filegz = gzip.GzipFile(gzipfilepath,"rb")
667 chunk = filegz.read(8192)
668 while chunk:
669 item.write(chunk)
670 chunk = filegz.read(8192)
671 filegz.close()
672 item.flush()
673 item.close()
674 return filepath
675
677 import bz2
678 filepath = bzip2filepath[:-4]
679 item = open(filepath,"wb")
680 filebz2 = bz2.BZ2File(bzip2filepath,"rb")
681 chunk = filebz2.read(8192)
682 while chunk:
683 item.write(chunk)
684 chunk = filebz2.read(8192)
685 filebz2.close()
686 item.flush()
687 item.close()
688 return filepath
689
691 if os.path.isfile(etpConst['etpdatabaseclientfilepath']):
692 rnd = get_random_number()
693 source = etpConst['etpdatabaseclientfilepath']
694 dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd)
695 shutil.copy2(source,dest)
696 user = os.stat(source)[4]
697 group = os.stat(source)[5]
698 os.chown(dest,user,group)
699 shutil.copystat(source,dest)
700 return dest
701 return ""
702
707
709 xpakpath = suck_xpak(tbz2file, etpConst['entropyunpackdir'])
710 f = open(xpakpath,"rb")
711 data = f.read()
712 f.close()
713 os.remove(xpakpath)
714 return data
715
717 try:
718 import entropy.xpak as xpak
719 if tmpdir is None:
720 tmpdir = etpConst['packagestmpdir']+"/"+os.path.basename(xpakfile)[:-5]+"/"
721 if os.path.isdir(tmpdir):
722 shutil.rmtree(tmpdir,True)
723 os.makedirs(tmpdir)
724 xpakdata = xpak.getboth(xpakfile)
725 xpak.xpand(xpakdata,tmpdir)
726 del xpakdata
727 try:
728 os.remove(xpakfile)
729 except OSError:
730 pass
731 except:
732 return None
733 return tmpdir
734
736
737 dest_filename = os.path.basename(tbz2file)[:-5]+".xpak"
738 xpakpath = os.path.join(outputpath, dest_filename)
739 old = open(tbz2file,"rb")
740
741
742 old.seek(0, os.SEEK_END)
743
744 bytes = old.tell()
745 counter = bytes - 1
746 xpak_end = "XPAKSTOP"
747 xpak_start = "XPAKPACK"
748 xpak_entry_point = "X"
749 xpak_tag_len = len(xpak_start)
750 chunk_len = 3
751 data_start_position = None
752 data_end_position = None
753
754 while counter >= (0 - chunk_len):
755
756 old.seek(counter - bytes, os.SEEK_END)
757 if (bytes - (abs(counter - bytes))) < chunk_len:
758 chunk_len = 1
759 read_bytes = old.read(chunk_len)
760 read_len = len(read_bytes)
761
762 entry_idx = read_bytes.rfind(xpak_entry_point)
763 if entry_idx != -1:
764
765 cut_gotten = read_bytes[entry_idx:]
766 offset = xpak_tag_len - len(cut_gotten)
767 chunk = cut_gotten + old.read(offset)
768
769 if (chunk == xpak_end) and (data_start_position is None):
770 data_end_position = old.tell()
771
772 elif (chunk == xpak_start) and (data_end_position is not None):
773 data_start_position = old.tell() - xpak_tag_len
774 break
775
776 counter -= read_len
777
778 if data_start_position is None:
779 return None
780 if data_end_position is None:
781 return None
782
783
784
785
786 db = open(xpakpath,"wb")
787 old.seek(data_start_position)
788 to_read = data_end_position - data_start_position
789 while to_read > 0:
790 data = old.read(to_read)
791 db.write(data)
792 to_read -= len(data)
793
794 db.flush()
795 db.close()
796 old.close()
797 return xpakpath
798
810
822
824
825 old = open(tbz2file, "rb")
826 if not dbpath:
827 dbpath = tbz2file[:-5] + ".db"
828
829 start_position = locate_edb(old)
830 if not start_position:
831 old.close()
832 try:
833 os.remove(dbpath)
834 except OSError:
835 return None
836 return None
837
838 db = open(dbpath, "wb")
839 data = old.read(1024)
840 while data:
841 db.write(data)
842 data = old.read(1024)
843 db.flush()
844 db.close()
845
846 return dbpath
847
849
850
851 fileobj.seek(0, os.SEEK_END)
852
853 bytes = fileobj.tell()
854 counter = bytes - 1
855
856 db_tag = etpConst['databasestarttag']
857 db_tag_len = len(db_tag)
858 give_up_threshold = 1024000 * 30
859 entry_point = db_tag[::-1][0]
860 max_read_len = 8
861 start_position = None
862
863 while counter >= 0:
864 cur_threshold = abs((counter-bytes))
865 if cur_threshold >= give_up_threshold:
866 start_position = None
867 break
868 fileobj.seek(counter-bytes, os.SEEK_END)
869 read_bytes = fileobj.read(max_read_len)
870 read_len = len(read_bytes)
871 entry_idx = read_bytes.rfind(entry_point)
872 if entry_idx != -1:
873 rollback = (read_len - entry_idx) * -1
874 fileobj.seek(rollback, os.SEEK_CUR)
875 chunk = fileobj.read(db_tag_len)
876 if chunk == db_tag:
877 start_position = fileobj.tell()
878 break
879 counter -= read_len
880
881 return start_position
882
884 old = open(tbz2file, "rb")
885
886 start_position = locate_edb(old)
887 if not start_position:
888 old.close()
889 return None
890
891 new_path = os.path.join(savedir, os.path.basename(tbz2file))
892 new = open(new_path, "wb")
893
894 old.seek(0)
895 counter = 0
896 max_read_len = 1024
897 db_tag = etpConst['databasestarttag']
898 db_tag_len = len(db_tag)
899 start_position -= db_tag_len
900
901 while counter < start_position:
902 delta = start_position - counter
903 if delta < max_read_len:
904 max_read_len = delta
905 bytes = old.read(max_read_len)
906 read_bytes = len(bytes)
907 new.write(bytes)
908 counter += read_bytes
909
910 new.flush()
911 new.close()
912 old.close()
913 return savedir+"/"+os.path.basename(tbz2file)
914
915
917 md5hash = md5sum(filepath)
918 hashfile = filepath+etpConst['packagesmd5fileext']
919 f = open(hashfile,"w")
920 name = os.path.basename(filepath)
921 f.write(md5hash+" "+name+"\n")
922 f.flush()
923 f.close()
924 return hashfile
925
927 sha512hash = sha512(filepath)
928 hashfile = filepath+etpConst['packagessha512fileext']
929 f = open(hashfile,"w")
930 tbz2name = os.path.basename(filepath)
931 f.write(sha512hash+" "+tbz2name+"\n")
932 f.flush()
933 f.close()
934 return hashfile
935
937 sha256hash = sha256(filepath)
938 hashfile = filepath+etpConst['packagessha256fileext']
939 f = open(hashfile,"w")
940 tbz2name = os.path.basename(filepath)
941 f.write(sha256hash+" "+tbz2name+"\n")
942 f.flush()
943 f.close()
944 return hashfile
945
947 sha1hash = sha1(filepath)
948 hashfile = filepath+etpConst['packagessha1fileext']
949 f = open(hashfile,"w")
950 tbz2name = os.path.basename(filepath)
951 f.write(sha1hash+" "+tbz2name+"\n")
952 f.flush()
953 f.close()
954 return hashfile
955
957 checksum = str(checksum)
958 result = md5sum(filepath)
959 result = str(result)
960 if checksum == result:
961 return True
962 return False
963
965 checksum = str(checksum)
966 result = sha512(filepath)
967 result = str(result)
968 if checksum == result:
969 return True
970 return False
971
973 checksum = str(checksum)
974 result = sha256(filepath)
975 result = str(result)
976 if checksum == result:
977 return True
978 return False
979
981 checksum = str(checksum)
982 result = sha1(filepath)
983 result = str(result)
984 if checksum == result:
985 return True
986 return False
987
989 m = hashlib.md5()
990 m.update(string)
991 return m.hexdigest()
992
993
995 sort_dict = {}
996
997 for item in update_list:
998
999 year = item.split("-")[1]
1000 if sort_dict.has_key(year):
1001 sort_dict[year].append(item)
1002 else:
1003 sort_dict[year] = []
1004 sort_dict[year].append(item)
1005 new_list = []
1006 keys = sort_dict.keys()
1007 keys.sort()
1008 for key in keys:
1009 sort_dict[key].sort()
1010 new_list += sort_dict[key]
1011 del sort_dict
1012 return new_list
1013
1015 data = []
1016 if os.access(filepath, os.R_OK | os.F_OK):
1017 gen_f = open(filepath,"r")
1018 content = gen_f.readlines()
1019 gen_f.close()
1020
1021 content = [x.strip().rsplit("#", 1)[0].strip() for x in content \
1022 if not x.startswith("#") and x.strip()]
1023 for line in content:
1024 if line in data:
1025 continue
1026 data.append(line)
1027 return data
1028
1029
1031
1032
1033 if os.path.isfile(file) and os.path.isfile(fromfile):
1034 old = md5sum(fromfile)
1035 new = md5sum(file)
1036 if old == new:
1037 return file, False
1038
1039 counter = -1
1040 newfile = ""
1041 previousfile = ""
1042
1043 while 1:
1044 counter += 1
1045 txtcounter = str(counter)
1046 oldtxtcounter = str(counter-1)
1047 txtcounter_len = 4-len(txtcounter)
1048 cnt = 0
1049 while cnt < txtcounter_len:
1050 txtcounter = "0"+txtcounter
1051 oldtxtcounter = "0"+oldtxtcounter
1052 cnt += 1
1053 newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file)
1054 if counter > 0:
1055 previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file)
1056 else:
1057 previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1058 if not os.path.exists(newfile):
1059 break
1060 if not newfile:
1061 newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1062 else:
1063
1064 if os.path.exists(previousfile):
1065
1066
1067 new = md5sum(fromfile)
1068 old = md5sum(previousfile)
1069 if new == old:
1070 return previousfile, False
1071
1072
1073 new = md5sum(file)
1074 old = md5sum(previousfile)
1075 if (new == old):
1076 return previousfile, False
1077
1078 return newfile, True
1079
1081
1082 logline = False
1083 logoutput = []
1084 f = open(file,"r")
1085 reallog = f.readlines()
1086 f.close()
1087
1088 for line in reallog:
1089 if line.startswith("INFO: postinst") or line.startswith("LOG: postinst"):
1090 logline = True
1091 continue
1092
1093 elif line.startswith("LOG:"):
1094 logline = False
1095 continue
1096 if (logline) and (line.strip()):
1097
1098 logoutput.append(line.strip())
1099 return logoutput
1100
1101
1102
1103
1104
1105 ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\\d*)*)(-r(\\d+))?$")
1106 suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
1107 suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
1108 endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
1109
1110
1112 myparts = mypkg.split('-')
1113 for x in myparts:
1114 if ververify(x):
1115 return 0
1116 return 1
1117
1119
1120 myver = myverx[:]
1121 if myver.endswith("*"):
1122 myver = myver[:-1]
1123 if ver_regexp.match(myver):
1124 return 1
1125 else:
1126 if not silent:
1127 print "!!! syntax error in version: %s" % myver
1128 return 0
1129
1130
1131
1132
1133
1134 valid_category = re.compile("^\w[\w-]*")
1135 invalid_atom_chars_regexp = re.compile("[()|@]")
1136
1138 """
1139 Check to see if a depend atom is valid
1140
1141 Example usage:
1142 >>> isvalidatom('media-libs/test-3.0')
1143 0
1144 >>> isvalidatom('>=media-libs/test-3.0')
1145 1
1146
1147 @param atom: The depend atom to check against
1148 @type atom: String
1149 @rtype: Integer
1150 @return: One of the following:
1151 1) 0 if the atom is invalid
1152 2) 1 if the atom is valid
1153 """
1154 atom = remove_tag(myatom)
1155 atom = remove_usedeps(atom)
1156 if invalid_atom_chars_regexp.search(atom):
1157 return 0
1158 if allow_blockers and atom[:1] == "!":
1159 if atom[1:2] == "!":
1160 atom = atom[2:]
1161 else:
1162 atom = atom[1:]
1163
1164
1165 if atom.count("/") > 1:
1166 return 0
1167
1168 cpv = dep_getcpv(atom)
1169 cpv_catsplit = catsplit(cpv)
1170 mycpv_cps = None
1171 if cpv:
1172 if len(cpv_catsplit) == 2:
1173 if valid_category.match(cpv_catsplit[0]) is None:
1174 return 0
1175 if cpv_catsplit[0] == "null":
1176
1177 mycpv_cps = catpkgsplit(cpv.replace("null/", "cat/", 1))
1178 if mycpv_cps:
1179 mycpv_cps = list(mycpv_cps)
1180 mycpv_cps[0] = "null"
1181 if not mycpv_cps:
1182 mycpv_cps = catpkgsplit(cpv)
1183
1184 operator = get_operator(atom)
1185 if operator:
1186 if operator[0] in "<>" and remove_slot(atom).endswith("*"):
1187 return 0
1188 if mycpv_cps:
1189 if len(cpv_catsplit) == 2:
1190
1191 return 1
1192 else:
1193 return 0
1194 else:
1195
1196 return 0
1197 if mycpv_cps:
1198
1199 return 0
1200
1201 if len(cpv_catsplit) == 2:
1202
1203 return 1
1204 return 0
1205
1207 return mydep.split("/", 1)
1208
1210 """
1211 Return the operator used in a depstring.
1212
1213 Example usage:
1214 >>> from portage.dep import *
1215 >>> get_operator(">=test-1.0")
1216 '>='
1217
1218 @param mydep: The dep string to check
1219 @type mydep: String
1220 @rtype: String
1221 @return: The operator. One of:
1222 '~', '=', '>', '<', '=*', '>=', or '<='
1223 """
1224 if mydep:
1225 mydep = remove_slot(mydep)
1226 if not mydep:
1227 return None
1228 if mydep[0] == "~":
1229 operator = "~"
1230 elif mydep[0] == "=":
1231 if mydep[-1] == "*":
1232 operator = "=*"
1233 else:
1234 operator = "="
1235 elif mydep[0] in "><":
1236 if len(mydep) > 1 and mydep[1] == "=":
1237 operator = mydep[0:2]
1238 else:
1239 operator = mydep[0]
1240 else:
1241 operator = None
1242
1243 return operator
1244
1246 """
1247 Checks to see if the depstring is only the package name (no version parts)
1248
1249 Example usage:
1250 >>> isjustname('media-libs/test-3.0')
1251 0
1252 >>> isjustname('test')
1253 1
1254 >>> isjustname('media-libs/test')
1255 1
1256
1257 @param mypkg: The package atom to check
1258 @param mypkg: String
1259 @rtype: Integer
1260 @return: One of the following:
1261 1) 0 if the package string is not just the package name
1262 2) 1 if it is
1263 """
1264
1265 myparts = mypkg.split('-')
1266 for x in myparts:
1267 if ververify(x):
1268 return 0
1269 return 1
1270
1272 """
1273 Checks to see if a package is in category/package-version or package-version format,
1274 possibly returning a cached result.
1275
1276 Example usage:
1277 >>> isspecific('media-libs/test')
1278 0
1279 >>> isspecific('media-libs/test-3.0')
1280 1
1281
1282 @param mypkg: The package depstring to check against
1283 @type mypkg: String
1284 @rtype: Integer
1285 @return: One of the following:
1286 1) 0 if the package string is not specific
1287 2) 1 if it is
1288 """
1289 mysplit = mypkg.split("/")
1290 if not isjustname(mysplit[-1]):
1291 return 1
1292 return 0
1293
1294
1296 """
1297 Takes a Category/Package-Version-Rev and returns a list of each.
1298
1299 @param mydata: Data to split
1300 @type mydata: string
1301 @param silent: suppress error messages
1302 @type silent: Boolean (integer)
1303 @rype: list
1304 @return:
1305 1. If each exists, it returns [cat, pkgname, version, rev]
1306 2. If cat is not specificed in mydata, cat will be "null"
1307 3. if rev does not exist it will be '-r0'
1308 4. If cat is invalid (specified but has incorrect syntax)
1309 an InvalidData Exception will be thrown
1310 """
1311
1312
1313 mysplit=mydata.split("/")
1314 p_split=None
1315 if len(mysplit)==1:
1316 retval=["null"]
1317 p_split=pkgsplit(mydata,silent=silent)
1318 elif len(mysplit)==2:
1319 retval=[mysplit[0]]
1320 p_split=pkgsplit(mysplit[1],silent=silent)
1321 if not p_split:
1322 return None
1323 retval.extend(p_split)
1324 return retval
1325
1327 myparts=mypkg.split("-")
1328
1329 if len(myparts)<2:
1330 if not silent:
1331 print "!!! Name error in",mypkg+": missing a version or name part."
1332 return None
1333 for x in myparts:
1334 if len(x)==0:
1335 if not silent:
1336 print "!!! Name error in",mypkg+": empty \"-\" part."
1337 return None
1338
1339
1340 revok=0
1341 myrev=myparts[-1]
1342
1343 if len(myrev) and myrev[0]=="r":
1344 try:
1345 int(myrev[1:])
1346 revok=1
1347 except ValueError:
1348 pass
1349 if revok:
1350 verPos = -2
1351 revision = myparts[-1]
1352 else:
1353 verPos = -1
1354 revision = "r0"
1355
1356 if ververify(myparts[verPos]):
1357 if len(myparts)== (-1*verPos):
1358 return None
1359 else:
1360 for x in myparts[:verPos]:
1361 if ververify(x):
1362 return None
1363
1364 myval=["-".join(myparts[:verPos]),myparts[verPos],revision]
1365 return myval
1366 else:
1367 return None
1368
1370 """
1371 Return the category/package-name of a depstring.
1372
1373 Example usage:
1374 >>> dep_getkey('media-libs/test-3.0')
1375 'media-libs/test'
1376
1377 @param mydep: The depstring to retrieve the category/package-name of
1378 @type mydep: String
1379 @rtype: String
1380 @return: The package category/package-version
1381 """
1382 if not mydepx: return mydepx
1383 mydep = mydepx[:]
1384 mydep = remove_tag(mydep)
1385 mydep = remove_usedeps(mydep)
1386
1387 mydep = dep_getcpv(mydep)
1388 if mydep and isspecific(mydep):
1389 mysplit = catpkgsplit(mydep)
1390 if not mysplit:
1391 return mydep
1392 return mysplit[0] + "/" + mysplit[1]
1393
1394 return mydep
1395
1396
1398 """
1399 Return the category-package-version with any operators/slot specifications stripped off
1400
1401 Example usage:
1402 >>> dep_getcpv('>=media-libs/test-3.0')
1403 'media-libs/test-3.0'
1404
1405 @param mydep: The depstring
1406 @type mydep: String
1407 @rtype: String
1408 @return: The depstring with the operator removed
1409 """
1410
1411 if mydep and mydep[0] == "*":
1412 mydep = mydep[1:]
1413 if mydep and mydep[-1] == "*":
1414 mydep = mydep[:-1]
1415 if mydep and mydep[0] == "!":
1416 mydep = mydep[1:]
1417 if mydep[:2] in [">=", "<="]:
1418 mydep = mydep[2:]
1419 elif mydep[:1] in "=<>~":
1420 mydep = mydep[1:]
1421 colon = mydep.rfind(":")
1422 if colon != -1:
1423 mydep = mydep[:colon]
1424
1425 return mydep
1426
1428 """
1429
1430 # Imported from portage.dep
1431 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1432
1433 Retrieve the slot on a depend.
1434
1435 Example usage:
1436 >>> dep_getslot('app-misc/test:3')
1437 '3'
1438
1439 @param mydep: The depstring to retrieve the slot of
1440 @type mydep: String
1441 @rtype: String
1442 @return: The slot
1443 """
1444 colon = mydep.find(":")
1445 if colon != -1:
1446 bracket = mydep.find("[", colon)
1447 if bracket == -1:
1448 return mydep[colon+1:]
1449 else:
1450 return mydep[colon+1:bracket]
1451 return None
1452
1454
1455 """
1456
1457 # Imported from portage.dep
1458 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1459
1460 Pull a listing of USE Dependencies out of a dep atom.
1461
1462 Example usage:
1463 >>> dep_getusedeps('app-misc/test:3[foo,-bar]')
1464 ('foo','-bar')
1465
1466 @param depend: The depstring to process
1467 @type depend: String
1468 @rtype: List
1469 @return: List of use flags ( or [] if no flags exist )
1470 """
1471
1472 use_list = []
1473 open_bracket = depend.find('[')
1474
1475 comma_separated = False
1476 bracket_count = 0
1477 while( open_bracket != -1 ):
1478 bracket_count += 1
1479 if bracket_count > 1:
1480 raise InvalidAtom("USE Dependency with more " + \
1481 "than one set of brackets: %s" % (depend,))
1482 close_bracket = depend.find(']', open_bracket )
1483 if close_bracket == -1:
1484 raise InvalidAtom("USE Dependency with no closing bracket: %s" % depend )
1485 use = depend[open_bracket + 1: close_bracket]
1486
1487 if not use:
1488 raise InvalidAtom("USE Dependency with " + \
1489 "no use flag ([]): %s" % depend )
1490 if not comma_separated:
1491 comma_separated = "," in use
1492
1493 if comma_separated and bracket_count > 1:
1494 raise InvalidAtom("USE Dependency contains a mixture of " + \
1495 "comma and bracket separators: %s" % depend )
1496
1497 if comma_separated:
1498 for x in use.split(","):
1499 if x:
1500 use_list.append(x)
1501 else:
1502 raise InvalidAtom("USE Dependency with no use " + \
1503 "flag next to comma: %s" % depend )
1504 else:
1505 use_list.append(use)
1506
1507
1508 open_bracket = depend.find( '[', open_bracket+1 )
1509
1510 return tuple(use_list)
1511
1513 mydepend = depend[:]
1514
1515 close_bracket = mydepend.find(']')
1516 after_closebracket = ''
1517 if close_bracket != -1: after_closebracket = mydepend[close_bracket+1:]
1518
1519 open_bracket = mydepend.find('[')
1520 if open_bracket != -1: mydepend = mydepend[:open_bracket]
1521
1522 return mydepend+after_closebracket
1523
1525 """
1526
1527 # Imported from portage.dep
1528 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1529
1530 Removes dep components from the right side of an atom:
1531 * slot
1532 * use
1533 * repo
1534 """
1535 colon = mydep.find(":")
1536 if colon != -1:
1537 mydep = mydep[:colon]
1538 else:
1539 bracket = mydep.find("[")
1540 if bracket != -1:
1541 mydep = mydep[:bracket]
1542 return mydep
1543
1544
1546 myver = ver.split("-")
1547 if myver[-1][0] == "r":
1548 return '-'.join(myver[:-1])
1549 return ver
1550
1552 colon = mydep.rfind("#")
1553 if colon == -1:
1554 return mydep
1555 return mydep[:colon]
1556
1564
1566
1567 colon = mydep.rfind("~")
1568 if colon != -1:
1569 myrev = mydep[colon+1:]
1570 try:
1571 myrev = int(myrev)
1572 except ValueError:
1573 return None
1574 return myrev
1575 return None
1576
1577
1578 dep_revmatch = re.compile('^r[0-9]')
1580 myver = mydep.split("-")
1581 myrev = myver[-1]
1582 if dep_revmatch.match(myrev):
1583 return myrev
1584 else:
1585 return "r0"
1586
1587
1589 colon = mydep.rfind("@")
1590 if colon != -1:
1591 mydata = mydep[colon+1:]
1592 mydata = mydata.split(",")
1593 if not mydata:
1594 mydata = None
1595 return mydep[:colon],mydata
1596 else:
1597 return mydep,None
1598
1600
1601 """
1602 Retrieve the slot on a depend.
1603
1604 Example usage:
1605 >>> dep_gettag('app-misc/test#2.6.23-sabayon-r1')
1606 '2.6.23-sabayon-r1'
1607
1608 """
1609
1610 colon = dep.rfind("#")
1611 if colon != -1:
1612 mydep = dep[colon+1:]
1613 rslt = remove_slot(mydep)
1614 return rslt
1615 return None
1616
1618 try:
1619 while atom:
1620 if atom[0] in ('>','<','=','~',):
1621 atom = atom[1:]
1622 continue
1623 break
1624 except IndexError:
1625 pass
1626 return atom
1627
1628
1629
1630
1632
1633 if ver1 == ver2:
1634 return 0
1635
1636 match1 = None
1637 match2 = None
1638 if ver1:
1639 match1 = ver_regexp.match(ver1)
1640 if ver2:
1641 match2 = ver_regexp.match(ver2)
1642
1643
1644 invalid = False
1645 invalid_rc = 0
1646 if not match1:
1647 invalid = True
1648 elif not match1.groups():
1649 invalid = True
1650 elif not match2:
1651 invalid_rc = 1
1652 invalid = True
1653 elif not match2.groups():
1654 invalid_rc = 1
1655 invalid = True
1656 if invalid: return invalid_rc
1657
1658
1659
1660 list1 = [int(match1.group(2))]
1661 list2 = [int(match2.group(2))]
1662
1663
1664 if len(match1.group(3)) or len(match2.group(3)):
1665 vlist1 = match1.group(3)[1:].split(".")
1666 vlist2 = match2.group(3)[1:].split(".")
1667 for i in range(0, max(len(vlist1), len(vlist2))):
1668
1669
1670
1671 if len(vlist1) <= i or len(vlist1[i]) == 0:
1672 list1.append(-1)
1673 list2.append(int(vlist2[i]))
1674 elif len(vlist2) <= i or len(vlist2[i]) == 0:
1675 list1.append(int(vlist1[i]))
1676 list2.append(-1)
1677
1678 elif (vlist1[i][0] != "0" and vlist2[i][0] != "0"):
1679 list1.append(int(vlist1[i]))
1680 list2.append(int(vlist2[i]))
1681
1682 else:
1683 list1.append(float("0."+vlist1[i]))
1684 list2.append(float("0."+vlist2[i]))
1685
1686
1687 if len(match1.group(5)):
1688 list1.append(ord(match1.group(5)))
1689 if len(match2.group(5)):
1690 list2.append(ord(match2.group(5)))
1691
1692 for i in range(0, max(len(list1), len(list2))):
1693 if len(list1) <= i:
1694 return -1
1695 elif len(list2) <= i:
1696 return 1
1697 elif list1[i] != list2[i]:
1698 return list1[i] - list2[i]
1699
1700
1701 list1 = match1.group(6).split("_")[1:]
1702 list2 = match2.group(6).split("_")[1:]
1703
1704 for i in range(0, max(len(list1), len(list2))):
1705 if len(list1) <= i:
1706 s1 = ("p","0")
1707 else:
1708 s1 = suffix_regexp.match(list1[i]).groups()
1709 if len(list2) <= i:
1710 s2 = ("p","0")
1711 else:
1712 s2 = suffix_regexp.match(list2[i]).groups()
1713 if s1[0] != s2[0]:
1714 return suffix_value[s1[0]] - suffix_value[s2[0]]
1715 if s1[1] != s2[1]:
1716
1717
1718 try:
1719 r1 = int(s1[1])
1720 except ValueError:
1721 r1 = 0
1722 try:
1723 r2 = int(s2[1])
1724 except ValueError:
1725 r2 = 0
1726 return r1 - r2
1727
1728
1729 if match1.group(10):
1730 r1 = int(match1.group(10))
1731 else:
1732 r1 = 0
1733 if match2.group(10):
1734 r2 = int(match2.group(10))
1735 else:
1736 r2 = 0
1737 return r1 - r2
1738
1740 '''
1741 @description: compare two lists composed by [version,tag,revision] and [version,tag,revision]
1742 if listA > listB --> positive number
1743 if listA == listB --> 0
1744 if listA < listB --> negative number
1745 @input package: listA[version,tag,rev] and listB[version,tag,rev]
1746 @output: integer number
1747 '''
1748
1749
1750 rc = 0
1751 if listA[1] and listB[1]:
1752 rc = cmp(listA[1],listB[1])
1753 if rc == 0:
1754 rc = compare_versions(listA[0],listB[0])
1755
1756 if rc == 0:
1757
1758 if listA[1] > listB[1]:
1759 return 1
1760 elif listA[1] < listB[1]:
1761 return -1
1762 else:
1763
1764 if listA[2] > listB[2]:
1765 return 1
1766 elif listA[2] < listB[2]:
1767 return -1
1768 else:
1769 return 0
1770 return rc
1771
1773 '''
1774 @description: reorder a version list
1775 @input versionlist: a list
1776 @output: the ordered list
1777 '''
1778 rc = compare_versions(a,b)
1779 if rc < 0: return -1
1780 elif rc > 0: return 1
1781 else: return 0
1783 return sorted(versions,g_n_w_cmp,reverse = True)
1784
1786
1787 if len(versions) == 1:
1788 return versions
1789
1790 versionlist = versions[:]
1791
1792 rc = False
1793 while not rc:
1794 change = False
1795 for x in range(len(versionlist)):
1796 pkgA = versionlist[x]
1797 try:
1798 pkgB = versionlist[x+1]
1799 except:
1800 pkgB = "0"
1801 result = compare_versions(pkgA,pkgB)
1802 if result < 0:
1803 versionlist[x] = pkgB
1804 versionlist[x+1] = pkgA
1805 change = True
1806 if not change:
1807 rc = True
1808
1809 return versionlist
1810
1811
1813 '''
1814 @description: reorder a version list
1815 @input versionlist: a list
1816 @output: the ordered list
1817 '''
1818 rc = entropy_compare_versions(a,b)
1819 if rc < 0: return -1
1820 elif rc > 0: return 1
1821 else: return 0
1822
1824 return sorted(versions,g_e_n_w_cmp,reverse = True)
1825
1827 '''
1828 descendent order
1829 versions = [(version,tag,revision),(version,tag,revision)]
1830 '''
1831 if len(versions) == 1:
1832 return versions
1833
1834 myversions = versions[:]
1835
1836
1837 rc = False
1838 while not rc:
1839 change = False
1840 for x in range(len(myversions)):
1841 pkgA = myversions[x]
1842 try:
1843 pkgB = myversions[x+1]
1844 except:
1845 pkgB = ("0","",0)
1846 result = entropy_compare_versions(pkgA,pkgB)
1847 if result < 0:
1848 myversions[x] = pkgB
1849 myversions[x+1] = pkgA
1850 change = True
1851 if not change:
1852 rc = True
1853
1854 return myversions
1855
1856
1858 try:
1859 t = int(x)
1860 del t
1861 return True
1862 except:
1863 return False
1864
1865
1866 -def istextfile(filename, blocksize = 512):
1867 f = open(filename)
1868 r = istext(f.read(blocksize))
1869 f.close()
1870 return r
1871
1873 import string
1874 _null_trans = string.maketrans("", "")
1875 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
1876
1877 if "\0" in s:
1878 return False
1879
1880 if not s:
1881 return True
1882
1883
1884
1885 t = s.translate(_null_trans, text_characters)
1886
1887
1888
1889 if len(t)/len(s) > 0.30:
1890 return False
1891 return True
1892
1893
1894
1895
1897 mydata = {}
1898 return [mydata.setdefault(e,e) for e in alist if e not in mydata]
1899
1900
1901
1902 mappings = {
1903 "'":"''",
1904 '"':'""',
1905 ' ':'+'
1906 }
1907
1909 arg_lst = []
1910 if len(args)==1:
1911 return escape_single(args[0])
1912 for x in args:
1913 arg_lst.append(escape_single(x))
1914 return tuple(arg_lst)
1915
1917 if type(x)==type(()) or type(x)==type([]):
1918 return escape(x)
1919 if type(x)==type(""):
1920 tmpstr=''
1921 for d in range(len(x)):
1922 if x[d] in mappings.keys():
1923 if x[d] in ("'", '"'):
1924 if d+1<len(x):
1925 if x[d+1]!=x[d]:
1926 tmpstr+=mappings[x[d]]
1927 else:
1928 tmpstr+=mappings[x[d]]
1929 else:
1930 tmpstr+=mappings[x[d]]
1931 else:
1932 tmpstr+=x[d]
1933 else:
1934 tmpstr=x
1935 return tmpstr
1936
1938 if type(val)==type(""):
1939 tmpstr=''
1940 for key,item in mappings.items():
1941 val=val.replace(item,key)
1942 tmpstr = val
1943 else:
1944 tmpstr=val
1945 return tmpstr
1946
1948 arg_lst = []
1949 for x in args:
1950 arg_lst.append(unescape(x))
1951 return tuple(arg_lst)
1952
1954 myuri = spliturl(uri)[1]
1955
1956 myuri = myuri.split("@")[len(myuri.split("@"))-1]
1957 return myuri
1958
1960 import urlparse
1961 return urlparse.urlsplit(url)
1962
1964 cmd = "cd \""+pathtocompress+"\" && tar cjf \""+storepath+"\" " + \
1965 ". &> /dev/null"
1966 return subprocess.call(cmd, shell = True)
1967
1969
1970 uid = kwds.get('spf_uid')
1971 if uid != None: kwds.pop('spf_uid')
1972
1973 gid = kwds.get('spf_gid')
1974 if gid != None: kwds.pop('spf_gid')
1975
1976 write_pid_func = kwds.get('write_pid_func')
1977 if write_pid_func != None:
1978 kwds.pop('write_pid_func')
1979
1980 try:
1981 import cPickle as pickle
1982 except ImportError:
1983 import pickle
1984 pread, pwrite = os.pipe()
1985 pid = os.fork()
1986 if pid > 0:
1987 if write_pid_func != None:
1988 write_pid_func(pid)
1989 os.close(pwrite)
1990 f = os.fdopen(pread, 'rb')
1991 status, result = pickle.load(f)
1992 os.waitpid(pid, 0)
1993 f.close()
1994 if status == 0:
1995 return result
1996 else:
1997 raise result
1998 else:
1999 os.close(pread)
2000 if gid != None:
2001 os.setgid(gid)
2002 if uid != None:
2003 os.setuid(uid)
2004 try:
2005 result = f(*args, **kwds)
2006 status = 0
2007 except Exception, exc:
2008 result = exc
2009 status = 1
2010 f = os.fdopen(pwrite, 'wb')
2011 try:
2012 pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
2013 except pickle.PicklingError, exc:
2014 pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
2015 f.close()
2016 os._exit(0)
2017
2018
2020
2021 if extractPath == None:
2022 extractPath = os.path.dirname(filepath)
2023 if not os.path.isfile(filepath):
2024 raise FileNotFound('FileNotFound: archive does not exist')
2025
2026 try:
2027 tar = tarfile.open(filepath,"r")
2028 except tarfile.ReadError:
2029 if catchEmpty:
2030 return 0
2031 raise
2032 except EOFError:
2033 return -1
2034
2035 def fix_uid_gid(tarinfo, epath):
2036
2037 uname = tarinfo.uname
2038 gname = tarinfo.gname
2039 ugdata_valid = False
2040 try:
2041 int(gname)
2042 int(uname)
2043 except ValueError:
2044 ugdata_valid = True
2045 try:
2046 if ugdata_valid:
2047
2048
2049 uid, gid = get_uid_from_user(uname), \
2050 get_gid_from_group(gname)
2051 os.lchown(epath, uid, gid)
2052 except OSError:
2053 pass
2054
2055 def mycmp(a,b):
2056 return cmp(a[0].name, b[0].name)
2057
2058 try:
2059
2060 encoded_path = extractPath.encode('utf-8')
2061 def mymf(tarinfo):
2062 epath = os.path.join(encoded_path, tarinfo.name)
2063 if tarinfo.isdir():
2064
2065
2066 try:
2067 os.makedirs(epath, 0777)
2068 except EnvironmentError:
2069 pass
2070 return tarinfo, epath
2071
2072 tar.extract(tarinfo, encoded_path)
2073
2074 del tar.members[:]
2075 return tarinfo, epath
2076
2077 entries = sorted(map(mymf, tar), mycmp, reverse = True)
2078
2079
2080 def mymf2(tardata):
2081 tarinfo, epath = tardata
2082 try:
2083
2084 tar.chown(tarinfo, epath)
2085 fix_uid_gid(tarinfo, epath)
2086 tar.utime(tarinfo, epath)
2087 mode = tarinfo.mode
2088
2089
2090 if not os.path.islink(epath):
2091 tar.chmod(tarinfo, epath)
2092
2093 except tarfile.ExtractError:
2094 if tar.errorlevel > 1:
2095 raise
2096
2097 done = map(mymf2, entries)
2098 del done
2099
2100 except EOFError:
2101 return -1
2102
2103 finally:
2104 tar.close()
2105
2106 if os.listdir(extractPath):
2107 return 0
2108 return -1
2109
2111 size = str(round(float(bytes)/1024,1))
2112 if bytes < 1024:
2113 size = str(round(float(bytes)))+"b"
2114 elif bytes < 1023999:
2115 size += "kB"
2116 elif bytes > 1023999:
2117 size = str(round(float(size)/1024,1))
2118 size += "MB"
2119 return size
2120
2122 ftppassword = uri.split("@")[:-1]
2123 if not ftppassword: return uri
2124 ftppassword = '@'.join(ftppassword)
2125 ftppassword = ftppassword.split(":")[-1]
2126 if not ftppassword:
2127 return uri
2128 newuri = uri.replace(ftppassword,"xxxxxxxx")
2129 return newuri
2130
2132 ftpuser = ftpuri.split("ftp://")[-1].split(":")[0]
2133 if (ftpuser == ""):
2134 ftpuser = "anonymous@"
2135 ftppassword = "anonymous"
2136 else:
2137 ftppassword = ftpuri.split("@")[:-1]
2138 if len(ftppassword) > 1:
2139 ftppassword = '@'.join(ftppassword)
2140 ftppassword = ftppassword.split(":")[-1]
2141 if (ftppassword == ""):
2142 ftppassword = "anonymous"
2143 else:
2144 ftppassword = ftppassword[0]
2145 ftppassword = ftppassword.split(":")[-1]
2146 if (ftppassword == ""):
2147 ftppassword = "anonymous"
2148
2149 ftpport = ftpuri.split(":")[-1]
2150 try:
2151 ftpport = int(ftpport)
2152 except ValueError:
2153 ftpport = 21
2154
2155 ftpdir = '/'
2156 if ftpuri.count("/") > 2:
2157 ftpdir = ftpuri.split("ftp://")[-1]
2158 ftpdir = ftpdir.split("/")[-1]
2159 ftpdir = ftpdir.split(":")[0]
2160 if ftpdir.endswith("/"):
2161 ftpdir = ftpdir[:len(ftpdir)-1]
2162 if not ftpdir: ftpdir = "/"
2163
2164 return ftpuser, ftppassword, ftpport, ftpdir
2165
2167 return os.path.getmtime(path)
2168
2176
2178 from datetime import datetime
2179
2180 unixtime = os.path.getmtime(path)
2181 humantime = datetime.fromtimestamp(unixtime)
2182
2183 humantime = str(humantime)
2184 outputtime = ""
2185 for char in humantime:
2186 if char != "-" and char != " " and char != ":":
2187 outputtime += char
2188 return outputtime
2189
2191 from datetime import datetime
2192 humantime = str(datetime.fromtimestamp(unixtime))
2193 return humantime
2194
2196 from datetime import datetime
2197 return datetime.fromtimestamp(unixtime)
2198
2201
2203 return time.strftime("%Y")
2204
2206
2207 mysecs = seconds
2208 myminutes = 0
2209 myhours = 0
2210 mydays = 0
2211
2212 while mysecs >= 60:
2213 mysecs -= 60
2214 myminutes += 1
2215
2216 while myminutes >= 60:
2217 myminutes -= 60
2218 myhours += 1
2219
2220 while myhours >= 24:
2221 myhours -= 24
2222 mydays += 1
2223
2224 output = []
2225 output.append(str(mysecs)+"s")
2226 if myminutes > 0 or myhours > 0:
2227 output.append(str(myminutes)+"m")
2228 if myhours > 0 or mydays > 0:
2229 output.append(str(myhours)+"h")
2230 if mydays > 0:
2231 output.append(str(mydays)+"d")
2232 output.reverse()
2233 return ':'.join(output)
2234
2235
2237
2238 if not toCleanDirs:
2239 toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ]
2240 counter = 0
2241
2242 for xdir in toCleanDirs:
2243 print_info(red(" * ")+"Cleaning "+darkgreen(xdir)+" directory...", back = True)
2244 if os.path.isdir(xdir):
2245 dircontent = os.listdir(xdir)
2246 if dircontent != []:
2247 for data in dircontent:
2248 subprocess.call(["rm","-rf",os.path.join(xdir,data)])
2249 counter += 1
2250
2251 print_info(green(" * ")+"Cleaned: "+str(counter)+" files and directories")
2252 return 0
2253
2254 -def flatten(l, ltypes=(list, tuple)):
2255 i = 0
2256 while i < len(l):
2257 while isinstance(l[i], ltypes):
2258 if not l[i]:
2259 l.pop(i)
2260 if not len(l):
2261 break
2262 else:
2263 l[i:i+1] = list(l[i])
2264 i += 1
2265 return l
2266
2268 content = []
2269 if os.path.isfile(etpConst['repositoriesconf']):
2270 f = open(etpConst['repositoriesconf'])
2271 content = f.readlines()
2272 f.close()
2273 return content
2274
2276 content = read_repositories_conf()
2277 content = [x.strip() for x in content]
2278 repolines = [x for x in content if x.startswith("repository|") and (len(x.split("|")) == 5)]
2279 content = [x for x in content if x not in repolines]
2280 for repoid in ordered_repository_list:
2281
2282 for x in repolines:
2283 repoidline = x.split("|")[1]
2284 if repoid == repoidline:
2285 content.append(x)
2286 _save_repositories_content(content)
2287
2289
2290 if repodata['repoid'].endswith(".tbz2"):
2291 return
2292
2293 content = read_repositories_conf()
2294 content = [x.strip() for x in content]
2295 if not disable and not enable:
2296 content = [x for x in content if not x.startswith("repository|"+repodata['repoid'])]
2297 if remove:
2298
2299 content = [x for x in content if not (x.startswith("#") and not x.startswith("##") and (x.find("repository|"+repodata['repoid']) != -1))]
2300 if not remove:
2301
2302 repolines = [x for x in content if x.startswith("repository|") or (x.startswith("#") and not x.startswith("##") and (x.find("repository|") != -1))]
2303 content = [x for x in content if x not in repolines]
2304
2305 repolines = [x for x in repolines if (len(x.split("|")) == 5)]
2306 repolines_data = {}
2307 repocount = 0
2308 for x in repolines:
2309 repolines_data[repocount] = {}
2310 repolines_data[repocount]['repoid'] = x.split("|")[1]
2311 repolines_data[repocount]['line'] = x
2312 if disable and x.split("|")[1] == repodata['repoid']:
2313 if not x.startswith("#"):
2314 x = "#"+x
2315 repolines_data[repocount]['line'] = x
2316 elif enable and x.split("|")[1] == repodata['repoid'] and x.startswith("#"):
2317 repolines_data[repocount]['line'] = x[1:]
2318 repocount += 1
2319
2320 if not disable and not enable:
2321
2322 line = "repository|%s|%s|%s|%s#%s#%s,%s" % ( repodata['repoid'],
2323 repodata['description'],
2324 ' '.join(repodata['plain_packages']),
2325 repodata['plain_database'],
2326 repodata['dbcformat'],
2327 repodata['service_port'],
2328 repodata['ssl_service_port'],
2329 )
2330
2331
2332 to_remove = set()
2333 for cc in repolines_data:
2334 if repolines_data[cc]['line'].startswith("#") and \
2335 (repolines_data[cc]['line'].find("repository|"+repodata['repoid']) != -1):
2336
2337 to_remove.add(cc)
2338 for x in to_remove:
2339 del repolines_data[x]
2340
2341 repolines_data[repocount] = {}
2342 repolines_data[repocount]['repoid'] = repodata['repoid']
2343 repolines_data[repocount]['line'] = line
2344
2345
2346 keys = repolines_data.keys()
2347 keys.sort()
2348 for cc in keys:
2349
2350
2351 line = repolines_data[cc]['line']
2352 content.append(line)
2353
2354 _save_repositories_content(content)
2355
2357 if os.path.isfile(etpConst['repositoriesconf']):
2358 if os.path.isfile(etpConst['repositoriesconf']+".old"):
2359 os.remove(etpConst['repositoriesconf']+".old")
2360 shutil.copy2(etpConst['repositoriesconf'],etpConst['repositoriesconf']+".old")
2361 f = open(etpConst['repositoriesconf'],"w")
2362 for x in content:
2363 f.write(x+"\n")
2364 f.flush()
2365 f.close()
2366
2368
2369
2370 if not os.access(os.path.dirname(config_file),os.W_OK):
2371 return False
2372
2373 content = []
2374 if os.path.isfile(config_file):
2375 f = open(config_file,"r")
2376 content = [x.strip() for x in f.readlines()]
2377 f.close()
2378
2379
2380 config_file_tmp = config_file+".tmp"
2381 f = open(config_file_tmp,"w")
2382 param_found = False
2383 if data:
2384 proposed_line = "%s|%s" % (name,data,)
2385 myreg = re.compile('^(%s)?[|].*$' % (name,))
2386 else:
2387 proposed_line = "# %s|" % (name,)
2388 myreg_rem = re.compile('^(%s)?[|].*$' % (name,))
2389 myreg = re.compile('^#([ \t]+?)?(%s)?[|].*$' % (name,))
2390 new_content = []
2391 for line in content:
2392 if myreg_rem.match(line):
2393 continue
2394 new_content.append(line)
2395 content = new_content
2396
2397 for line in content:
2398 if myreg.match(line):
2399 param_found = True
2400 line = proposed_line
2401 f.write(line+"\n")
2402 if not param_found:
2403 f.write(proposed_line+"\n")
2404 f.flush()
2405 f.close()
2406 shutil.move(config_file_tmp,config_file)
2407 return True
2408
2411
2413 if not os.path.exists(tbz2file):
2414 return False
2415 try:
2416 obj = open(tbz2file, "r")
2417 entry_point = locate_edb(obj)
2418 if entry_point is None:
2419 obj.close()
2420 return False
2421 obj.close()
2422 return True
2423 except (IOError, OSError,):
2424 return False
2425
2427 invalid = [ord(x) for x in string if ord(x) not in xrange(32,127)]
2428 if invalid: return False
2429 return True
2430
2432 try:
2433 os.stat(path)
2434 except OSError:
2435 return False
2436 return True
2437
2439 if re.findall(r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])', myhash):
2440 return True
2441 return False
2442
2443
2445 try:
2446 import cStringIO as stringio
2447 except ImportError:
2448 import StringIO as stringio
2449 return stringio.StringIO()
2450
2452 count = 0
2453 f.seek(count, os.SEEK_END)
2454 size = f.tell()
2455 while count > (size*-1):
2456 count -= 1
2457 f.seek(count, os.SEEK_END)
2458 myc = f.read(1)
2459 if myc == "\n":
2460 break
2461 f.seek(count+1, os.SEEK_END)
2462 pos = f.tell()
2463 f.truncate(pos)
2464
2466 import struct
2467 f = open(elf_file,"rb")
2468 f.seek(4)
2469 elf_class = f.read(1)
2470 f.close()
2471 elf_class = struct.unpack('B',elf_class)[0]
2472 return elf_class
2473
2475 import struct
2476 f = open(elf_file,"rb")
2477 data = f.read(4)
2478 f.close()
2479 try:
2480 data = struct.unpack('BBBB',data)
2481 except struct.error:
2482 return False
2483 if data == (127, 69, 76, 70):
2484 return True
2485 return False
2486
2487 readelf_avail_check = False
2488 ldd_avail_check = False
2489
2490
2498
2506
2507
2508
2510 global readelf_avail_check
2511 if not readelf_avail_check:
2512 if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
2513 raise FileNotFound('FileNotFound: no readelf')
2514 readelf_avail_check = True
2515 data = [x.strip().split()[-1][1:-1].split(":") for x in getstatusoutput('readelf -d %s' % (elf_file,))[1].split("\n") if not ((x.find("(RPATH)") == -1) and (x.find("(RUNPATH)") == -1))]
2516 mypaths = []
2517 for mypath in data:
2518 for xpath in mypath:
2519 xpath = xpath.replace("$ORIGIN",os.path.dirname(elf_file))
2520 mypaths.append(xpath)
2521 return mypaths
2522
2524 from xml.dom import minidom
2525 doc = minidom.Document()
2526 ugc = doc.createElement("entropy")
2527 for key, value in dictionary.items():
2528 item = doc.createElement('item')
2529 item.setAttribute('value',key)
2530 if isinstance(value,str):
2531 mytype = "str"
2532 elif isinstance(value,unicode):
2533 mytype = "unicode"
2534 elif isinstance(value,list):
2535 mytype = "list"
2536 elif isinstance(value,set):
2537 mytype = "set"
2538 elif isinstance(value,frozenset):
2539 mytype = "frozenset"
2540 elif isinstance(value,dict):
2541 mytype = "dict"
2542 elif isinstance(value,tuple):
2543 mytype = "tuple"
2544 elif isinstance(value,int):
2545 mytype = "int"
2546 elif isinstance(value,float):
2547 mytype = "float"
2548 elif value == None:
2549 mytype = "None"
2550 value = "None"
2551 else: raise TypeError
2552 item.setAttribute('type',mytype)
2553 item_value = doc.createTextNode("%s" % (value,))
2554 item.appendChild(item_value)
2555 ugc.appendChild(item)
2556 doc.appendChild(ugc)
2557 return doc.toxml()
2558
2560 from xml.dom import minidom
2561 doc = minidom.parseString(xml_string)
2562 entropies = doc.getElementsByTagName("entropy")
2563 if not entropies: return {}
2564 entropy = entropies[0]
2565 items = entropy.getElementsByTagName('item')
2566
2567 my_map = {
2568 "str": str,
2569 "unicode": unicode,
2570 "list": list,
2571 "set": set,
2572 "frozenset": frozenset,
2573 "dict": dict,
2574 "tuple": tuple,
2575 "int": int,
2576 "float": float,
2577 "None": None,
2578 }
2579
2580 mydict = {}
2581 for item in items:
2582 key = item.getAttribute('value')
2583 if not key: continue
2584 mytype = item.getAttribute('type')
2585 mytype_m = my_map.get(mytype)
2586 if mytype_m == None: raise TypeError
2587 try:
2588 data = item.firstChild.data
2589 except AttributeError:
2590 data = ''
2591 if mytype in ("list","set","frozenset","dict","tuple",):
2592 if data:
2593 if data[0] not in ("(","[","s","{",): data = ''
2594 mydict[key] = eval(data)
2595 elif mytype == "None":
2596 mydict[key] = None
2597 else:
2598 mydict[key] = mytype_m(data)
2599 return mydict
2600
2602 from xml.dom import minidom
2603 doc = minidom.Document()
2604 ugc = doc.createElement("entropy")
2605 for key, value in dictionary.items():
2606 item = doc.createElement('item')
2607 item.setAttribute('value',key)
2608 item_value = doc.createTextNode(value)
2609 item.appendChild(item_value)
2610 ugc.appendChild(item)
2611 doc.appendChild(ugc)
2612 return doc.toxml()
2613
2615 from xml.dom import minidom
2616 doc = minidom.parseString(xml_string)
2617 entropies = doc.getElementsByTagName("entropy")
2618 if not entropies:
2619 return {}
2620 entropy = entropies[0]
2621 items = entropy.getElementsByTagName('item')
2622 mydict = {}
2623 for item in items:
2624 key = item.getAttribute('value')
2625 if not key: continue
2626 try:
2627 data = item.firstChild.data
2628 except AttributeError:
2629 data = ''
2630 mydict[key] = data
2631 return mydict
2632
2634 if package_tag:
2635 package_tag = "#%s" % (package_tag,)
2636 else:
2637 package_tag = ''
2638
2639 package_name = "%s:%s-%s" % (category, name, version,)
2640 package_name += package_tag
2641 package_name += etpConst['packagesext']
2642 return package_name
2643
2645 if package_tag:
2646 package_tag = "#%s" % (package_tag,)
2647 else:
2648 package_tag = ''
2649 package_name = "%s/%s-%s" % (category,name,version,)
2650 package_name += package_tag
2651 return package_name
2652
2654 f = open(filepath,"r")
2655 items = set()
2656 line = f.readline()
2657 while line:
2658 x = line.strip().rsplit("#",1)[0]
2659 if x and (not x.startswith('#')):
2660 items.add(x)
2661 line = f.readline()
2662 f.close()
2663 return items
2664
2666
2667 ldpaths = []
2668 try:
2669 f = open(etpConst['systemroot']+"/etc/ld.so.conf","r")
2670 paths = f.readlines()
2671 for path in paths:
2672 path = path.strip()
2673 if path:
2674 if path[0] == "/":
2675 ldpaths.append(os.path.normpath(path))
2676 f.close()
2677 except (IOError,OSError,TypeError,ValueError,IndexError,):
2678 pass
2679
2680
2681 if "/lib" not in ldpaths:
2682 ldpaths.append("/lib")
2683 if "/usr/lib" not in ldpaths:
2684 ldpaths.append("/usr/lib")
2685
2686 return ldpaths
2687
2689 path = set()
2690 paths = os.getenv("PATH")
2691 if paths != None:
2692 paths = set(paths.split(":"))
2693 path |= paths
2694 return path
2695
2697 mynewlist = []
2698 for item in mylist:
2699 try:
2700 mynewlist.append(item.decode("utf-8"))
2701 except UnicodeDecodeError:
2702 try:
2703 mynewlist.append(item.decode("latin1").decode("utf-8"))
2704 except:
2705 raise
2706 return mynewlist
2707