1
2
3 '''
4 # DESCRIPTION:
5 # generic tools for all the handlers applications
6
7 Copyright (C) 2007-2008 Fabio Erculiani
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 '''
23
24 from __future__ import with_statement
25 import random
26 import stat
27 import errno
28 import re
29 import os
30 import threading
31 import time
32 import shutil
33 import tarfile
34 import subprocess
35 import grp
36 import pwd
37 import hashlib
38 from entropy.output import *
39 from entropy.const import *
40 from entropy.exceptions import *
41
44
46
47 if uid == None:
48 uid = os.getuid()
49 if uid == 0:
50 return True
51
52 try:
53 username = pwd.getpwuid(uid)[0]
54 except KeyError:
55 return False
56
57 try:
58 data = grp.getgrnam(etpConst['sysgroup'])
59 except KeyError:
60 return False
61
62
63 etp_group_users = data[3]
64
65 if not etp_group_users or \
66 username not in etp_group_users:
67 return False
68
69 return True
70
72 try:
73 return pwd.getpwnam(username)[2]
74 except (KeyError, IndexError,):
75 return -1
76
78 try:
79 return grp.getgrnam(groupname)[2]
80 except (KeyError, IndexError,):
81 return -1
82
84 try:
85 return pwd.getpwuid(uid)[0]
86 except KeyError:
87 return None
88
90 try:
91 return grp.getgrgid(gid)[0]
92 except (KeyError, IndexError,):
93 return -1
94
97
99 import traceback
100 traceback.print_exc(file = f)
101
103 import traceback
104 from cStringIO import StringIO
105 buf = StringIO()
106 traceback.print_exc(file = buf)
107 return buf.getvalue()
108
110 import traceback
111 if not returndata: traceback.print_exc()
112 data = []
113 tb = sys.exc_info()[2]
114 while 1:
115 if not tb.tb_next:
116 break
117 tb = tb.tb_next
118 stack = []
119 stack.append(tb.tb_frame)
120
121 for frame in stack:
122 if not returndata:
123 print
124 print "Frame %s in %s at line %s" % (frame.f_code.co_name,
125 frame.f_code.co_filename,
126 frame.f_lineno)
127 else:
128 data.append("Frame %s in %s at line %s" % (frame.f_code.co_name,
129 frame.f_code.co_filename,
130 frame.f_lineno))
131 for key, value in frame.f_locals.items():
132 if not returndata:
133 print "\t%20s = " % key,
134 else:
135 data.append("\t%20s = " % key,)
136 try:
137 if not returndata:
138 print value
139 else:
140 data.append(value)
141 except:
142 if not returndata: print "<ERROR WHILE PRINTING VALUE>"
143 return data
144
145
146
147
149
150 import socket
151 import urllib2
152 socket.setdefaulttimeout(60)
153
154 from entropy.core import SystemSettings
155 sys_settings = SystemSettings()
156 proxy_settings = sys_settings['system']['proxy']
157 try:
158 mydict = {}
159 if proxy_settings['ftp']:
160 mydict['ftp'] = proxy_settings['ftp']
161 if proxy_settings['http']:
162 mydict['http'] = proxy_settings['http']
163 if mydict:
164 mydict['username'] = proxy_settings['username']
165 mydict['password'] = proxy_settings['password']
166 add_proxy_opener(urllib2, mydict)
167 else:
168
169 urllib2._opener = None
170 item = urllib2.urlopen(url)
171 result = item.readlines()
172 item.close()
173 del item
174 if (not result):
175 socket.setdefaulttimeout(2)
176 return False
177 socket.setdefaulttimeout(2)
178 return result
179 except:
180 socket.setdefaulttimeout(2)
181 return False
182
184 f = open(path,"r")
185 x = f.read(4)
186 if x == '\x89PNG':
187 return True
188 return False
189
191 f = open(path,"r")
192 x = f.read(10)
193 if x == '\xff\xd8\xff\xe0\x00\x10JFIF':
194 return True
195 return False
196
198 f = open(path,"r")
199 x = f.read(2)
200 if x == 'BM':
201 return True
202 return False
203
205 f = open(path,"r")
206 x = f.read(5)
207 if x == 'GIF89':
208 return True
209 return False
210
216
218 import types
219 if type(module) != types.ModuleType:
220 raise InvalidDataType("InvalidDataType: not a module")
221 if not data:
222 return
223
224 username = None
225 password = None
226 authinfo = None
227 if data.has_key('password'):
228 username = data.pop('username')
229 if data.has_key('password'):
230 username = data.pop('password')
231 if username == None or password == None:
232 username = None
233 password = None
234 else:
235 passmgr = module.HTTPPasswordMgrWithDefaultRealm()
236 if data['http']:
237 passmgr.add_password(None, data['http'], username, password)
238 if data['ftp']:
239 passmgr.add_password(None, data['ftp'], username, password)
240 authinfo = module.ProxyBasicAuthHandler(passmgr)
241
242 proxy_support = module.ProxyHandler(data)
243 if authinfo:
244 opener = module.build_opener(proxy_support, authinfo)
245 else:
246 opener = module.build_opener(proxy_support)
247 module.install_opener(opener)
248
250 try:
251 mystring = str(string)
252 del mystring
253 except:
254 return False
255 return True
256
258 try:
259 unicode(string)
260 except:
261 return False
262 return True
263
265 monster = "(?:[a-z0-9!#$%&'*+/=?^_{|}~-]+(?:.[a-z0-9!#$%" + \
266 "&'*+/=?^_{|}~-]+)*|\"(?:" + \
267 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]" + \
268 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*\")@(?:(?:[a-z0-9]" + \
269 "(?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" + \
270 "|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.)" + \
271 "{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?" + \
272 "|[a-z0-9-]*[a-z0-9]:(?:" + \
273 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]" + \
274 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"
275 evil = re.compile(monster)
276 if evil.match(email):
277 return True
278 return False
279
282
284 mystat = os.lstat(file_path)
285 return int(mystat.st_size)
286
288 size = 0
289 for myfile in file_list:
290 try:
291 size += get_file_size(myfile)
292 except (OSError,IOError,):
293 continue
294 return size
295
297 import statvfs
298 st = os.statvfs(mountpoint)
299 freeblocks = st[statvfs.F_BFREE]
300 blocksize = st[statvfs.F_BSIZE]
301 freespace = freeblocks*blocksize
302 if bytes_required > freespace:
303
304 return False
305 return True
306
308 """Return (status, output) of executing cmd in a shell."""
309 pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
310 text = pipe.read()
311 sts = pipe.close()
312 if sts is None: sts = 0
313 if text[-1:] == '\n': text = text[:-1]
314 return sts, text
315
316
317
318
319
320
321 -def movefile(src, dest, src_basedir = None):
322
323 sstat = os.lstat(src)
324 destexists = 1
325 try:
326 dstat = os.lstat(dest)
327 except (OSError, IOError,):
328 dstat = os.lstat(os.path.dirname(dest))
329 destexists = 0
330
331 if destexists:
332 if stat.S_ISLNK(dstat[stat.ST_MODE]):
333 try:
334 os.unlink(dest)
335 destexists = 0
336 except (OSError, IOError,):
337 pass
338
339 if stat.S_ISLNK(sstat[stat.ST_MODE]):
340 try:
341 target = os.readlink(src)
342 if src_basedir != None:
343 if target.find(src_basedir) == 0:
344 target = target[len(src_basedir):]
345 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
346 os.unlink(dest)
347 os.symlink(target,dest)
348 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
349 return True
350 except SystemExit:
351 raise
352 except Exception, e:
353 print "!!! failed to properly create symlink:"
354 print "!!!",dest,"->",target
355 print "!!!",e
356 return False
357
358 renamefailed = True
359 if sstat.st_dev == dstat.st_dev:
360 try:
361 os.rename(src,dest)
362 renamefailed = False
363 except Exception, e:
364 if e[0] != errno.EXDEV:
365
366 print "!!! Failed to move",src,"to",dest
367 print "!!!",e
368 return False
369
370
371 if renamefailed:
372 didcopy = True
373 if stat.S_ISREG(sstat[stat.ST_MODE]):
374 try:
375 while 1:
376 tmp_dest = "%s#entropy_new_%s" % (dest,get_random_number(),)
377 if not os.path.lexists(tmp_dest): break
378 shutil.copyfile(src,tmp_dest)
379 os.rename(tmp_dest,dest)
380 didcopy = True
381 except SystemExit, e:
382 raise
383 except Exception, e:
384 print '!!! copy',src,'->',dest,'failed.'
385 print "!!!",e
386 return False
387 else:
388
389 a = getstatusoutput("mv -f '%s' '%s'" % (src,dest,))
390 if a[0]!=0:
391 print "!!! Failed to move special file:"
392 print "!!! '"+src+"' to '"+dest+"'"
393 print "!!!",a
394 return False
395 try:
396 if didcopy:
397 if stat.S_ISLNK(sstat[stat.ST_MODE]):
398 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
399 else:
400 os.chown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
401 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE]))
402 os.unlink(src)
403 except SystemExit, e:
404 raise
405 except Exception, e:
406 print "!!! Failed to chown/chmod/unlink in movefile()"
407 print "!!!",dest
408 print "!!!",e
409 return False
410
411 try:
412 os.utime(dest, (sstat.st_atime, sstat.st_mtime))
413 except OSError:
414
415
416 try:
417 long(os.stat(dest).st_mtime)
418 return True
419 except OSError, e:
420 print "!!! Failed to stat in movefile()\n"
421 print "!!! %s\n" % dest
422 print "!!! %s\n" % str(e)
423 return False
424
425 return True
426
428 mycount = count
429 while mycount > 0:
430 os.system("sleep 0.35; echo -ne \"\a\"; sleep 0.35")
431 mycount -= 1
432
434 if etpConst['applicationlock']:
435 if not silent:
436 print_error(red("Another instance of Equo is running. Action: ")+bold(str(option))+red(" denied."))
437 print_error(red("If I am lying (maybe). Please remove ")+bold(etpConst['pidfile']))
438 if not gentle:
439 raise SystemExit(10)
440 else:
441 return True
442 return False
443
445 try:
446 return abs(hash(os.urandom(2)))%99999
447 except NotImplementedError:
448 random.seed()
449 return random.randint(10000,99999)
450
451 -def countdown(secs=5, what="Counting...", back = False):
452 if secs:
453 if back:
454 try:
455 print red(">>"), what,
456 except UnicodeEncodeError:
457 print red(">>"),what.encode('utf-8'),
458 else:
459 try:
460 print what
461 except UnicodeEncodeError:
462 print what.encode('utf-8')
463 for i in range(secs)[::-1]:
464 sys.stdout.write(red(str(i+1)+" "))
465 sys.stdout.flush()
466 time.sleep(1)
467
469 m = hashlib.md5()
470 readfile = open(filepath)
471 block = readfile.read(1024)
472 while block:
473 m.update(block)
474 block = readfile.read(1024)
475 readfile.close()
476 return m.hexdigest()
477
479 m = hashlib.sha512()
480 readfile = open(filepath)
481 block = readfile.read(1024)
482 while block:
483 m.update(block)
484 block = readfile.read(1024)
485 readfile.close()
486 return m.hexdigest()
487
489 m = hashlib.sha256()
490 readfile = open(filepath)
491 block = readfile.read(1024)
492 while block:
493 m.update(block)
494 block = readfile.read(1024)
495 readfile.close()
496 return m.hexdigest()
497
499 m = hashlib.sha1()
500 readfile = open(filepath)
501 block = readfile.read(1024)
502 while block:
503 m.update(block)
504 block = readfile.read(1024)
505 readfile.close()
506 return m.hexdigest()
507
509 if not os.path.isdir(directory):
510 raise DirectoryNotFound("DirectoryNotFound: directory just does not exist.")
511 myfiles = os.listdir(directory)
512 m = hashlib.md5()
513 if not myfiles:
514 if get_obj:
515 return m
516 return "0"
517
518 for currentdir,subdirs,files in os.walk(directory):
519 for myfile in files:
520 myfile = os.path.join(currentdir,myfile)
521 readfile = open(myfile)
522 block = readfile.read(1024)
523 while block:
524 m.update(block)
525 block = readfile.read(1024)
526 readfile.close()
527 if get_obj:
528 return m
529 return m.hexdigest()
530
531
532
533 -def getfd(filespec, readOnly = 0):
534 import types
535 if type(filespec) == types.IntType:
536 return filespec
537 if filespec == None:
538 filespec = "/dev/null"
539
540 flags = os.O_RDWR | os.O_CREAT
541 if (readOnly):
542 flags = os.O_RDONLY
543 return os.open(filespec, flags)
544
546 f_out = open(destination_path,"wb")
547 f_in = opener(file_path,"rb")
548 data = f_in.read(8192)
549 while data:
550 f_out.write(data)
551 data = f_in.read(8192)
552 f_out.flush()
553 f_out.close()
554 f_in.close()
555
556 -def compress_file(file_path, destination_path, opener, compress_level = None):
557 f_in = open(file_path,"rb")
558 if compress_level != None:
559 f_out = opener(destination_path,"wb",compresslevel = compress_level)
560 else:
561 f_out = opener(destination_path,"wb")
562 data = f_in.read(8192)
563 while data:
564 f_out.write(data)
565 data = f_in.read(8192)
566 if hasattr(f_out,'flush'):
567 f_out.flush()
568 f_out.close()
569 f_in.close()
570
571
573
574 if compressor not in ("bz2","gz",):
575 raise AttributeError("invalid compressor specified")
576
577 id_strings = {}
578 tar = tarfile.open(dest_file,"w:%s" % (compressor,))
579 try:
580 for path in files_to_compress:
581 exist = os.lstat(path)
582 tarinfo = tar.gettarinfo(path, os.path.basename(path))
583 tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
584 tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))
585 if not stat.S_ISREG(exist.st_mode): continue
586 tarinfo.type = tarfile.REGTYPE
587 with open(path) as f:
588 tar.addfile(tarinfo, f)
589 finally:
590 tar.close()
591
593
594 try:
595 tar = tarfile.open(compressed_file,"r")
596 except tarfile.ReadError:
597 if catch_empty:
598 return True
599 return False
600 except EOFError:
601 return False
602
603 try:
604
605 dest_path = dest_path.encode('utf-8')
606 def mymf(tarinfo):
607 if tarinfo.isdir():
608
609
610 try:
611 os.makedirs(os.path.join(dest_path, tarinfo.name), 0777)
612 except EnvironmentError:
613 pass
614 return tarinfo
615 tar.extract(tarinfo, dest_path)
616 del tar.members[:]
617 return tarinfo
618
619 def mycmp(a,b):
620 return cmp(a.name, b.name)
621
622 directories = sorted(map(mymf, tar), mycmp, reverse = True)
623
624
625 def mymf2(tarinfo):
626 epath = os.path.join(dest_path, tarinfo.name)
627 try:
628 tar.chown(tarinfo, epath)
629
630
631
632 uname = tarinfo.uname
633 gname = tarinfo.gname
634 ugdata_valid = False
635 try:
636 int(gname)
637 int(uname)
638 except ValueError:
639 ugdata_valid = True
640
641 try:
642 if ugdata_valid:
643
644
645 uid, gid = get_uid_from_user(uname), \
646 get_gid_from_group(gname)
647 os.lchown(epath, uid, gid)
648 except OSError:
649 pass
650
651 tar.utime(tarinfo, epath)
652 tar.chmod(tarinfo, epath)
653 except tarfile.ExtractError:
654 if tar.errorlevel > 1:
655 return False
656 done = map(mymf2, directories)
657 del done
658
659 except EOFError:
660 return False
661
662 finally:
663 tar.close()
664
665 return True
666
668 import gzip
669 filepath = gzipfilepath[:-3]
670 item = open(filepath,"wb")
671 filegz = gzip.GzipFile(gzipfilepath,"rb")
672 chunk = filegz.read(8192)
673 while chunk:
674 item.write(chunk)
675 chunk = filegz.read(8192)
676 filegz.close()
677 item.flush()
678 item.close()
679 return filepath
680
682 import bz2
683 filepath = bzip2filepath[:-4]
684 item = open(filepath,"wb")
685 filebz2 = bz2.BZ2File(bzip2filepath,"rb")
686 chunk = filebz2.read(8192)
687 while chunk:
688 item.write(chunk)
689 chunk = filebz2.read(8192)
690 filebz2.close()
691 item.flush()
692 item.close()
693 return filepath
694
696 if os.path.isfile(etpConst['etpdatabaseclientfilepath']):
697 rnd = get_random_number()
698 source = etpConst['etpdatabaseclientfilepath']
699 dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd)
700 shutil.copy2(source,dest)
701 user = os.stat(source)[4]
702 group = os.stat(source)[5]
703 os.chown(dest,user,group)
704 shutil.copystat(source,dest)
705 return dest
706 return ""
707
712
714 xpakpath = suck_xpak(tbz2file, etpConst['entropyunpackdir'])
715 f = open(xpakpath,"rb")
716 data = f.read()
717 f.close()
718 os.remove(xpakpath)
719 return data
720
722 try:
723 import entropy.xpak as xpak
724 if tmpdir is None:
725 tmpdir = etpConst['packagestmpdir']+"/"+os.path.basename(xpakfile)[:-5]+"/"
726 if os.path.isdir(tmpdir):
727 shutil.rmtree(tmpdir,True)
728 os.makedirs(tmpdir)
729 xpakdata = xpak.getboth(xpakfile)
730 xpak.xpand(xpakdata,tmpdir)
731 del xpakdata
732 try:
733 os.remove(xpakfile)
734 except:
735 pass
736 except:
737 return None
738 return tmpdir
739
741
742 xpakpath = outputpath+"/"+os.path.basename(tbz2file)[:-5]+".xpak"
743 old = open(tbz2file,"rb")
744 db = open(xpakpath,"wb")
745 db_tmp = open(xpakpath+".reverse","wb")
746 allowWrite = False
747
748
749 old.seek(0,2)
750
751 bytes = old.tell()
752 counter = bytes
753
754 while counter >= 0:
755 old.seek(counter-bytes,2)
756 byte = old.read(1)
757 if byte == "P" or byte == "K":
758 old.seek(counter-bytes-7,2)
759 chunk = old.read(7)+byte
760 if chunk == "XPAKPACK":
761 allowWrite = False
762 db_tmp.write(chunk[::-1])
763 break
764 elif chunk == "XPAKSTOP":
765 allowWrite = True
766 old.seek(counter-bytes,2)
767 if (allowWrite):
768 db_tmp.write(byte)
769 counter -= 1
770
771 db_tmp.flush()
772 db_tmp.close()
773 db_tmp = open(xpakpath+".reverse","rb")
774
775 db_tmp.seek(0,2)
776 bytes = db_tmp.tell()
777 counter = bytes
778 while counter >= 0:
779 db_tmp.seek(counter-bytes,2)
780 byte = db_tmp.read(1)
781 db.write(byte)
782 counter -= 1
783
784 db.flush()
785 db.close()
786 db_tmp.close()
787 old.close()
788 try:
789 os.remove(xpakpath+".reverse")
790 except OSError:
791 pass
792 return xpakpath
793
806
818
820 old = open(tbz2file,"rb")
821 if not dbpath:
822 dbpath = tbz2file[:-5]+".db"
823 db = open(dbpath,"wb")
824
825
826 old.seek(0,2)
827
828 bytes = old.tell()
829 counter = bytes
830 dbcontent = []
831
832 while counter >= 0:
833 old.seek(counter-bytes,2)
834 byte = old.read(1)
835 if byte == "|":
836 old.seek(counter-bytes-31,2)
837 chunk = old.read(31)+byte
838 if chunk == etpConst['databasestarttag']:
839 break
840 dbcontent.append(byte)
841 counter -= 1
842 if not dbcontent:
843 old.close()
844 db.close()
845 try:
846 os.remove(dbpath)
847 except:
848 pass
849 return None
850 dbcontent.reverse()
851 for x in dbcontent:
852 db.write(x)
853
854 db.flush()
855 db.close()
856 old.close()
857 return dbpath
858
860 old = open(tbz2file,"rb")
861 new = open(savedir+"/"+os.path.basename(tbz2file),"wb")
862
863
864 old.seek(0,2)
865
866 bytes = old.tell()
867 counter = bytes
868
869 while counter >= 0:
870 old.seek(counter-bytes,2)
871 byte = old.read(1)
872 if byte == "|":
873 old.seek(counter-bytes-31,2)
874 chunk = old.read(31)+byte
875 if chunk == etpConst['databasestarttag']:
876 old.seek(counter-bytes-32,2)
877 break
878 counter -= 1
879
880 endingbyte = old.tell()
881 old.seek(0)
882 while old.tell() <= endingbyte:
883 byte = old.read(1)
884 new.write(byte)
885 counter += 1
886
887 new.flush()
888 new.close()
889 old.close()
890 return savedir+"/"+os.path.basename(tbz2file)
891
892
894 md5hash = md5sum(filepath)
895 hashfile = filepath+etpConst['packagesmd5fileext']
896 f = open(hashfile,"w")
897 name = os.path.basename(filepath)
898 f.write(md5hash+" "+name+"\n")
899 f.flush()
900 f.close()
901 return hashfile
902
904 sha512hash = sha512(filepath)
905 hashfile = filepath+etpConst['packagessha512fileext']
906 f = open(hashfile,"w")
907 tbz2name = os.path.basename(filepath)
908 f.write(sha512hash+" "+filepath+"\n")
909 f.flush()
910 f.close()
911 return hashfile
912
914 sha256hash = sha256(filepath)
915 hashfile = filepath+etpConst['packagessha256fileext']
916 f = open(hashfile,"w")
917 tbz2name = os.path.basename(filepath)
918 f.write(sha256hash+" "+filepath+"\n")
919 f.flush()
920 f.close()
921 return hashfile
922
924 sha1hash = sha1(filepath)
925 hashfile = filepath+etpConst['packagessha1fileext']
926 f = open(hashfile,"w")
927 tbz2name = os.path.basename(filepath)
928 f.write(sha1hash+" "+filepath+"\n")
929 f.flush()
930 f.close()
931 return hashfile
932
934 checksum = str(checksum)
935 result = md5sum(filepath)
936 result = str(result)
937 if checksum == result:
938 return True
939 return False
940
942 checksum = str(checksum)
943 result = sha512(filepath)
944 result = str(result)
945 if checksum == result:
946 return True
947 return False
948
950 checksum = str(checksum)
951 result = sha256(filepath)
952 result = str(result)
953 if checksum == result:
954 return True
955 return False
956
958 checksum = str(checksum)
959 result = sha1(filepath)
960 result = str(result)
961 if checksum == result:
962 return True
963 return False
964
966 m = hashlib.md5()
967 m.update(string)
968 return m.hexdigest()
969
970
972 sort_dict = {}
973
974 for item in update_list:
975
976 year = item.split("-")[1]
977 if sort_dict.has_key(year):
978 sort_dict[year].append(item)
979 else:
980 sort_dict[year] = []
981 sort_dict[year].append(item)
982 new_list = []
983 keys = sort_dict.keys()
984 keys.sort()
985 for key in keys:
986 sort_dict[key].sort()
987 new_list += sort_dict[key]
988 del sort_dict
989 return new_list
990
992 data = []
993 if os.access(filepath, os.R_OK | os.F_OK):
994 gen_f = open(filepath,"r")
995 content = gen_f.readlines()
996 gen_f.close()
997
998 content = [x.strip().rsplit("#", 1)[0].strip() for x in content \
999 if not x.startswith("#") and x.strip()]
1000 for line in content:
1001 if line in data:
1002 continue
1003 data.append(line)
1004 return data
1005
1006
1008
1009
1010 if os.path.isfile(file) and os.path.isfile(fromfile):
1011 old = md5sum(fromfile)
1012 new = md5sum(file)
1013 if old == new:
1014 return file, False
1015
1016 counter = -1
1017 newfile = ""
1018 previousfile = ""
1019
1020 while 1:
1021 counter += 1
1022 txtcounter = str(counter)
1023 oldtxtcounter = str(counter-1)
1024 txtcounter_len = 4-len(txtcounter)
1025 cnt = 0
1026 while cnt < txtcounter_len:
1027 txtcounter = "0"+txtcounter
1028 oldtxtcounter = "0"+oldtxtcounter
1029 cnt += 1
1030 newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file)
1031 if counter > 0:
1032 previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file)
1033 else:
1034 previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1035 if not os.path.exists(newfile):
1036 break
1037 if not newfile:
1038 newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1039 else:
1040
1041 if os.path.exists(previousfile):
1042
1043
1044 new = md5sum(fromfile)
1045 old = md5sum(previousfile)
1046 if new == old:
1047 return previousfile, False
1048
1049
1050 new = md5sum(file)
1051 old = md5sum(previousfile)
1052 if (new == old):
1053 return previousfile, False
1054
1055 return newfile, True
1056
1058
1059 logline = False
1060 logoutput = []
1061 f = open(file,"r")
1062 reallog = f.readlines()
1063 f.close()
1064
1065 for line in reallog:
1066 if line.startswith("INFO: postinst") or line.startswith("LOG: postinst"):
1067 logline = True
1068 continue
1069
1070 elif line.startswith("LOG:"):
1071 logline = False
1072 continue
1073 if (logline) and (line.strip()):
1074
1075 logoutput.append(line.strip())
1076 return logoutput
1077
1078
1079
1080
1081
1082 ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\\d*)*)(-r(\\d+))?$")
1083 suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
1084 suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
1085 endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
1086
1087
1089 myparts = mypkg.split('-')
1090 for x in myparts:
1091 if ververify(x):
1092 return 0
1093 return 1
1094
1096
1097 myver = myverx[:]
1098 if myver.endswith("*"):
1099 myver = myver[:-1]
1100 if ver_regexp.match(myver):
1101 return 1
1102 else:
1103 if not silent:
1104 print "!!! syntax error in version: %s" % myver
1105 return 0
1106
1107
1108
1109
1110
1111 valid_category = re.compile("^\w[\w-]*")
1112 invalid_atom_chars_regexp = re.compile("[()|@]")
1113
1115 """
1116 Check to see if a depend atom is valid
1117
1118 Example usage:
1119 >>> isvalidatom('media-libs/test-3.0')
1120 0
1121 >>> isvalidatom('>=media-libs/test-3.0')
1122 1
1123
1124 @param atom: The depend atom to check against
1125 @type atom: String
1126 @rtype: Integer
1127 @return: One of the following:
1128 1) 0 if the atom is invalid
1129 2) 1 if the atom is valid
1130 """
1131 atom = remove_tag(myatom)
1132 atom = remove_usedeps(atom)
1133 if invalid_atom_chars_regexp.search(atom):
1134 return 0
1135 if allow_blockers and atom[:1] == "!":
1136 if atom[1:2] == "!":
1137 atom = atom[2:]
1138 else:
1139 atom = atom[1:]
1140
1141
1142 if atom.count("/") > 1:
1143 return 0
1144
1145 cpv = dep_getcpv(atom)
1146 cpv_catsplit = catsplit(cpv)
1147 mycpv_cps = None
1148 if cpv:
1149 if len(cpv_catsplit) == 2:
1150 if valid_category.match(cpv_catsplit[0]) is None:
1151 return 0
1152 if cpv_catsplit[0] == "null":
1153
1154 mycpv_cps = catpkgsplit(cpv.replace("null/", "cat/", 1))
1155 if mycpv_cps:
1156 mycpv_cps = list(mycpv_cps)
1157 mycpv_cps[0] = "null"
1158 if not mycpv_cps:
1159 mycpv_cps = catpkgsplit(cpv)
1160
1161 operator = get_operator(atom)
1162 if operator:
1163 if operator[0] in "<>" and remove_slot(atom).endswith("*"):
1164 return 0
1165 if mycpv_cps:
1166 if len(cpv_catsplit) == 2:
1167
1168 return 1
1169 else:
1170 return 0
1171 else:
1172
1173 return 0
1174 if mycpv_cps:
1175
1176 return 0
1177
1178 if len(cpv_catsplit) == 2:
1179
1180 return 1
1181 return 0
1182
1184 return mydep.split("/", 1)
1185
1187 """
1188 Return the operator used in a depstring.
1189
1190 Example usage:
1191 >>> from portage.dep import *
1192 >>> get_operator(">=test-1.0")
1193 '>='
1194
1195 @param mydep: The dep string to check
1196 @type mydep: String
1197 @rtype: String
1198 @return: The operator. One of:
1199 '~', '=', '>', '<', '=*', '>=', or '<='
1200 """
1201 if mydep:
1202 mydep = remove_slot(mydep)
1203 if not mydep:
1204 return None
1205 if mydep[0] == "~":
1206 operator = "~"
1207 elif mydep[0] == "=":
1208 if mydep[-1] == "*":
1209 operator = "=*"
1210 else:
1211 operator = "="
1212 elif mydep[0] in "><":
1213 if len(mydep) > 1 and mydep[1] == "=":
1214 operator = mydep[0:2]
1215 else:
1216 operator = mydep[0]
1217 else:
1218 operator = None
1219
1220 return operator
1221
1223 """
1224 Checks to see if the depstring is only the package name (no version parts)
1225
1226 Example usage:
1227 >>> isjustname('media-libs/test-3.0')
1228 0
1229 >>> isjustname('test')
1230 1
1231 >>> isjustname('media-libs/test')
1232 1
1233
1234 @param mypkg: The package atom to check
1235 @param mypkg: String
1236 @rtype: Integer
1237 @return: One of the following:
1238 1) 0 if the package string is not just the package name
1239 2) 1 if it is
1240 """
1241
1242 myparts = mypkg.split('-')
1243 for x in myparts:
1244 if ververify(x):
1245 return 0
1246 return 1
1247
1249 """
1250 Checks to see if a package is in category/package-version or package-version format,
1251 possibly returning a cached result.
1252
1253 Example usage:
1254 >>> isspecific('media-libs/test')
1255 0
1256 >>> isspecific('media-libs/test-3.0')
1257 1
1258
1259 @param mypkg: The package depstring to check against
1260 @type mypkg: String
1261 @rtype: Integer
1262 @return: One of the following:
1263 1) 0 if the package string is not specific
1264 2) 1 if it is
1265 """
1266 mysplit = mypkg.split("/")
1267 if not isjustname(mysplit[-1]):
1268 return 1
1269 return 0
1270
1271
1273 """
1274 Takes a Category/Package-Version-Rev and returns a list of each.
1275
1276 @param mydata: Data to split
1277 @type mydata: string
1278 @param silent: suppress error messages
1279 @type silent: Boolean (integer)
1280 @rype: list
1281 @return:
1282 1. If each exists, it returns [cat, pkgname, version, rev]
1283 2. If cat is not specificed in mydata, cat will be "null"
1284 3. if rev does not exist it will be '-r0'
1285 4. If cat is invalid (specified but has incorrect syntax)
1286 an InvalidData Exception will be thrown
1287 """
1288
1289
1290 mysplit=mydata.split("/")
1291 p_split=None
1292 if len(mysplit)==1:
1293 retval=["null"]
1294 p_split=pkgsplit(mydata,silent=silent)
1295 elif len(mysplit)==2:
1296 retval=[mysplit[0]]
1297 p_split=pkgsplit(mysplit[1],silent=silent)
1298 if not p_split:
1299 return None
1300 retval.extend(p_split)
1301 return retval
1302
1304 myparts=mypkg.split("-")
1305
1306 if len(myparts)<2:
1307 if not silent:
1308 print "!!! Name error in",mypkg+": missing a version or name part."
1309 return None
1310 for x in myparts:
1311 if len(x)==0:
1312 if not silent:
1313 print "!!! Name error in",mypkg+": empty \"-\" part."
1314 return None
1315
1316
1317 revok=0
1318 myrev=myparts[-1]
1319
1320 if len(myrev) and myrev[0]=="r":
1321 try:
1322 int(myrev[1:])
1323 revok=1
1324 except ValueError:
1325 pass
1326 if revok:
1327 verPos = -2
1328 revision = myparts[-1]
1329 else:
1330 verPos = -1
1331 revision = "r0"
1332
1333 if ververify(myparts[verPos]):
1334 if len(myparts)== (-1*verPos):
1335 return None
1336 else:
1337 for x in myparts[:verPos]:
1338 if ververify(x):
1339 return None
1340
1341 myval=["-".join(myparts[:verPos]),myparts[verPos],revision]
1342 return myval
1343 else:
1344 return None
1345
1347 """
1348 Return the category/package-name of a depstring.
1349
1350 Example usage:
1351 >>> dep_getkey('media-libs/test-3.0')
1352 'media-libs/test'
1353
1354 @param mydep: The depstring to retrieve the category/package-name of
1355 @type mydep: String
1356 @rtype: String
1357 @return: The package category/package-version
1358 """
1359 if not mydepx: return mydepx
1360 mydep = mydepx[:]
1361 mydep = remove_tag(mydep)
1362 mydep = remove_usedeps(mydep)
1363
1364 mydep = dep_getcpv(mydep)
1365 if mydep and isspecific(mydep):
1366 mysplit = catpkgsplit(mydep)
1367 if not mysplit:
1368 return mydep
1369 return mysplit[0] + "/" + mysplit[1]
1370
1371 return mydep
1372
1373
1375 """
1376 Return the category-package-version with any operators/slot specifications stripped off
1377
1378 Example usage:
1379 >>> dep_getcpv('>=media-libs/test-3.0')
1380 'media-libs/test-3.0'
1381
1382 @param mydep: The depstring
1383 @type mydep: String
1384 @rtype: String
1385 @return: The depstring with the operator removed
1386 """
1387
1388 if mydep and mydep[0] == "*":
1389 mydep = mydep[1:]
1390 if mydep and mydep[-1] == "*":
1391 mydep = mydep[:-1]
1392 if mydep and mydep[0] == "!":
1393 mydep = mydep[1:]
1394 if mydep[:2] in [">=", "<="]:
1395 mydep = mydep[2:]
1396 elif mydep[:1] in "=<>~":
1397 mydep = mydep[1:]
1398 colon = mydep.rfind(":")
1399 if colon != -1:
1400 mydep = mydep[:colon]
1401
1402 return mydep
1403
1405 """
1406
1407 # Imported from portage.dep
1408 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1409
1410 Retrieve the slot on a depend.
1411
1412 Example usage:
1413 >>> dep_getslot('app-misc/test:3')
1414 '3'
1415
1416 @param mydep: The depstring to retrieve the slot of
1417 @type mydep: String
1418 @rtype: String
1419 @return: The slot
1420 """
1421 colon = mydep.find(":")
1422 if colon != -1:
1423 bracket = mydep.find("[", colon)
1424 if bracket == -1:
1425 return mydep[colon+1:]
1426 else:
1427 return mydep[colon+1:bracket]
1428 return None
1429
1431
1432 """
1433
1434 # Imported from portage.dep
1435 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1436
1437 Pull a listing of USE Dependencies out of a dep atom.
1438
1439 Example usage:
1440 >>> dep_getusedeps('app-misc/test:3[foo,-bar]')
1441 ('foo','-bar')
1442
1443 @param depend: The depstring to process
1444 @type depend: String
1445 @rtype: List
1446 @return: List of use flags ( or [] if no flags exist )
1447 """
1448
1449 use_list = []
1450 open_bracket = depend.find('[')
1451
1452 comma_separated = False
1453 bracket_count = 0
1454 while( open_bracket != -1 ):
1455 bracket_count += 1
1456 if bracket_count > 1:
1457 raise InvalidAtom("USE Dependency with more " + \
1458 "than one set of brackets: %s" % (depend,))
1459 close_bracket = depend.find(']', open_bracket )
1460 if close_bracket == -1:
1461 raise InvalidAtom("USE Dependency with no closing bracket: %s" % depend )
1462 use = depend[open_bracket + 1: close_bracket]
1463
1464 if not use:
1465 raise InvalidAtom("USE Dependency with " + \
1466 "no use flag ([]): %s" % depend )
1467 if not comma_separated:
1468 comma_separated = "," in use
1469
1470 if comma_separated and bracket_count > 1:
1471 raise InvalidAtom("USE Dependency contains a mixture of " + \
1472 "comma and bracket separators: %s" % depend )
1473
1474 if comma_separated:
1475 for x in use.split(","):
1476 if x:
1477 use_list.append(x)
1478 else:
1479 raise InvalidAtom("USE Dependency with no use " + \
1480 "flag next to comma: %s" % depend )
1481 else:
1482 use_list.append(use)
1483
1484
1485 open_bracket = depend.find( '[', open_bracket+1 )
1486
1487 return tuple(use_list)
1488
1490 mydepend = depend[:]
1491
1492 close_bracket = mydepend.find(']')
1493 after_closebracket = ''
1494 if close_bracket != -1: after_closebracket = mydepend[close_bracket+1:]
1495
1496 open_bracket = mydepend.find('[')
1497 if open_bracket != -1: mydepend = mydepend[:open_bracket]
1498
1499 return mydepend+after_closebracket
1500
1502 """
1503
1504 # Imported from portage.dep
1505 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1506
1507 Removes dep components from the right side of an atom:
1508 * slot
1509 * use
1510 * repo
1511 """
1512 colon = mydep.find(":")
1513 if colon != -1:
1514 mydep = mydep[:colon]
1515 else:
1516 bracket = mydep.find("[")
1517 if bracket != -1:
1518 mydep = mydep[:bracket]
1519 return mydep
1520
1521
1523 myver = ver.split("-")
1524 if myver[-1][0] == "r":
1525 return '-'.join(myver[:-1])
1526 return ver
1527
1529 colon = mydep.rfind("#")
1530 if colon == -1:
1531 return mydep
1532 return mydep[:colon]
1533
1541
1543
1544 colon = mydep.rfind("~")
1545 if colon != -1:
1546 myrev = mydep[colon+1:]
1547 try:
1548 myrev = int(myrev)
1549 except ValueError:
1550 return None
1551 return myrev
1552 return None
1553
1554
1555 dep_revmatch = re.compile('^r[0-9]')
1557 myver = mydep.split("-")
1558 myrev = myver[-1]
1559 if dep_revmatch.match(myrev):
1560 return myrev
1561 else:
1562 return "r0"
1563
1564
1566 colon = mydep.rfind("@")
1567 if colon != -1:
1568 mydata = mydep[colon+1:]
1569 mydata = mydata.split(",")
1570 if not mydata:
1571 mydata = None
1572 return mydep[:colon],mydata
1573 else:
1574 return mydep,None
1575
1577
1578 """
1579 Retrieve the slot on a depend.
1580
1581 Example usage:
1582 >>> dep_gettag('app-misc/test#2.6.23-sabayon-r1')
1583 '2.6.23-sabayon-r1'
1584
1585 """
1586
1587 colon = dep.rfind("#")
1588 if colon != -1:
1589 mydep = dep[colon+1:]
1590 rslt = remove_slot(mydep)
1591 return rslt
1592 return None
1593
1595 try:
1596 while atom:
1597 if atom[0] in ('>','<','=','~',):
1598 atom = atom[1:]
1599 continue
1600 break
1601 except IndexError:
1602 pass
1603 return atom
1604
1605
1606
1607
1609
1610 if ver1 == ver2:
1611 return 0
1612
1613 match1 = None
1614 match2 = None
1615 if ver1:
1616 match1 = ver_regexp.match(ver1)
1617 if ver2:
1618 match2 = ver_regexp.match(ver2)
1619
1620
1621 invalid = False
1622 invalid_rc = 0
1623 if not match1:
1624 invalid = True
1625 elif not match1.groups():
1626 invalid = True
1627 elif not match2:
1628 invalid_rc = 1
1629 invalid = True
1630 elif not match2.groups():
1631 invalid_rc = 1
1632 invalid = True
1633 if invalid: return invalid_rc
1634
1635
1636
1637 list1 = [int(match1.group(2))]
1638 list2 = [int(match2.group(2))]
1639
1640
1641 if len(match1.group(3)) or len(match2.group(3)):
1642 vlist1 = match1.group(3)[1:].split(".")
1643 vlist2 = match2.group(3)[1:].split(".")
1644 for i in range(0, max(len(vlist1), len(vlist2))):
1645
1646
1647
1648 if len(vlist1) <= i or len(vlist1[i]) == 0:
1649 list1.append(-1)
1650 list2.append(int(vlist2[i]))
1651 elif len(vlist2) <= i or len(vlist2[i]) == 0:
1652 list1.append(int(vlist1[i]))
1653 list2.append(-1)
1654
1655 elif (vlist1[i][0] != "0" and vlist2[i][0] != "0"):
1656 list1.append(int(vlist1[i]))
1657 list2.append(int(vlist2[i]))
1658
1659 else:
1660 list1.append(float("0."+vlist1[i]))
1661 list2.append(float("0."+vlist2[i]))
1662
1663
1664 if len(match1.group(5)):
1665 list1.append(ord(match1.group(5)))
1666 if len(match2.group(5)):
1667 list2.append(ord(match2.group(5)))
1668
1669 for i in range(0, max(len(list1), len(list2))):
1670 if len(list1) <= i:
1671 return -1
1672 elif len(list2) <= i:
1673 return 1
1674 elif list1[i] != list2[i]:
1675 return list1[i] - list2[i]
1676
1677
1678 list1 = match1.group(6).split("_")[1:]
1679 list2 = match2.group(6).split("_")[1:]
1680
1681 for i in range(0, max(len(list1), len(list2))):
1682 if len(list1) <= i:
1683 s1 = ("p","0")
1684 else:
1685 s1 = suffix_regexp.match(list1[i]).groups()
1686 if len(list2) <= i:
1687 s2 = ("p","0")
1688 else:
1689 s2 = suffix_regexp.match(list2[i]).groups()
1690 if s1[0] != s2[0]:
1691 return suffix_value[s1[0]] - suffix_value[s2[0]]
1692 if s1[1] != s2[1]:
1693
1694
1695 try:
1696 r1 = int(s1[1])
1697 except ValueError:
1698 r1 = 0
1699 try:
1700 r2 = int(s2[1])
1701 except ValueError:
1702 r2 = 0
1703 return r1 - r2
1704
1705
1706 if match1.group(10):
1707 r1 = int(match1.group(10))
1708 else:
1709 r1 = 0
1710 if match2.group(10):
1711 r2 = int(match2.group(10))
1712 else:
1713 r2 = 0
1714 return r1 - r2
1715
1717 '''
1718 @description: compare two lists composed by [version,tag,revision] and [version,tag,revision]
1719 if listA > listB --> positive number
1720 if listA == listB --> 0
1721 if listA < listB --> negative number
1722 @input package: listA[version,tag,rev] and listB[version,tag,rev]
1723 @output: integer number
1724 '''
1725
1726
1727 rc = 0
1728 if listA[1] and listB[1]:
1729 rc = cmp(listA[1],listB[1])
1730 if rc == 0:
1731 rc = compare_versions(listA[0],listB[0])
1732
1733 if rc == 0:
1734
1735 if listA[1] > listB[1]:
1736 return 1
1737 elif listA[1] < listB[1]:
1738 return -1
1739 else:
1740
1741 if listA[2] > listB[2]:
1742 return 1
1743 elif listA[2] < listB[2]:
1744 return -1
1745 else:
1746 return 0
1747 return rc
1748
1750 '''
1751 @description: reorder a version list
1752 @input versionlist: a list
1753 @output: the ordered list
1754 '''
1755 rc = compare_versions(a,b)
1756 if rc < 0: return -1
1757 elif rc > 0: return 1
1758 else: return 0
1760 return sorted(versions,g_n_w_cmp,reverse = True)
1761
1763
1764 if len(versions) == 1:
1765 return versions
1766
1767 versionlist = versions[:]
1768
1769 rc = False
1770 while not rc:
1771 change = False
1772 for x in range(len(versionlist)):
1773 pkgA = versionlist[x]
1774 try:
1775 pkgB = versionlist[x+1]
1776 except:
1777 pkgB = "0"
1778 result = compare_versions(pkgA,pkgB)
1779 if result < 0:
1780 versionlist[x] = pkgB
1781 versionlist[x+1] = pkgA
1782 change = True
1783 if not change:
1784 rc = True
1785
1786 return versionlist
1787
1788
1790 '''
1791 @description: reorder a version list
1792 @input versionlist: a list
1793 @output: the ordered list
1794 '''
1795 rc = entropy_compare_versions(a,b)
1796 if rc < 0: return -1
1797 elif rc > 0: return 1
1798 else: return 0
1799
1801 return sorted(versions,g_e_n_w_cmp,reverse = True)
1802
1804 '''
1805 descendent order
1806 versions = [(version,tag,revision),(version,tag,revision)]
1807 '''
1808 if len(versions) == 1:
1809 return versions
1810
1811 myversions = versions[:]
1812
1813
1814 rc = False
1815 while not rc:
1816 change = False
1817 for x in range(len(myversions)):
1818 pkgA = myversions[x]
1819 try:
1820 pkgB = myversions[x+1]
1821 except:
1822 pkgB = ("0","",0)
1823 result = entropy_compare_versions(pkgA,pkgB)
1824 if result < 0:
1825 myversions[x] = pkgB
1826 myversions[x+1] = pkgA
1827 change = True
1828 if not change:
1829 rc = True
1830
1831 return myversions
1832
1833
1835 try:
1836 t = int(x)
1837 del t
1838 return True
1839 except:
1840 return False
1841
1842
1843 -def istextfile(filename, blocksize = 512):
1844 f = open(filename)
1845 r = istext(f.read(blocksize))
1846 f.close()
1847 return r
1848
1850 import string
1851 _null_trans = string.maketrans("", "")
1852 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
1853
1854 if "\0" in s:
1855 return False
1856
1857 if not s:
1858 return True
1859
1860
1861
1862 t = s.translate(_null_trans, text_characters)
1863
1864
1865
1866 if len(t)/len(s) > 0.30:
1867 return False
1868 return True
1869
1870
1871
1872
1874 mydata = {}
1875 return [mydata.setdefault(e,e) for e in alist if e not in mydata]
1876
1877
1878
1879 mappings = {
1880 "'":"''",
1881 '"':'""',
1882 ' ':'+'
1883 }
1884
1886 arg_lst = []
1887 if len(args)==1:
1888 return escape_single(args[0])
1889 for x in args:
1890 arg_lst.append(escape_single(x))
1891 return tuple(arg_lst)
1892
1894 if type(x)==type(()) or type(x)==type([]):
1895 return escape(x)
1896 if type(x)==type(""):
1897 tmpstr=''
1898 for d in range(len(x)):
1899 if x[d] in mappings.keys():
1900 if x[d] in ("'", '"'):
1901 if d+1<len(x):
1902 if x[d+1]!=x[d]:
1903 tmpstr+=mappings[x[d]]
1904 else:
1905 tmpstr+=mappings[x[d]]
1906 else:
1907 tmpstr+=mappings[x[d]]
1908 else:
1909 tmpstr+=x[d]
1910 else:
1911 tmpstr=x
1912 return tmpstr
1913
1915 if type(val)==type(""):
1916 tmpstr=''
1917 for key,item in mappings.items():
1918 val=val.replace(item,key)
1919 tmpstr = val
1920 else:
1921 tmpstr=val
1922 return tmpstr
1923
1925 arg_lst = []
1926 for x in args:
1927 arg_lst.append(unescape(x))
1928 return tuple(arg_lst)
1929
1931 myuri = spliturl(uri)[1]
1932
1933 myuri = myuri.split("@")[len(myuri.split("@"))-1]
1934 return myuri
1935
1937 import urlparse
1938 return urlparse.urlsplit(url)
1939
1941 cmd = "cd \""+pathtocompress+"\" && tar cjf \""+storepath+"\" " + \
1942 ". &> /dev/null"
1943 return subprocess.call(cmd, shell = True)
1944
1946
1947 uid = kwds.get('spf_uid')
1948 if uid != None: kwds.pop('spf_uid')
1949
1950 gid = kwds.get('spf_gid')
1951 if gid != None: kwds.pop('spf_gid')
1952
1953 write_pid_func = kwds.get('write_pid_func')
1954 if write_pid_func != None:
1955 kwds.pop('write_pid_func')
1956
1957 try:
1958 import cPickle as pickle
1959 except ImportError:
1960 import pickle
1961 pread, pwrite = os.pipe()
1962 pid = os.fork()
1963 if pid > 0:
1964 if write_pid_func != None:
1965 write_pid_func(pid)
1966 os.close(pwrite)
1967 f = os.fdopen(pread, 'rb')
1968 status, result = pickle.load(f)
1969 os.waitpid(pid, 0)
1970 f.close()
1971 if status == 0:
1972 return result
1973 else:
1974 raise result
1975 else:
1976 os.close(pread)
1977 if gid != None:
1978 os.setgid(gid)
1979 if uid != None:
1980 os.setuid(uid)
1981 try:
1982 result = f(*args, **kwds)
1983 status = 0
1984 except Exception, exc:
1985 result = exc
1986 status = 1
1987 f = os.fdopen(pwrite, 'wb')
1988 try:
1989 pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
1990 except pickle.PicklingError, exc:
1991 pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
1992 f.close()
1993 os._exit(0)
1994
1995
1997
1998 if extractPath == None:
1999 extractPath = os.path.dirname(filepath)
2000 if not os.path.isfile(filepath):
2001 raise FileNotFound('FileNotFound: archive does not exist')
2002
2003 try:
2004 tar = tarfile.open(filepath,"r")
2005 except tarfile.ReadError:
2006 if catchEmpty:
2007 return 0
2008 raise
2009 except EOFError:
2010 return -1
2011
2012 try:
2013
2014 encoded_path = extractPath.encode('utf-8')
2015 def mymf(tarinfo):
2016 if tarinfo.isdir():
2017
2018
2019 try:
2020 os.makedirs(os.path.join(encoded_path, tarinfo.name), 0777)
2021 except EnvironmentError:
2022 pass
2023 return tarinfo
2024
2025 tar.extract(tarinfo, encoded_path)
2026 del tar.members[:]
2027 return tarinfo
2028
2029 def mycmp(a,b):
2030 return cmp(a.name,b.name)
2031
2032 entries = sorted(map(mymf, tar), mycmp, reverse = True)
2033
2034
2035 def mymf2(tarinfo):
2036 epath = os.path.join(encoded_path, tarinfo.name)
2037 try:
2038 tar.chown(tarinfo, epath)
2039
2040
2041
2042 uname = tarinfo.uname
2043 gname = tarinfo.gname
2044 ugdata_valid = False
2045 try:
2046 int(gname)
2047 int(uname)
2048 except ValueError:
2049 ugdata_valid = True
2050
2051 try:
2052 if ugdata_valid:
2053
2054
2055 uid, gid = get_uid_from_user(uname), \
2056 get_gid_from_group(gname)
2057 os.lchown(epath, uid, gid)
2058 except OSError:
2059 pass
2060
2061 tar.utime(tarinfo, epath)
2062 tar.chmod(tarinfo, epath)
2063 except tarfile.ExtractError:
2064 if tar.errorlevel > 1:
2065 raise
2066 done = map(mymf2, entries)
2067 del done
2068
2069 except EOFError:
2070 return -1
2071
2072 finally:
2073 tar.close()
2074
2075 if os.listdir(extractPath):
2076 return 0
2077 return -1
2078
2080 size = str(round(float(bytes)/1024,1))
2081 if bytes < 1024:
2082 size = str(round(float(bytes)))+"b"
2083 elif bytes < 1023999:
2084 size += "kB"
2085 elif bytes > 1023999:
2086 size = str(round(float(size)/1024,1))
2087 size += "MB"
2088 return size
2089
2091 ftppassword = uri.split("@")[:-1]
2092 if not ftppassword: return uri
2093 ftppassword = '@'.join(ftppassword)
2094 ftppassword = ftppassword.split(":")[-1]
2095 if not ftppassword:
2096 return uri
2097 newuri = uri.replace(ftppassword,"xxxxxxxx")
2098 return newuri
2099
2101 ftpuser = ftpuri.split("ftp://")[-1].split(":")[0]
2102 if (ftpuser == ""):
2103 ftpuser = "anonymous@"
2104 ftppassword = "anonymous"
2105 else:
2106 ftppassword = ftpuri.split("@")[:-1]
2107 if len(ftppassword) > 1:
2108 ftppassword = '@'.join(ftppassword)
2109 ftppassword = ftppassword.split(":")[-1]
2110 if (ftppassword == ""):
2111 ftppassword = "anonymous"
2112 else:
2113 ftppassword = ftppassword[0]
2114 ftppassword = ftppassword.split(":")[-1]
2115 if (ftppassword == ""):
2116 ftppassword = "anonymous"
2117
2118 ftpport = ftpuri.split(":")[-1]
2119 try:
2120 ftpport = int(ftpport)
2121 except ValueError:
2122 ftpport = 21
2123
2124 ftpdir = '/'
2125 if ftpuri.count("/") > 2:
2126 ftpdir = ftpuri.split("ftp://")[-1]
2127 ftpdir = ftpdir.split("/")[-1]
2128 ftpdir = ftpdir.split(":")[0]
2129 if ftpdir.endswith("/"):
2130 ftpdir = ftpdir[:len(ftpdir)-1]
2131 if not ftpdir: ftpdir = "/"
2132
2133 return ftpuser, ftppassword, ftpport, ftpdir
2134
2136 return os.path.getmtime(path)
2137
2145
2147 from datetime import datetime
2148
2149 unixtime = os.path.getmtime(path)
2150 humantime = datetime.fromtimestamp(unixtime)
2151
2152 humantime = str(humantime)
2153 outputtime = ""
2154 for char in humantime:
2155 if char != "-" and char != " " and char != ":":
2156 outputtime += char
2157 return outputtime
2158
2160 from datetime import datetime
2161 humantime = str(datetime.fromtimestamp(unixtime))
2162 return humantime
2163
2166
2168 return time.strftime("%Y")
2169
2171
2172 mysecs = seconds
2173 myminutes = 0
2174 myhours = 0
2175 mydays = 0
2176
2177 while mysecs >= 60:
2178 mysecs -= 60
2179 myminutes += 1
2180
2181 while myminutes >= 60:
2182 myminutes -= 60
2183 myhours += 1
2184
2185 while myhours >= 24:
2186 myhours -= 24
2187 mydays += 1
2188
2189 output = []
2190 output.append(str(mysecs)+"s")
2191 if myminutes > 0 or myhours > 0:
2192 output.append(str(myminutes)+"m")
2193 if myhours > 0 or mydays > 0:
2194 output.append(str(myhours)+"h")
2195 if mydays > 0:
2196 output.append(str(mydays)+"d")
2197 output.reverse()
2198 return ':'.join(output)
2199
2200
2202
2203 if not toCleanDirs:
2204 toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ]
2205 counter = 0
2206
2207 for xdir in toCleanDirs:
2208 print_info(red(" * ")+"Cleaning "+darkgreen(xdir)+" directory...", back = True)
2209 if os.path.isdir(xdir):
2210 dircontent = os.listdir(xdir)
2211 if dircontent != []:
2212 for data in dircontent:
2213 subprocess.call(["rm","-rf",os.path.join(xdir,data)])
2214 counter += 1
2215
2216 print_info(green(" * ")+"Cleaned: "+str(counter)+" files and directories")
2217 return 0
2218
2219 -def flatten(l, ltypes=(list, tuple)):
2220 i = 0
2221 while i < len(l):
2222 while isinstance(l[i], ltypes):
2223 if not l[i]:
2224 l.pop(i)
2225 if not len(l):
2226 break
2227 else:
2228 l[i:i+1] = list(l[i])
2229 i += 1
2230 return l
2231
2233 content = []
2234 if os.path.isfile(etpConst['repositoriesconf']):
2235 f = open(etpConst['repositoriesconf'])
2236 content = f.readlines()
2237 f.close()
2238 return content
2239
2241 content = read_repositories_conf()
2242 content = [x.strip() for x in content]
2243 repolines = [x for x in content if x.startswith("repository|") and (len(x.split("|")) == 5)]
2244 content = [x for x in content if x not in repolines]
2245 for repoid in ordered_repository_list:
2246
2247 for x in repolines:
2248 repoidline = x.split("|")[1]
2249 if repoid == repoidline:
2250 content.append(x)
2251 _save_repositories_content(content)
2252
2254
2255 if repodata['repoid'].endswith(".tbz2"):
2256 return
2257
2258 content = read_repositories_conf()
2259 content = [x.strip() for x in content]
2260 if not disable and not enable:
2261 content = [x for x in content if not x.startswith("repository|"+repodata['repoid'])]
2262 if remove:
2263
2264 content = [x for x in content if not (x.startswith("#") and not x.startswith("##") and (x.find("repository|"+repodata['repoid']) != -1))]
2265 if not remove:
2266
2267 repolines = [x for x in content if x.startswith("repository|") or (x.startswith("#") and not x.startswith("##") and (x.find("repository|") != -1))]
2268 content = [x for x in content if x not in repolines]
2269
2270 repolines = [x for x in repolines if (len(x.split("|")) == 5)]
2271 repolines_data = {}
2272 repocount = 0
2273 for x in repolines:
2274 repolines_data[repocount] = {}
2275 repolines_data[repocount]['repoid'] = x.split("|")[1]
2276 repolines_data[repocount]['line'] = x
2277 if disable and x.split("|")[1] == repodata['repoid']:
2278 if not x.startswith("#"):
2279 x = "#"+x
2280 repolines_data[repocount]['line'] = x
2281 elif enable and x.split("|")[1] == repodata['repoid'] and x.startswith("#"):
2282 repolines_data[repocount]['line'] = x[1:]
2283 repocount += 1
2284
2285 if not disable and not enable:
2286
2287 line = "repository|%s|%s|%s|%s#%s#%s,%s" % ( repodata['repoid'],
2288 repodata['description'],
2289 ' '.join(repodata['plain_packages']),
2290 repodata['plain_database'],
2291 repodata['dbcformat'],
2292 repodata['service_port'],
2293 repodata['ssl_service_port'],
2294 )
2295
2296
2297 to_remove = set()
2298 for cc in repolines_data:
2299 if repolines_data[cc]['line'].startswith("#") and \
2300 (repolines_data[cc]['line'].find("repository|"+repodata['repoid']) != -1):
2301
2302 to_remove.add(cc)
2303 for x in to_remove:
2304 del repolines_data[x]
2305
2306 repolines_data[repocount] = {}
2307 repolines_data[repocount]['repoid'] = repodata['repoid']
2308 repolines_data[repocount]['line'] = line
2309
2310
2311 keys = repolines_data.keys()
2312 keys.sort()
2313 for cc in keys:
2314
2315
2316 line = repolines_data[cc]['line']
2317 content.append(line)
2318
2319 _save_repositories_content(content)
2320
2322 if os.path.isfile(etpConst['repositoriesconf']):
2323 if os.path.isfile(etpConst['repositoriesconf']+".old"):
2324 os.remove(etpConst['repositoriesconf']+".old")
2325 shutil.copy2(etpConst['repositoriesconf'],etpConst['repositoriesconf']+".old")
2326 f = open(etpConst['repositoriesconf'],"w")
2327 for x in content:
2328 f.write(x+"\n")
2329 f.flush()
2330 f.close()
2331
2333
2334
2335 if not os.access(os.path.dirname(config_file),os.W_OK):
2336 return False
2337
2338 content = []
2339 if os.path.isfile(config_file):
2340 f = open(config_file,"r")
2341 content = [x.strip() for x in f.readlines()]
2342 f.close()
2343
2344
2345 config_file_tmp = config_file+".tmp"
2346 f = open(config_file_tmp,"w")
2347 param_found = False
2348 if data:
2349 proposed_line = "%s|%s" % (name,data,)
2350 myreg = re.compile('^(%s)?[|].*$' % (name,))
2351 else:
2352 proposed_line = "# %s|" % (name,)
2353 myreg_rem = re.compile('^(%s)?[|].*$' % (name,))
2354 myreg = re.compile('^#([ \t]+?)?(%s)?[|].*$' % (name,))
2355 new_content = []
2356 for line in content:
2357 if myreg_rem.match(line):
2358 continue
2359 new_content.append(line)
2360 content = new_content
2361
2362 for line in content:
2363 if myreg.match(line):
2364 param_found = True
2365 line = proposed_line
2366 f.write(line+"\n")
2367 if not param_found:
2368 f.write(proposed_line+"\n")
2369 f.flush()
2370 f.close()
2371 shutil.move(config_file_tmp,config_file)
2372 return True
2373
2376
2378 if not os.path.exists(tbz2file):
2379 return False
2380 return tarfile.is_tarfile(tbz2file)
2381
2383 invalid = [ord(x) for x in string if ord(x) not in xrange(32,127)]
2384 if invalid: return False
2385 return True
2386
2388 try:
2389 os.stat(path)
2390 except OSError:
2391 return False
2392 return True
2393
2395 if re.findall(r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])', myhash):
2396 return True
2397 return False
2398
2399
2401 try:
2402 import cStringIO as stringio
2403 except ImportError:
2404 import StringIO as stringio
2405 return stringio.StringIO()
2406
2408 count = 0
2409 f.seek(count,2)
2410 size = f.tell()
2411 while count > (size*-1):
2412 count -= 1
2413 f.seek(count,2)
2414 myc = f.read(1)
2415 if myc == "\n":
2416 break
2417 f.seek(count+1,2)
2418 pos = f.tell()
2419 f.truncate(pos)
2420
2422 import struct
2423 f = open(elf_file,"rb")
2424 f.seek(4)
2425 elf_class = f.read(1)
2426 f.close()
2427 elf_class = struct.unpack('B',elf_class)[0]
2428 return elf_class
2429
2431 import struct
2432 f = open(elf_file,"rb")
2433 data = f.read(4)
2434 f.close()
2435 try:
2436 data = struct.unpack('BBBB',data)
2437 except struct.error:
2438 return False
2439 if data == (127, 69, 76, 70):
2440 return True
2441 return False
2442
2443 readelf_avail_check = False
2444 ldd_avail_check = False
2445
2446
2454
2462
2463
2464
2466 global readelf_avail_check
2467 if not readelf_avail_check:
2468 if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
2469 raise FileNotFound('FileNotFound: no readelf')
2470 readelf_avail_check = True
2471 data = [x.strip().split()[-1][1:-1].split(":") for x in getstatusoutput('readelf -d %s' % (elf_file,))[1].split("\n") if not ((x.find("(RPATH)") == -1) and (x.find("(RUNPATH)") == -1))]
2472 mypaths = []
2473 for mypath in data:
2474 for xpath in mypath:
2475 xpath = xpath.replace("$ORIGIN",os.path.dirname(elf_file))
2476 mypaths.append(xpath)
2477 return mypaths
2478
2480 from xml.dom import minidom
2481 doc = minidom.Document()
2482 ugc = doc.createElement("entropy")
2483 for key, value in dictionary.items():
2484 item = doc.createElement('item')
2485 item.setAttribute('value',key)
2486 if isinstance(value,str):
2487 mytype = "str"
2488 elif isinstance(value,unicode):
2489 mytype = "unicode"
2490 elif isinstance(value,list):
2491 mytype = "list"
2492 elif isinstance(value,set):
2493 mytype = "set"
2494 elif isinstance(value,frozenset):
2495 mytype = "frozenset"
2496 elif isinstance(value,dict):
2497 mytype = "dict"
2498 elif isinstance(value,tuple):
2499 mytype = "tuple"
2500 elif isinstance(value,int):
2501 mytype = "int"
2502 elif isinstance(value,float):
2503 mytype = "float"
2504 elif value == None:
2505 mytype = "None"
2506 value = "None"
2507 else: raise TypeError
2508 item.setAttribute('type',mytype)
2509 item_value = doc.createTextNode("%s" % (value,))
2510 item.appendChild(item_value)
2511 ugc.appendChild(item)
2512 doc.appendChild(ugc)
2513 return doc.toxml()
2514
2516 from xml.dom import minidom
2517 doc = minidom.parseString(xml_string)
2518 entropies = doc.getElementsByTagName("entropy")
2519 if not entropies: return {}
2520 entropy = entropies[0]
2521 items = entropy.getElementsByTagName('item')
2522
2523 my_map = {
2524 "str": str,
2525 "unicode": unicode,
2526 "list": list,
2527 "set": set,
2528 "frozenset": frozenset,
2529 "dict": dict,
2530 "tuple": tuple,
2531 "int": int,
2532 "float": float,
2533 "None": None,
2534 }
2535
2536 mydict = {}
2537 for item in items:
2538 key = item.getAttribute('value')
2539 if not key: continue
2540 mytype = item.getAttribute('type')
2541 mytype_m = my_map.get(mytype)
2542 if mytype_m == None: raise TypeError
2543 try:
2544 data = item.firstChild.data
2545 except AttributeError:
2546 data = ''
2547 if mytype in ("list","set","frozenset","dict","tuple",):
2548 if data:
2549 if data[0] not in ("(","[","s","{",): data = ''
2550 mydict[key] = eval(data)
2551 elif mytype == "None":
2552 mydict[key] = None
2553 else:
2554 mydict[key] = mytype_m(data)
2555 return mydict
2556
2558 from xml.dom import minidom
2559 doc = minidom.Document()
2560 ugc = doc.createElement("entropy")
2561 for key, value in dictionary.items():
2562 item = doc.createElement('item')
2563 item.setAttribute('value',key)
2564 item_value = doc.createTextNode(value)
2565 item.appendChild(item_value)
2566 ugc.appendChild(item)
2567 doc.appendChild(ugc)
2568 return doc.toxml()
2569
2571 from xml.dom import minidom
2572 doc = minidom.parseString(xml_string)
2573 entropies = doc.getElementsByTagName("entropy")
2574 if not entropies:
2575 return {}
2576 entropy = entropies[0]
2577 items = entropy.getElementsByTagName('item')
2578 mydict = {}
2579 for item in items:
2580 key = item.getAttribute('value')
2581 if not key: continue
2582 try:
2583 data = item.firstChild.data
2584 except AttributeError:
2585 data = ''
2586 mydict[key] = data
2587 return mydict
2588
2590 if package_tag:
2591 package_tag = "#%s" % (package_tag,)
2592 else:
2593 package_tag = ''
2594
2595 package_name = "%s:%s-%s" % (category, name, version,)
2596 package_name += package_tag
2597 package_name += etpConst['packagesext']
2598 return package_name
2599
2601 if package_tag:
2602 package_tag = "#%s" % (package_tag,)
2603 else:
2604 package_tag = ''
2605 package_name = "%s/%s-%s" % (category,name,version,)
2606 package_name += package_tag
2607 return package_name
2608
2610 f = open(filepath,"r")
2611 items = set()
2612 line = f.readline()
2613 while line:
2614 x = line.strip().rsplit("#",1)[0]
2615 if x and (not x.startswith('#')):
2616 items.add(x)
2617 line = f.readline()
2618 f.close()
2619 return items
2620
2622
2623 ldpaths = []
2624 try:
2625 f = open(etpConst['systemroot']+"/etc/ld.so.conf","r")
2626 paths = f.readlines()
2627 for path in paths:
2628 path = path.strip()
2629 if path:
2630 if path[0] == "/":
2631 ldpaths.append(os.path.normpath(path))
2632 f.close()
2633 except (IOError,OSError,TypeError,ValueError,IndexError,):
2634 pass
2635
2636
2637 if "/lib" not in ldpaths:
2638 ldpaths.append("/lib")
2639 if "/usr/lib" not in ldpaths:
2640 ldpaths.append("/usr/lib")
2641
2642 return ldpaths
2643
2645 path = set()
2646 paths = os.getenv("PATH")
2647 if paths != None:
2648 paths = set(paths.split(":"))
2649 path |= paths
2650 return path
2651
2653 mynewlist = []
2654 for item in mylist:
2655 try:
2656 mynewlist.append(item.decode("utf-8"))
2657 except UnicodeDecodeError:
2658 try:
2659 mynewlist.append(item.decode("latin1").decode("utf-8"))
2660 except:
2661 raise
2662 return mynewlist
2663