1
2
3 '''
4 # DESCRIPTION:
5 # generic tools for all the handlers applications
6
7 Copyright (C) 2007-2008 Fabio Erculiani
8
9 This program is free software; you can redistribute it and/or modify
10 it under the terms of the GNU General Public License as published by
11 the Free Software Foundation; either version 2 of the License, or
12 (at your option) any later version.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 '''
23
24 from __future__ import with_statement
25 import random
26 import stat
27 import errno
28 import re
29 import os
30 import time
31 import shutil
32 import tarfile
33 import subprocess
34 import grp
35 import pwd
36 import hashlib
37 from entropy.output import *
38 from entropy.const import *
39 from entropy.exceptions import *
40
43
45
46 if uid == None:
47 uid = os.getuid()
48 if uid == 0:
49 return True
50
51 try:
52 username = pwd.getpwuid(uid)[0]
53 except KeyError:
54 return False
55
56 try:
57 data = grp.getgrnam(etpConst['sysgroup'])
58 except KeyError:
59 return False
60
61
62 etp_group_users = data[3]
63
64 if not etp_group_users or \
65 username not in etp_group_users:
66 return False
67
68 return True
69
71 try:
72 return pwd.getpwnam(username)[2]
73 except (KeyError, IndexError,):
74 return -1
75
77 try:
78 return grp.getgrnam(groupname)[2]
79 except (KeyError, IndexError,):
80 return -1
81
83 try:
84 return pwd.getpwuid(uid)[0]
85 except KeyError:
86 return None
87
89 try:
90 return grp.getgrgid(gid)[0]
91 except (KeyError, IndexError,):
92 return -1
93
96
98 import traceback
99 traceback.print_exc(file = f)
100
102 import traceback
103 from cStringIO import StringIO
104 buf = StringIO()
105 traceback.print_exc(file = buf)
106 return buf.getvalue()
107
109 import traceback
110 if not returndata: traceback.print_exc()
111 data = []
112 tb = sys.exc_info()[2]
113 while 1:
114 if not tb.tb_next:
115 break
116 tb = tb.tb_next
117 stack = []
118 stack.append(tb.tb_frame)
119
120 for frame in stack:
121 if not returndata:
122 print
123 print "Frame %s in %s at line %s" % (frame.f_code.co_name,
124 frame.f_code.co_filename,
125 frame.f_lineno)
126 else:
127 data.append("Frame %s in %s at line %s" % (frame.f_code.co_name,
128 frame.f_code.co_filename,
129 frame.f_lineno))
130 for key, value in frame.f_locals.items():
131 if not returndata:
132 print "\t%20s = " % key,
133 else:
134 data.append("\t%20s = " % key,)
135 try:
136 if not returndata:
137 print value
138 else:
139 data.append(value)
140 except:
141 if not returndata: print "<ERROR WHILE PRINTING VALUE>"
142 return data
143
144
145
146
148
149 import socket
150 import urllib2
151 socket.setdefaulttimeout(60)
152
153 from entropy.core import SystemSettings
154 sys_settings = SystemSettings()
155 proxy_settings = sys_settings['system']['proxy']
156 try:
157 mydict = {}
158 if proxy_settings['ftp']:
159 mydict['ftp'] = proxy_settings['ftp']
160 if proxy_settings['http']:
161 mydict['http'] = proxy_settings['http']
162 if mydict:
163 mydict['username'] = proxy_settings['username']
164 mydict['password'] = proxy_settings['password']
165 add_proxy_opener(urllib2, mydict)
166 else:
167
168 urllib2._opener = None
169 item = urllib2.urlopen(url)
170 result = item.readlines()
171 item.close()
172 del item
173 if (not result):
174 socket.setdefaulttimeout(2)
175 return False
176 socket.setdefaulttimeout(2)
177 return result
178 except:
179 socket.setdefaulttimeout(2)
180 return False
181
183 f = open(path,"r")
184 x = f.read(4)
185 if x == '\x89PNG':
186 return True
187 return False
188
190 f = open(path,"r")
191 x = f.read(10)
192 if x == '\xff\xd8\xff\xe0\x00\x10JFIF':
193 return True
194 return False
195
197 f = open(path,"r")
198 x = f.read(2)
199 if x == 'BM':
200 return True
201 return False
202
204 f = open(path,"r")
205 x = f.read(5)
206 if x == 'GIF89':
207 return True
208 return False
209
215
217 april_first = "01-04"
218 cur_time = time.strftime("%d-%m")
219 if april_first == cur_time:
220 return True
221 return False
222
224 import types
225 if type(module) != types.ModuleType:
226 raise InvalidDataType("InvalidDataType: not a module")
227 if not data:
228 return
229
230 username = None
231 password = None
232 authinfo = None
233 if data.has_key('password'):
234 username = data.pop('username')
235 if data.has_key('password'):
236 username = data.pop('password')
237 if username == None or password == None:
238 username = None
239 password = None
240 else:
241 passmgr = module.HTTPPasswordMgrWithDefaultRealm()
242 if data['http']:
243 passmgr.add_password(None, data['http'], username, password)
244 if data['ftp']:
245 passmgr.add_password(None, data['ftp'], username, password)
246 authinfo = module.ProxyBasicAuthHandler(passmgr)
247
248 proxy_support = module.ProxyHandler(data)
249 if authinfo:
250 opener = module.build_opener(proxy_support, authinfo)
251 else:
252 opener = module.build_opener(proxy_support)
253 module.install_opener(opener)
254
256 try:
257 mystring = str(string)
258 del mystring
259 except:
260 return False
261 return True
262
264 try:
265 unicode(string)
266 except:
267 return False
268 return True
269
271 monster = "(?:[a-z0-9!#$%&'*+/=?^_{|}~-]+(?:.[a-z0-9!#$%" + \
272 "&'*+/=?^_{|}~-]+)*|\"(?:" + \
273 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]" + \
274 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*\")@(?:(?:[a-z0-9]" + \
275 "(?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?" + \
276 "|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.)" + \
277 "{3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?" + \
278 "|[a-z0-9-]*[a-z0-9]:(?:" + \
279 "[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]" + \
280 "|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])"
281 evil = re.compile(monster)
282 if evil.match(email):
283 return True
284 return False
285
288
290 my = file_path[:]
291 if isinstance(my, unicode):
292 my = my.encode("utf-8")
293 mystat = os.lstat(my)
294 return int(mystat.st_size)
295
297 size = 0
298 for myfile in file_list:
299 try:
300 size += get_file_size(myfile)
301 except (OSError,IOError,):
302 continue
303 return size
304
306 import statvfs
307 st = os.statvfs(mountpoint)
308 freeblocks = st[statvfs.F_BFREE]
309 blocksize = st[statvfs.F_BSIZE]
310 freespace = freeblocks*blocksize
311 if bytes_required > freespace:
312
313 return False
314 return True
315
317 """Return (status, output) of executing cmd in a shell."""
318 pipe = os.popen('{ ' + cmd + '; } 2>&1', 'r')
319 text = pipe.read()
320 sts = pipe.close()
321 if sts is None: sts = 0
322 if text[-1:] == '\n': text = text[:-1]
323 return sts, text
324
325
326
327
328
329
330 -def movefile(src, dest, src_basedir = None):
331
332 sstat = os.lstat(src)
333 destexists = 1
334 try:
335 dstat = os.lstat(dest)
336 except (OSError, IOError,):
337 dstat = os.lstat(os.path.dirname(dest))
338 destexists = 0
339
340 if destexists:
341 if stat.S_ISLNK(dstat[stat.ST_MODE]):
342 try:
343 os.unlink(dest)
344 destexists = 0
345 except (OSError, IOError,):
346 pass
347
348 if stat.S_ISLNK(sstat[stat.ST_MODE]):
349 try:
350 target = os.readlink(src)
351 if src_basedir != None:
352 if target.find(src_basedir) == 0:
353 target = target[len(src_basedir):]
354 if destexists and not stat.S_ISDIR(dstat[stat.ST_MODE]):
355 os.unlink(dest)
356 os.symlink(target,dest)
357 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
358 return True
359 except SystemExit:
360 raise
361 except Exception, e:
362 print "!!! failed to properly create symlink:"
363 print "!!!",dest,"->",target
364 print "!!!",e
365 return False
366
367 renamefailed = True
368 if sstat.st_dev == dstat.st_dev:
369 try:
370 os.rename(src,dest)
371 renamefailed = False
372 except Exception, e:
373 if e[0] != errno.EXDEV:
374
375 print "!!! Failed to move",src,"to",dest
376 print "!!!",e
377 return False
378
379
380 if renamefailed:
381 didcopy = True
382 if stat.S_ISREG(sstat[stat.ST_MODE]):
383 try:
384 while 1:
385 tmp_dest = "%s#entropy_new_%s" % (dest,get_random_number(),)
386 if not os.path.lexists(tmp_dest): break
387 shutil.copyfile(src,tmp_dest)
388 os.rename(tmp_dest,dest)
389 didcopy = True
390 except SystemExit, e:
391 raise
392 except Exception, e:
393 print '!!! copy',src,'->',dest,'failed.'
394 print "!!!",e
395 return False
396 else:
397
398 a = getstatusoutput("mv -f '%s' '%s'" % (src,dest,))
399 if a[0]!=0:
400 print "!!! Failed to move special file:"
401 print "!!! '"+src+"' to '"+dest+"'"
402 print "!!!",a
403 return False
404 try:
405 if didcopy:
406 if stat.S_ISLNK(sstat[stat.ST_MODE]):
407 os.lchown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
408 else:
409 os.chown(dest,sstat[stat.ST_UID],sstat[stat.ST_GID])
410 os.chmod(dest, stat.S_IMODE(sstat[stat.ST_MODE]))
411 os.unlink(src)
412 except SystemExit, e:
413 raise
414 except Exception, e:
415 print "!!! Failed to chown/chmod/unlink in movefile()"
416 print "!!!",dest
417 print "!!!",e
418 return False
419
420 try:
421 os.utime(dest, (sstat.st_atime, sstat.st_mtime))
422 except OSError:
423
424
425 try:
426 long(os.stat(dest).st_mtime)
427 return True
428 except OSError, e:
429 print "!!! Failed to stat in movefile()\n"
430 print "!!! %s\n" % dest
431 print "!!! %s\n" % str(e)
432 return False
433
434 return True
435
437 mycount = count
438 while mycount > 0:
439 os.system("sleep 0.35; echo -ne \"\a\"; sleep 0.35")
440 mycount -= 1
441
443 if etpConst['applicationlock']:
444 if not gentle:
445 raise SystemExit(10)
446 return True
447 return False
448
450 try:
451 return abs(hash(os.urandom(2)))%99999
452 except NotImplementedError:
453 random.seed()
454 return random.randint(10000,99999)
455
457 chunks = []
458 my = mystr[:]
459 mylen = len(my)
460 mycount = 0
461 while mylen:
462 chunk = my[:chunk_len]
463 chunks.append(chunk)
464 my_chunk_len = len(chunk)
465 my = my[my_chunk_len:]
466 mylen -= my_chunk_len
467 return chunks
468
469 -def countdown(secs=5, what="Counting...", back = False):
470 if secs:
471 if back:
472 try:
473 print red(">>"), what,
474 except UnicodeEncodeError:
475 print red(">>"),what.encode('utf-8'),
476 else:
477 try:
478 print what
479 except UnicodeEncodeError:
480 print what.encode('utf-8')
481 for i in range(secs)[::-1]:
482 sys.stdout.write(red(str(i+1)+" "))
483 sys.stdout.flush()
484 time.sleep(1)
485
487 m = hashlib.md5()
488 readfile = open(filepath)
489 block = readfile.read(1024)
490 while block:
491 m.update(block)
492 block = readfile.read(1024)
493 readfile.close()
494 return m.hexdigest()
495
497 m = hashlib.sha512()
498 readfile = open(filepath)
499 block = readfile.read(1024)
500 while block:
501 m.update(block)
502 block = readfile.read(1024)
503 readfile.close()
504 return m.hexdigest()
505
507 m = hashlib.sha256()
508 readfile = open(filepath)
509 block = readfile.read(1024)
510 while block:
511 m.update(block)
512 block = readfile.read(1024)
513 readfile.close()
514 return m.hexdigest()
515
517 m = hashlib.sha1()
518 readfile = open(filepath)
519 block = readfile.read(1024)
520 while block:
521 m.update(block)
522 block = readfile.read(1024)
523 readfile.close()
524 return m.hexdigest()
525
527 if not os.path.isdir(directory):
528 raise DirectoryNotFound("DirectoryNotFound: directory just does not exist.")
529 myfiles = os.listdir(directory)
530 m = hashlib.md5()
531 if not myfiles:
532 if get_obj:
533 return m
534 return "0"
535
536 for currentdir,subdirs,files in os.walk(directory):
537 for myfile in files:
538 myfile = os.path.join(currentdir,myfile)
539 readfile = open(myfile)
540 block = readfile.read(1024)
541 while block:
542 m.update(block)
543 block = readfile.read(1024)
544 readfile.close()
545 if get_obj:
546 return m
547 return m.hexdigest()
548
549
550
551 -def getfd(filespec, readOnly = 0):
552 import types
553 if type(filespec) == types.IntType:
554 return filespec
555 if filespec == None:
556 filespec = "/dev/null"
557
558 flags = os.O_RDWR | os.O_CREAT
559 if (readOnly):
560 flags = os.O_RDONLY
561 return os.open(filespec, flags)
562
564 f_out = open(destination_path,"wb")
565 f_in = opener(file_path,"rb")
566 data = f_in.read(8192)
567 while data:
568 f_out.write(data)
569 data = f_in.read(8192)
570 f_out.flush()
571 f_out.close()
572 f_in.close()
573
574 -def compress_file(file_path, destination_path, opener, compress_level = None):
575 f_in = open(file_path,"rb")
576 if compress_level != None:
577 f_out = opener(destination_path,"wb",compresslevel = compress_level)
578 else:
579 f_out = opener(destination_path,"wb")
580 data = f_in.read(8192)
581 while data:
582 f_out.write(data)
583 data = f_in.read(8192)
584 if hasattr(f_out,'flush'):
585 f_out.flush()
586 f_out.close()
587 f_in.close()
588
589
591
592 if compressor not in ("bz2","gz",):
593 raise AttributeError("invalid compressor specified")
594
595 id_strings = {}
596 tar = tarfile.open(dest_file,"w:%s" % (compressor,))
597 try:
598 for path in files_to_compress:
599 exist = os.lstat(path)
600 tarinfo = tar.gettarinfo(path, os.path.basename(path))
601 tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
602 tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))
603 if not stat.S_ISREG(exist.st_mode): continue
604 tarinfo.type = tarfile.REGTYPE
605 with open(path) as f:
606 tar.addfile(tarinfo, f)
607 finally:
608 tar.close()
609
611
612 try:
613 tar = tarfile.open(compressed_file,"r")
614 except tarfile.ReadError:
615 if catch_empty:
616 return True
617 return False
618 except EOFError:
619 return False
620
621 try:
622
623 dest_path = dest_path.encode('utf-8')
624 def mymf(tarinfo):
625 if tarinfo.isdir():
626
627
628 try:
629 os.makedirs(os.path.join(dest_path, tarinfo.name), 0777)
630 except EnvironmentError:
631 pass
632 return tarinfo
633 tar.extract(tarinfo, dest_path)
634 del tar.members[:]
635 return tarinfo
636
637 def mycmp(a,b):
638 return cmp(a.name, b.name)
639
640 directories = sorted(map(mymf, tar), mycmp, reverse = True)
641
642
643 def mymf2(tarinfo):
644 epath = os.path.join(dest_path, tarinfo.name)
645 try:
646 tar.chown(tarinfo, epath)
647
648
649
650 uname = tarinfo.uname
651 gname = tarinfo.gname
652 ugdata_valid = False
653 try:
654 int(gname)
655 int(uname)
656 except ValueError:
657 ugdata_valid = True
658
659 try:
660 if ugdata_valid:
661
662
663 uid, gid = get_uid_from_user(uname), \
664 get_gid_from_group(gname)
665 os.lchown(epath, uid, gid)
666 except OSError:
667 pass
668
669 tar.utime(tarinfo, epath)
670 tar.chmod(tarinfo, epath)
671 except tarfile.ExtractError:
672 if tar.errorlevel > 1:
673 return False
674 done = map(mymf2, directories)
675 del done
676
677 except EOFError:
678 return False
679
680 finally:
681 tar.close()
682
683 return True
684
686 import gzip
687 filepath = gzipfilepath[:-3]
688 item = open(filepath,"wb")
689 filegz = gzip.GzipFile(gzipfilepath,"rb")
690 chunk = filegz.read(8192)
691 while chunk:
692 item.write(chunk)
693 chunk = filegz.read(8192)
694 filegz.close()
695 item.flush()
696 item.close()
697 return filepath
698
700 import bz2
701 filepath = bzip2filepath[:-4]
702 item = open(filepath,"wb")
703 filebz2 = bz2.BZ2File(bzip2filepath,"rb")
704 chunk = filebz2.read(8192)
705 while chunk:
706 item.write(chunk)
707 chunk = filebz2.read(8192)
708 filebz2.close()
709 item.flush()
710 item.close()
711 return filepath
712
714 if os.path.isfile(etpConst['etpdatabaseclientfilepath']):
715 rnd = get_random_number()
716 source = etpConst['etpdatabaseclientfilepath']
717 dest = etpConst['etpdatabaseclientfilepath']+".backup."+str(rnd)
718 shutil.copy2(source,dest)
719 user = os.stat(source)[4]
720 group = os.stat(source)[5]
721 os.chown(dest,user,group)
722 shutil.copystat(source,dest)
723 return dest
724 return ""
725
730
732 xpakpath = suck_xpak(tbz2file, etpConst['entropyunpackdir'])
733 f = open(xpakpath,"rb")
734 data = f.read()
735 f.close()
736 os.remove(xpakpath)
737 return data
738
740 try:
741 import entropy.xpak as xpak
742 if tmpdir is None:
743 tmpdir = etpConst['packagestmpdir']+"/"+os.path.basename(xpakfile)[:-5]+"/"
744 if os.path.isdir(tmpdir):
745 shutil.rmtree(tmpdir,True)
746 os.makedirs(tmpdir)
747 xpakdata = xpak.getboth(xpakfile)
748 xpak.xpand(xpakdata,tmpdir)
749 del xpakdata
750 try:
751 os.remove(xpakfile)
752 except OSError:
753 pass
754 except:
755 return None
756 return tmpdir
757
759
760 dest_filename = os.path.basename(tbz2file)[:-5]+".xpak"
761 xpakpath = os.path.join(outputpath, dest_filename)
762 old = open(tbz2file,"rb")
763
764
765 old.seek(0, os.SEEK_END)
766
767 bytes = old.tell()
768 counter = bytes - 1
769 xpak_end = "XPAKSTOP"
770 xpak_start = "XPAKPACK"
771 xpak_entry_point = "X"
772 xpak_tag_len = len(xpak_start)
773 chunk_len = 3
774 data_start_position = None
775 data_end_position = None
776
777 while counter >= (0 - chunk_len):
778
779 old.seek(counter - bytes, os.SEEK_END)
780 if (bytes - (abs(counter - bytes))) < chunk_len:
781 chunk_len = 1
782 read_bytes = old.read(chunk_len)
783 read_len = len(read_bytes)
784
785 entry_idx = read_bytes.rfind(xpak_entry_point)
786 if entry_idx != -1:
787
788 cut_gotten = read_bytes[entry_idx:]
789 offset = xpak_tag_len - len(cut_gotten)
790 chunk = cut_gotten + old.read(offset)
791
792 if (chunk == xpak_end) and (data_start_position is None):
793 data_end_position = old.tell()
794
795 elif (chunk == xpak_start) and (data_end_position is not None):
796 data_start_position = old.tell() - xpak_tag_len
797 break
798
799 counter -= read_len
800
801 if data_start_position is None:
802 return None
803 if data_end_position is None:
804 return None
805
806
807
808
809 db = open(xpakpath,"wb")
810 old.seek(data_start_position)
811 to_read = data_end_position - data_start_position
812 while to_read > 0:
813 data = old.read(to_read)
814 db.write(data)
815 to_read -= len(data)
816
817 db.flush()
818 db.close()
819 old.close()
820 return xpakpath
821
833
845
847
848 old = open(tbz2file, "rb")
849 if not dbpath:
850 dbpath = tbz2file[:-5] + ".db"
851
852 start_position = locate_edb(old)
853 if not start_position:
854 old.close()
855 try:
856 os.remove(dbpath)
857 except OSError:
858 return None
859 return None
860
861 db = open(dbpath, "wb")
862 data = old.read(1024)
863 while data:
864 db.write(data)
865 data = old.read(1024)
866 db.flush()
867 db.close()
868
869 return dbpath
870
872
873
874 fileobj.seek(0, os.SEEK_END)
875
876 bytes = fileobj.tell()
877 counter = bytes - 1
878
879 db_tag = etpConst['databasestarttag']
880 db_tag_len = len(db_tag)
881 give_up_threshold = 1024000 * 30
882 entry_point = db_tag[::-1][0]
883 max_read_len = 8
884 start_position = None
885
886 while counter >= 0:
887 cur_threshold = abs((counter-bytes))
888 if cur_threshold >= give_up_threshold:
889 start_position = None
890 break
891 fileobj.seek(counter-bytes, os.SEEK_END)
892 read_bytes = fileobj.read(max_read_len)
893 read_len = len(read_bytes)
894 entry_idx = read_bytes.rfind(entry_point)
895 if entry_idx != -1:
896 rollback = (read_len - entry_idx) * -1
897 fileobj.seek(rollback, os.SEEK_CUR)
898 chunk = fileobj.read(db_tag_len)
899 if chunk == db_tag:
900 start_position = fileobj.tell()
901 break
902 counter -= read_len
903
904 return start_position
905
907 old = open(tbz2file, "rb")
908
909 start_position = locate_edb(old)
910 if not start_position:
911 old.close()
912 return None
913
914 new_path = os.path.join(savedir, os.path.basename(tbz2file))
915 new = open(new_path, "wb")
916
917 old.seek(0)
918 counter = 0
919 max_read_len = 1024
920 db_tag = etpConst['databasestarttag']
921 db_tag_len = len(db_tag)
922 start_position -= db_tag_len
923
924 while counter < start_position:
925 delta = start_position - counter
926 if delta < max_read_len:
927 max_read_len = delta
928 bytes = old.read(max_read_len)
929 read_bytes = len(bytes)
930 new.write(bytes)
931 counter += read_bytes
932
933 new.flush()
934 new.close()
935 old.close()
936 return savedir+"/"+os.path.basename(tbz2file)
937
938
940 md5hash = md5sum(filepath)
941 hashfile = filepath+etpConst['packagesmd5fileext']
942 f = open(hashfile,"w")
943 name = os.path.basename(filepath)
944 f.write(md5hash+" "+name+"\n")
945 f.flush()
946 f.close()
947 return hashfile
948
950 sha512hash = sha512(filepath)
951 hashfile = filepath+etpConst['packagessha512fileext']
952 f = open(hashfile,"w")
953 tbz2name = os.path.basename(filepath)
954 f.write(sha512hash+" "+tbz2name+"\n")
955 f.flush()
956 f.close()
957 return hashfile
958
960 sha256hash = sha256(filepath)
961 hashfile = filepath+etpConst['packagessha256fileext']
962 f = open(hashfile,"w")
963 tbz2name = os.path.basename(filepath)
964 f.write(sha256hash+" "+tbz2name+"\n")
965 f.flush()
966 f.close()
967 return hashfile
968
970 sha1hash = sha1(filepath)
971 hashfile = filepath+etpConst['packagessha1fileext']
972 f = open(hashfile,"w")
973 tbz2name = os.path.basename(filepath)
974 f.write(sha1hash+" "+tbz2name+"\n")
975 f.flush()
976 f.close()
977 return hashfile
978
980 checksum = str(checksum)
981 result = md5sum(filepath)
982 result = str(result)
983 if checksum == result:
984 return True
985 return False
986
988 checksum = str(checksum)
989 result = sha512(filepath)
990 result = str(result)
991 if checksum == result:
992 return True
993 return False
994
996 checksum = str(checksum)
997 result = sha256(filepath)
998 result = str(result)
999 if checksum == result:
1000 return True
1001 return False
1002
1004 checksum = str(checksum)
1005 result = sha1(filepath)
1006 result = str(result)
1007 if checksum == result:
1008 return True
1009 return False
1010
1012 m = hashlib.md5()
1013 m.update(string)
1014 return m.hexdigest()
1015
1016
1018 sort_dict = {}
1019
1020 for item in update_list:
1021
1022 year = item.split("-")[1]
1023 if sort_dict.has_key(year):
1024 sort_dict[year].append(item)
1025 else:
1026 sort_dict[year] = []
1027 sort_dict[year].append(item)
1028 new_list = []
1029 keys = sort_dict.keys()
1030 keys.sort()
1031 for key in keys:
1032 sort_dict[key].sort()
1033 new_list += sort_dict[key]
1034 del sort_dict
1035 return new_list
1036
1038 data = []
1039 if os.access(filepath, os.R_OK | os.F_OK):
1040 gen_f = open(filepath,"r")
1041 content = gen_f.readlines()
1042 gen_f.close()
1043
1044 content = [x.strip().rsplit("#", 1)[0].strip() for x in content \
1045 if not x.startswith("#") and x.strip()]
1046 for line in content:
1047 if line in data:
1048 continue
1049 data.append(line)
1050 return data
1051
1052
1054
1055
1056 if os.path.isfile(file) and os.path.isfile(fromfile):
1057 old = md5sum(fromfile)
1058 new = md5sum(file)
1059 if old == new:
1060 return file, False
1061
1062 counter = -1
1063 newfile = ""
1064 previousfile = ""
1065
1066 while 1:
1067 counter += 1
1068 txtcounter = str(counter)
1069 oldtxtcounter = str(counter-1)
1070 txtcounter_len = 4-len(txtcounter)
1071 cnt = 0
1072 while cnt < txtcounter_len:
1073 txtcounter = "0"+txtcounter
1074 oldtxtcounter = "0"+oldtxtcounter
1075 cnt += 1
1076 newfile = os.path.dirname(file)+"/"+"._cfg"+txtcounter+"_"+os.path.basename(file)
1077 if counter > 0:
1078 previousfile = os.path.dirname(file)+"/"+"._cfg"+oldtxtcounter+"_"+os.path.basename(file)
1079 else:
1080 previousfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1081 if not os.path.exists(newfile):
1082 break
1083 if not newfile:
1084 newfile = os.path.dirname(file)+"/"+"._cfg0000_"+os.path.basename(file)
1085 else:
1086
1087 if os.path.exists(previousfile):
1088
1089
1090 new = md5sum(fromfile)
1091 old = md5sum(previousfile)
1092 if new == old:
1093 return previousfile, False
1094
1095
1096 new = md5sum(file)
1097 old = md5sum(previousfile)
1098 if (new == old):
1099 return previousfile, False
1100
1101 return newfile, True
1102
1104
1105 logline = False
1106 logoutput = []
1107 f = open(file,"r")
1108 reallog = f.readlines()
1109 f.close()
1110
1111 for line in reallog:
1112 if line.startswith("INFO: postinst") or line.startswith("LOG: postinst"):
1113 logline = True
1114 continue
1115
1116 elif line.startswith("LOG:"):
1117 logline = False
1118 continue
1119 if (logline) and (line.strip()):
1120
1121 logoutput.append(line.strip())
1122 return logoutput
1123
1124
1125
1126
1127
1128 ver_regexp = re.compile("^(cvs\\.)?(\\d+)((\\.\\d+)*)([a-z]?)((_(pre|p|beta|alpha|rc)\\d*)*)(-r(\\d+))?$")
1129 suffix_regexp = re.compile("^(alpha|beta|rc|pre|p)(\\d*)$")
1130 suffix_value = {"pre": -2, "p": 0, "alpha": -4, "beta": -3, "rc": -1}
1131 endversion_keys = ["pre", "p", "alpha", "beta", "rc"]
1132
1133
1135 myparts = mypkg.split('-')
1136 for x in myparts:
1137 if ververify(x):
1138 return 0
1139 return 1
1140
1142
1143 myver = myverx[:]
1144 if myver.endswith("*"):
1145 myver = myver[:-1]
1146 if ver_regexp.match(myver):
1147 return 1
1148 else:
1149 if not silent:
1150 print "!!! syntax error in version: %s" % myver
1151 return 0
1152
1153
1154
1155
1156
1157 valid_category = re.compile("^\w[\w-]*")
1158 invalid_atom_chars_regexp = re.compile("[()|@]")
1159
1161 """
1162 Check to see if a depend atom is valid
1163
1164 Example usage:
1165 >>> isvalidatom('media-libs/test-3.0')
1166 0
1167 >>> isvalidatom('>=media-libs/test-3.0')
1168 1
1169
1170 @param atom: The depend atom to check against
1171 @type atom: String
1172 @rtype: Integer
1173 @return: One of the following:
1174 1) 0 if the atom is invalid
1175 2) 1 if the atom is valid
1176 """
1177 atom = remove_tag(myatom)
1178 atom = remove_usedeps(atom)
1179 if invalid_atom_chars_regexp.search(atom):
1180 return 0
1181 if allow_blockers and atom[:1] == "!":
1182 if atom[1:2] == "!":
1183 atom = atom[2:]
1184 else:
1185 atom = atom[1:]
1186
1187
1188 if atom.count("/") > 1:
1189 return 0
1190
1191 cpv = dep_getcpv(atom)
1192 cpv_catsplit = catsplit(cpv)
1193 mycpv_cps = None
1194 if cpv:
1195 if len(cpv_catsplit) == 2:
1196 if valid_category.match(cpv_catsplit[0]) is None:
1197 return 0
1198 if cpv_catsplit[0] == "null":
1199
1200 mycpv_cps = catpkgsplit(cpv.replace("null/", "cat/", 1))
1201 if mycpv_cps:
1202 mycpv_cps = list(mycpv_cps)
1203 mycpv_cps[0] = "null"
1204 if not mycpv_cps:
1205 mycpv_cps = catpkgsplit(cpv)
1206
1207 operator = get_operator(atom)
1208 if operator:
1209 if operator[0] in "<>" and remove_slot(atom).endswith("*"):
1210 return 0
1211 if mycpv_cps:
1212 if len(cpv_catsplit) == 2:
1213
1214 return 1
1215 else:
1216 return 0
1217 else:
1218
1219 return 0
1220 if mycpv_cps:
1221
1222 return 0
1223
1224 if len(cpv_catsplit) == 2:
1225
1226 return 1
1227 return 0
1228
1230 return mydep.split("/", 1)
1231
1233 """
1234 Return the operator used in a depstring.
1235
1236 Example usage:
1237 >>> from portage.dep import *
1238 >>> get_operator(">=test-1.0")
1239 '>='
1240
1241 @param mydep: The dep string to check
1242 @type mydep: String
1243 @rtype: String
1244 @return: The operator. One of:
1245 '~', '=', '>', '<', '=*', '>=', or '<='
1246 """
1247 if mydep:
1248 mydep = remove_slot(mydep)
1249 if not mydep:
1250 return None
1251 if mydep[0] == "~":
1252 operator = "~"
1253 elif mydep[0] == "=":
1254 if mydep[-1] == "*":
1255 operator = "=*"
1256 else:
1257 operator = "="
1258 elif mydep[0] in "><":
1259 if len(mydep) > 1 and mydep[1] == "=":
1260 operator = mydep[0:2]
1261 else:
1262 operator = mydep[0]
1263 else:
1264 operator = None
1265
1266 return operator
1267
1269 """
1270 Checks to see if the depstring is only the package name (no version parts)
1271
1272 Example usage:
1273 >>> isjustname('media-libs/test-3.0')
1274 0
1275 >>> isjustname('test')
1276 1
1277 >>> isjustname('media-libs/test')
1278 1
1279
1280 @param mypkg: The package atom to check
1281 @param mypkg: String
1282 @rtype: Integer
1283 @return: One of the following:
1284 1) 0 if the package string is not just the package name
1285 2) 1 if it is
1286 """
1287
1288 myparts = mypkg.split('-')
1289 for x in myparts:
1290 if ververify(x):
1291 return 0
1292 return 1
1293
1295 """
1296 Checks to see if a package is in category/package-version or package-version format,
1297 possibly returning a cached result.
1298
1299 Example usage:
1300 >>> isspecific('media-libs/test')
1301 0
1302 >>> isspecific('media-libs/test-3.0')
1303 1
1304
1305 @param mypkg: The package depstring to check against
1306 @type mypkg: String
1307 @rtype: Integer
1308 @return: One of the following:
1309 1) 0 if the package string is not specific
1310 2) 1 if it is
1311 """
1312 mysplit = mypkg.split("/")
1313 if not isjustname(mysplit[-1]):
1314 return 1
1315 return 0
1316
1317
1319 """
1320 Takes a Category/Package-Version-Rev and returns a list of each.
1321
1322 @param mydata: Data to split
1323 @type mydata: string
1324 @param silent: suppress error messages
1325 @type silent: Boolean (integer)
1326 @rype: list
1327 @return:
1328 1. If each exists, it returns [cat, pkgname, version, rev]
1329 2. If cat is not specificed in mydata, cat will be "null"
1330 3. if rev does not exist it will be '-r0'
1331 4. If cat is invalid (specified but has incorrect syntax)
1332 an InvalidData Exception will be thrown
1333 """
1334
1335
1336 mysplit=mydata.split("/")
1337 p_split=None
1338 if len(mysplit)==1:
1339 retval=["null"]
1340 p_split=pkgsplit(mydata,silent=silent)
1341 elif len(mysplit)==2:
1342 retval=[mysplit[0]]
1343 p_split=pkgsplit(mysplit[1],silent=silent)
1344 if not p_split:
1345 return None
1346 retval.extend(p_split)
1347 return retval
1348
1350 myparts=mypkg.split("-")
1351
1352 if len(myparts)<2:
1353 if not silent:
1354 print "!!! Name error in",mypkg+": missing a version or name part."
1355 return None
1356 for x in myparts:
1357 if len(x)==0:
1358 if not silent:
1359 print "!!! Name error in",mypkg+": empty \"-\" part."
1360 return None
1361
1362
1363 revok=0
1364 myrev=myparts[-1]
1365
1366 if len(myrev) and myrev[0]=="r":
1367 try:
1368 int(myrev[1:])
1369 revok=1
1370 except ValueError:
1371 pass
1372 if revok:
1373 verPos = -2
1374 revision = myparts[-1]
1375 else:
1376 verPos = -1
1377 revision = "r0"
1378
1379 if ververify(myparts[verPos]):
1380 if len(myparts)== (-1*verPos):
1381 return None
1382 else:
1383 for x in myparts[:verPos]:
1384 if ververify(x):
1385 return None
1386
1387 myval=["-".join(myparts[:verPos]),myparts[verPos],revision]
1388 return myval
1389 else:
1390 return None
1391
1393 """
1394 Return the category/package-name of a depstring.
1395
1396 Example usage:
1397 >>> dep_getkey('media-libs/test-3.0')
1398 'media-libs/test'
1399
1400 @param mydep: The depstring to retrieve the category/package-name of
1401 @type mydep: String
1402 @rtype: String
1403 @return: The package category/package-version
1404 """
1405 if not mydepx: return mydepx
1406 mydep = mydepx[:]
1407 mydep = remove_tag(mydep)
1408 mydep = remove_usedeps(mydep)
1409
1410 mydep = dep_getcpv(mydep)
1411 if mydep and isspecific(mydep):
1412 mysplit = catpkgsplit(mydep)
1413 if not mysplit:
1414 return mydep
1415 return mysplit[0] + "/" + mysplit[1]
1416
1417 return mydep
1418
1419
1421 """
1422 Return the category-package-version with any operators/slot specifications stripped off
1423
1424 Example usage:
1425 >>> dep_getcpv('>=media-libs/test-3.0')
1426 'media-libs/test-3.0'
1427
1428 @param mydep: The depstring
1429 @type mydep: String
1430 @rtype: String
1431 @return: The depstring with the operator removed
1432 """
1433
1434 if mydep and mydep[0] == "*":
1435 mydep = mydep[1:]
1436 if mydep and mydep[-1] == "*":
1437 mydep = mydep[:-1]
1438 if mydep and mydep[0] == "!":
1439 mydep = mydep[1:]
1440 if mydep[:2] in [">=", "<="]:
1441 mydep = mydep[2:]
1442 elif mydep[:1] in "=<>~":
1443 mydep = mydep[1:]
1444 colon = mydep.rfind(":")
1445 if colon != -1:
1446 mydep = mydep[:colon]
1447
1448 return mydep
1449
1451 """
1452
1453 # Imported from portage.dep
1454 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1455
1456 Retrieve the slot on a depend.
1457
1458 Example usage:
1459 >>> dep_getslot('app-misc/test:3')
1460 '3'
1461
1462 @param mydep: The depstring to retrieve the slot of
1463 @type mydep: String
1464 @rtype: String
1465 @return: The slot
1466 """
1467 colon = mydep.find(":")
1468 if colon != -1:
1469 bracket = mydep.find("[", colon)
1470 if bracket == -1:
1471 return mydep[colon+1:]
1472 else:
1473 return mydep[colon+1:bracket]
1474 return None
1475
1477
1478 """
1479
1480 # Imported from portage.dep
1481 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1482
1483 Pull a listing of USE Dependencies out of a dep atom.
1484
1485 Example usage:
1486 >>> dep_getusedeps('app-misc/test:3[foo,-bar]')
1487 ('foo','-bar')
1488
1489 @param depend: The depstring to process
1490 @type depend: String
1491 @rtype: List
1492 @return: List of use flags ( or [] if no flags exist )
1493 """
1494
1495 use_list = []
1496 open_bracket = depend.find('[')
1497
1498 comma_separated = False
1499 bracket_count = 0
1500 while( open_bracket != -1 ):
1501 bracket_count += 1
1502 if bracket_count > 1:
1503 raise InvalidAtom("USE Dependency with more " + \
1504 "than one set of brackets: %s" % (depend,))
1505 close_bracket = depend.find(']', open_bracket )
1506 if close_bracket == -1:
1507 raise InvalidAtom("USE Dependency with no closing bracket: %s" % depend )
1508 use = depend[open_bracket + 1: close_bracket]
1509
1510 if not use:
1511 raise InvalidAtom("USE Dependency with " + \
1512 "no use flag ([]): %s" % depend )
1513 if not comma_separated:
1514 comma_separated = "," in use
1515
1516 if comma_separated and bracket_count > 1:
1517 raise InvalidAtom("USE Dependency contains a mixture of " + \
1518 "comma and bracket separators: %s" % depend )
1519
1520 if comma_separated:
1521 for x in use.split(","):
1522 if x:
1523 use_list.append(x)
1524 else:
1525 raise InvalidAtom("USE Dependency with no use " + \
1526 "flag next to comma: %s" % depend )
1527 else:
1528 use_list.append(use)
1529
1530
1531 open_bracket = depend.find( '[', open_bracket+1 )
1532
1533 return tuple(use_list)
1534
1536 mydepend = depend[:]
1537
1538 close_bracket = mydepend.find(']')
1539 after_closebracket = ''
1540 if close_bracket != -1: after_closebracket = mydepend[close_bracket+1:]
1541
1542 open_bracket = mydepend.find('[')
1543 if open_bracket != -1: mydepend = mydepend[:open_bracket]
1544
1545 return mydepend+after_closebracket
1546
1548 """
1549
1550 # Imported from portage.dep
1551 # $Id: dep.py 11281 2008-07-30 06:12:19Z zmedico $
1552
1553 Removes dep components from the right side of an atom:
1554 * slot
1555 * use
1556 * repo
1557 """
1558 colon = mydep.find(":")
1559 if colon != -1:
1560 mydep = mydep[:colon]
1561 else:
1562 bracket = mydep.find("[")
1563 if bracket != -1:
1564 mydep = mydep[:bracket]
1565 return mydep
1566
1567
1569 myver = ver.split("-")
1570 if myver[-1][0] == "r":
1571 return '-'.join(myver[:-1])
1572 return ver
1573
1575 colon = mydep.rfind("#")
1576 if colon == -1:
1577 return mydep
1578 return mydep[:colon]
1579
1587
1589
1590 colon = mydep.rfind("~")
1591 if colon != -1:
1592 myrev = mydep[colon+1:]
1593 try:
1594 myrev = int(myrev)
1595 except ValueError:
1596 return None
1597 return myrev
1598 return None
1599
1600
1601 dep_revmatch = re.compile('^r[0-9]')
1603 myver = mydep.split("-")
1604 myrev = myver[-1]
1605 if dep_revmatch.match(myrev):
1606 return myrev
1607 else:
1608 return "r0"
1609
1610
1612 colon = mydep.rfind("@")
1613 if colon != -1:
1614 mydata = mydep[colon+1:]
1615 mydata = mydata.split(",")
1616 if not mydata:
1617 mydata = None
1618 return mydep[:colon],mydata
1619 else:
1620 return mydep,None
1621
1623
1624 """
1625 Retrieve the slot on a depend.
1626
1627 Example usage:
1628 >>> dep_gettag('app-misc/test#2.6.23-sabayon-r1')
1629 '2.6.23-sabayon-r1'
1630
1631 """
1632
1633 colon = dep.rfind("#")
1634 if colon != -1:
1635 mydep = dep[colon+1:]
1636 rslt = remove_slot(mydep)
1637 return rslt
1638 return None
1639
1641 try:
1642 while atom:
1643 if atom[0] in ('>','<','=','~',):
1644 atom = atom[1:]
1645 continue
1646 break
1647 except IndexError:
1648 pass
1649 return atom
1650
1651
1652
1653
1655
1656 if ver1 == ver2:
1657 return 0
1658
1659 match1 = None
1660 match2 = None
1661 if ver1:
1662 match1 = ver_regexp.match(ver1)
1663 if ver2:
1664 match2 = ver_regexp.match(ver2)
1665
1666
1667 invalid = False
1668 invalid_rc = 0
1669 if not match1:
1670 invalid = True
1671 elif not match1.groups():
1672 invalid = True
1673 elif not match2:
1674 invalid_rc = 1
1675 invalid = True
1676 elif not match2.groups():
1677 invalid_rc = 1
1678 invalid = True
1679 if invalid: return invalid_rc
1680
1681
1682
1683 list1 = [int(match1.group(2))]
1684 list2 = [int(match2.group(2))]
1685
1686
1687 if len(match1.group(3)) or len(match2.group(3)):
1688 vlist1 = match1.group(3)[1:].split(".")
1689 vlist2 = match2.group(3)[1:].split(".")
1690 for i in range(0, max(len(vlist1), len(vlist2))):
1691
1692
1693
1694 if len(vlist1) <= i or len(vlist1[i]) == 0:
1695 list1.append(-1)
1696 list2.append(int(vlist2[i]))
1697 elif len(vlist2) <= i or len(vlist2[i]) == 0:
1698 list1.append(int(vlist1[i]))
1699 list2.append(-1)
1700
1701 elif (vlist1[i][0] != "0" and vlist2[i][0] != "0"):
1702 list1.append(int(vlist1[i]))
1703 list2.append(int(vlist2[i]))
1704
1705 else:
1706 list1.append(float("0."+vlist1[i]))
1707 list2.append(float("0."+vlist2[i]))
1708
1709
1710 if len(match1.group(5)):
1711 list1.append(ord(match1.group(5)))
1712 if len(match2.group(5)):
1713 list2.append(ord(match2.group(5)))
1714
1715 for i in range(0, max(len(list1), len(list2))):
1716 if len(list1) <= i:
1717 return -1
1718 elif len(list2) <= i:
1719 return 1
1720 elif list1[i] != list2[i]:
1721 return list1[i] - list2[i]
1722
1723
1724 list1 = match1.group(6).split("_")[1:]
1725 list2 = match2.group(6).split("_")[1:]
1726
1727 for i in range(0, max(len(list1), len(list2))):
1728 if len(list1) <= i:
1729 s1 = ("p","0")
1730 else:
1731 s1 = suffix_regexp.match(list1[i]).groups()
1732 if len(list2) <= i:
1733 s2 = ("p","0")
1734 else:
1735 s2 = suffix_regexp.match(list2[i]).groups()
1736 if s1[0] != s2[0]:
1737 return suffix_value[s1[0]] - suffix_value[s2[0]]
1738 if s1[1] != s2[1]:
1739
1740
1741 try:
1742 r1 = int(s1[1])
1743 except ValueError:
1744 r1 = 0
1745 try:
1746 r2 = int(s2[1])
1747 except ValueError:
1748 r2 = 0
1749 return r1 - r2
1750
1751
1752 if match1.group(10):
1753 r1 = int(match1.group(10))
1754 else:
1755 r1 = 0
1756 if match2.group(10):
1757 r2 = int(match2.group(10))
1758 else:
1759 r2 = 0
1760 return r1 - r2
1761
1763 '''
1764 @description: compare two lists composed by [version,tag,revision] and [version,tag,revision]
1765 if listA > listB --> positive number
1766 if listA == listB --> 0
1767 if listA < listB --> negative number
1768 @input package: listA[version,tag,rev] and listB[version,tag,rev]
1769 @output: integer number
1770 '''
1771
1772
1773 rc = 0
1774 if listA[1] and listB[1]:
1775 rc = cmp(listA[1],listB[1])
1776 if rc == 0:
1777 rc = compare_versions(listA[0],listB[0])
1778
1779 if rc == 0:
1780
1781 if listA[1] > listB[1]:
1782 return 1
1783 elif listA[1] < listB[1]:
1784 return -1
1785 else:
1786
1787 if listA[2] > listB[2]:
1788 return 1
1789 elif listA[2] < listB[2]:
1790 return -1
1791 else:
1792 return 0
1793 return rc
1794
1796 '''
1797 @description: reorder a version list
1798 @input versionlist: a list
1799 @output: the ordered list
1800 '''
1801 rc = compare_versions(a,b)
1802 if rc < 0: return -1
1803 elif rc > 0: return 1
1804 else: return 0
1806 return sorted(versions,g_n_w_cmp,reverse = True)
1807
1809
1810 if len(versions) == 1:
1811 return versions
1812
1813 versionlist = versions[:]
1814
1815 rc = False
1816 while not rc:
1817 change = False
1818 for x in range(len(versionlist)):
1819 pkgA = versionlist[x]
1820 try:
1821 pkgB = versionlist[x+1]
1822 except:
1823 pkgB = "0"
1824 result = compare_versions(pkgA,pkgB)
1825 if result < 0:
1826 versionlist[x] = pkgB
1827 versionlist[x+1] = pkgA
1828 change = True
1829 if not change:
1830 rc = True
1831
1832 return versionlist
1833
1834
1836 '''
1837 @description: reorder a version list
1838 @input versionlist: a list
1839 @output: the ordered list
1840 '''
1841 rc = entropy_compare_versions(a,b)
1842 if rc < 0: return -1
1843 elif rc > 0: return 1
1844 else: return 0
1845
1847 return sorted(versions,g_e_n_w_cmp,reverse = True)
1848
1850 '''
1851 descendent order
1852 versions = [(version,tag,revision),(version,tag,revision)]
1853 '''
1854 if len(versions) == 1:
1855 return versions
1856
1857 myversions = versions[:]
1858
1859
1860 rc = False
1861 while not rc:
1862 change = False
1863 for x in range(len(myversions)):
1864 pkgA = myversions[x]
1865 try:
1866 pkgB = myversions[x+1]
1867 except:
1868 pkgB = ("0","",0)
1869 result = entropy_compare_versions(pkgA,pkgB)
1870 if result < 0:
1871 myversions[x] = pkgB
1872 myversions[x+1] = pkgA
1873 change = True
1874 if not change:
1875 rc = True
1876
1877 return myversions
1878
1879
1881 try:
1882 t = int(x)
1883 del t
1884 return True
1885 except:
1886 return False
1887
1888
1889 -def istextfile(filename, blocksize = 512):
1890 f = open(filename)
1891 r = istext(f.read(blocksize))
1892 f.close()
1893 return r
1894
1896 import string
1897 _null_trans = string.maketrans("", "")
1898 text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
1899
1900 if "\0" in s:
1901 return False
1902
1903 if not s:
1904 return True
1905
1906
1907
1908 t = s.translate(_null_trans, text_characters)
1909
1910
1911
1912 if len(t)/len(s) > 0.30:
1913 return False
1914 return True
1915
1916
1917
1918
1920 mydata = {}
1921 return [mydata.setdefault(e,e) for e in alist if e not in mydata]
1922
1923
1924
1925 mappings = {
1926 "'":"''",
1927 '"':'""',
1928 ' ':'+'
1929 }
1930
1932 arg_lst = []
1933 if len(args)==1:
1934 return escape_single(args[0])
1935 for x in args:
1936 arg_lst.append(escape_single(x))
1937 return tuple(arg_lst)
1938
1940 if type(x)==type(()) or type(x)==type([]):
1941 return escape(x)
1942 if type(x)==type(""):
1943 tmpstr=''
1944 for d in range(len(x)):
1945 if x[d] in mappings.keys():
1946 if x[d] in ("'", '"'):
1947 if d+1<len(x):
1948 if x[d+1]!=x[d]:
1949 tmpstr+=mappings[x[d]]
1950 else:
1951 tmpstr+=mappings[x[d]]
1952 else:
1953 tmpstr+=mappings[x[d]]
1954 else:
1955 tmpstr+=x[d]
1956 else:
1957 tmpstr=x
1958 return tmpstr
1959
1961 if type(val)==type(""):
1962 tmpstr=''
1963 for key,item in mappings.items():
1964 val=val.replace(item,key)
1965 tmpstr = val
1966 else:
1967 tmpstr=val
1968 return tmpstr
1969
1971 arg_lst = []
1972 for x in args:
1973 arg_lst.append(unescape(x))
1974 return tuple(arg_lst)
1975
1977 myuri = spliturl(uri)[1]
1978
1979 myuri = myuri.split("@")[len(myuri.split("@"))-1]
1980 return myuri
1981
1983 import urlparse
1984 return urlparse.urlsplit(url)
1985
1987 cmd = "cd \""+pathtocompress+"\" && tar cjf \""+storepath+"\" " + \
1988 ". &> /dev/null"
1989 return subprocess.call(cmd, shell = True)
1990
1992
1993 uid = kwds.get('spf_uid')
1994 if uid != None: kwds.pop('spf_uid')
1995
1996 gid = kwds.get('spf_gid')
1997 if gid != None: kwds.pop('spf_gid')
1998
1999 write_pid_func = kwds.get('write_pid_func')
2000 if write_pid_func != None:
2001 kwds.pop('write_pid_func')
2002
2003 try:
2004 import cPickle as pickle
2005 except ImportError:
2006 import pickle
2007 pread, pwrite = os.pipe()
2008 pid = os.fork()
2009 if pid > 0:
2010 if write_pid_func != None:
2011 write_pid_func(pid)
2012 os.close(pwrite)
2013 f = os.fdopen(pread, 'rb')
2014 status, result = pickle.load(f)
2015 os.waitpid(pid, 0)
2016 f.close()
2017 if status == 0:
2018 return result
2019 else:
2020 raise result
2021 else:
2022 os.close(pread)
2023 if gid != None:
2024 os.setgid(gid)
2025 if uid != None:
2026 os.setuid(uid)
2027 try:
2028 result = f(*args, **kwds)
2029 status = 0
2030 except Exception, exc:
2031 result = exc
2032 status = 1
2033 f = os.fdopen(pwrite, 'wb')
2034 try:
2035 pickle.dump((status,result), f, pickle.HIGHEST_PROTOCOL)
2036 except pickle.PicklingError, exc:
2037 pickle.dump((2,exc), f, pickle.HIGHEST_PROTOCOL)
2038 f.close()
2039 os._exit(0)
2040
2041
2043
2044 if extractPath == None:
2045 extractPath = os.path.dirname(filepath)
2046 if not os.path.isfile(filepath):
2047 raise FileNotFound('FileNotFound: archive does not exist')
2048
2049 try:
2050 tar = tarfile.open(filepath,"r")
2051 except tarfile.ReadError:
2052 if catchEmpty:
2053 return 0
2054 raise
2055 except EOFError:
2056 return -1
2057
2058 def fix_uid_gid(tarinfo, epath):
2059
2060 uname = tarinfo.uname
2061 gname = tarinfo.gname
2062 ugdata_valid = False
2063 try:
2064 int(gname)
2065 int(uname)
2066 except ValueError:
2067 ugdata_valid = True
2068 try:
2069 if ugdata_valid:
2070
2071
2072 uid, gid = get_uid_from_user(uname), \
2073 get_gid_from_group(gname)
2074 os.lchown(epath, uid, gid)
2075 except OSError:
2076 pass
2077
2078 def mycmp(a,b):
2079 return cmp(a[0].name, b[0].name)
2080
2081 try:
2082
2083 encoded_path = extractPath.encode('utf-8')
2084 def mymf(tarinfo):
2085 epath = os.path.join(encoded_path, tarinfo.name)
2086 if tarinfo.isdir():
2087
2088
2089 try:
2090 os.makedirs(epath, 0777)
2091 except EnvironmentError:
2092 pass
2093 return tarinfo, epath
2094
2095 tar.extract(tarinfo, encoded_path)
2096
2097 del tar.members[:]
2098 return tarinfo, epath
2099
2100 entries = sorted(map(mymf, tar), mycmp, reverse = True)
2101
2102
2103 def mymf2(tardata):
2104 tarinfo, epath = tardata
2105 try:
2106
2107 tar.chown(tarinfo, epath)
2108 fix_uid_gid(tarinfo, epath)
2109 tar.utime(tarinfo, epath)
2110 mode = tarinfo.mode
2111
2112
2113 if not os.path.islink(epath):
2114 tar.chmod(tarinfo, epath)
2115
2116 except tarfile.ExtractError:
2117 if tar.errorlevel > 1:
2118 raise
2119
2120 done = map(mymf2, entries)
2121 del done
2122
2123 except EOFError:
2124 return -1
2125
2126 finally:
2127 tar.close()
2128
2129 if os.listdir(extractPath):
2130 return 0
2131 return -1
2132
2134 size = str(round(float(bytes)/1024,1))
2135 if bytes < 1024:
2136 size = str(round(float(bytes)))+"b"
2137 elif bytes < 1023999:
2138 size += "kB"
2139 elif bytes > 1023999:
2140 size = str(round(float(size)/1024,1))
2141 size += "MB"
2142 return size
2143
2145 ftppassword = uri.split("@")[:-1]
2146 if not ftppassword: return uri
2147 ftppassword = '@'.join(ftppassword)
2148 ftppassword = ftppassword.split(":")[-1]
2149 if not ftppassword:
2150 return uri
2151 newuri = uri.replace(ftppassword,"xxxxxxxx")
2152 return newuri
2153
2155 ftpuser = ftpuri.split("ftp://")[-1].split(":")[0]
2156 if (ftpuser == ""):
2157 ftpuser = "anonymous@"
2158 ftppassword = "anonymous"
2159 else:
2160 ftppassword = ftpuri.split("@")[:-1]
2161 if len(ftppassword) > 1:
2162 ftppassword = '@'.join(ftppassword)
2163 ftppassword = ftppassword.split(":")[-1]
2164 if (ftppassword == ""):
2165 ftppassword = "anonymous"
2166 else:
2167 ftppassword = ftppassword[0]
2168 ftppassword = ftppassword.split(":")[-1]
2169 if (ftppassword == ""):
2170 ftppassword = "anonymous"
2171
2172 ftpport = ftpuri.split(":")[-1]
2173 try:
2174 ftpport = int(ftpport)
2175 except ValueError:
2176 ftpport = 21
2177
2178 ftpdir = '/'
2179 if ftpuri.count("/") > 2:
2180 ftpdir = ftpuri.split("ftp://")[-1]
2181 ftpdir = ftpdir.split("/")[-1]
2182 ftpdir = ftpdir.split(":")[0]
2183 if ftpdir.endswith("/"):
2184 ftpdir = ftpdir[:len(ftpdir)-1]
2185 if not ftpdir: ftpdir = "/"
2186
2187 return ftpuser, ftppassword, ftpport, ftpdir
2188
2190 return os.path.getmtime(path)
2191
2199
2201 from datetime import datetime
2202
2203 unixtime = os.path.getmtime(path)
2204 humantime = datetime.fromtimestamp(unixtime)
2205
2206 humantime = str(humantime)
2207 outputtime = ""
2208 for char in humantime:
2209 if char != "-" and char != " " and char != ":":
2210 outputtime += char
2211 return outputtime
2212
2214 from datetime import datetime
2215 humantime = str(datetime.fromtimestamp(unixtime))
2216 return humantime
2217
2219 from datetime import datetime
2220 return datetime.fromtimestamp(unixtime)
2221
2224
2226 return time.strftime("%Y")
2227
2229
2230 mysecs = seconds
2231 myminutes = 0
2232 myhours = 0
2233 mydays = 0
2234
2235 while mysecs >= 60:
2236 mysecs -= 60
2237 myminutes += 1
2238
2239 while myminutes >= 60:
2240 myminutes -= 60
2241 myhours += 1
2242
2243 while myhours >= 24:
2244 myhours -= 24
2245 mydays += 1
2246
2247 output = []
2248 output.append(str(mysecs)+"s")
2249 if myminutes > 0 or myhours > 0:
2250 output.append(str(myminutes)+"m")
2251 if myhours > 0 or mydays > 0:
2252 output.append(str(myhours)+"h")
2253 if mydays > 0:
2254 output.append(str(mydays)+"d")
2255 output.reverse()
2256 return ':'.join(output)
2257
2258
2260
2261 if not toCleanDirs:
2262 toCleanDirs = [ etpConst['packagestmpdir'], etpConst['logdir'] ]
2263 counter = 0
2264
2265 for xdir in toCleanDirs:
2266 print_info(red(" * ")+"Cleaning "+darkgreen(xdir)+" directory...", back = True)
2267 if os.path.isdir(xdir):
2268 dircontent = os.listdir(xdir)
2269 if dircontent != []:
2270 for data in dircontent:
2271 subprocess.call(["rm","-rf",os.path.join(xdir,data)])
2272 counter += 1
2273
2274 print_info(green(" * ")+"Cleaned: "+str(counter)+" files and directories")
2275 return 0
2276
2277 -def flatten(l, ltypes=(list, tuple)):
2278 i = 0
2279 while i < len(l):
2280 while isinstance(l[i], ltypes):
2281 if not l[i]:
2282 l.pop(i)
2283 if not len(l):
2284 break
2285 else:
2286 l[i:i+1] = list(l[i])
2287 i += 1
2288 return l
2289
2291 content = []
2292 if os.path.isfile(etpConst['repositoriesconf']):
2293 f = open(etpConst['repositoriesconf'])
2294 content = f.readlines()
2295 f.close()
2296 return content
2297
2299 content = read_repositories_conf()
2300 content = [x.strip() for x in content]
2301 repolines = [x for x in content if x.startswith("repository|") and (len(x.split("|")) == 5)]
2302 content = [x for x in content if x not in repolines]
2303 for repoid in ordered_repository_list:
2304
2305 for x in repolines:
2306 repoidline = x.split("|")[1]
2307 if repoid == repoidline:
2308 content.append(x)
2309 _save_repositories_content(content)
2310
2312
2313 if repodata['repoid'].endswith(".tbz2"):
2314 return
2315
2316 content = read_repositories_conf()
2317 content = [x.strip() for x in content]
2318 if not disable and not enable:
2319 content = [x for x in content if not x.startswith("repository|"+repodata['repoid'])]
2320 if remove:
2321
2322 content = [x for x in content if not (x.startswith("#") and not x.startswith("##") and (x.find("repository|"+repodata['repoid']) != -1))]
2323 if not remove:
2324
2325 repolines = [x for x in content if x.startswith("repository|") or (x.startswith("#") and not x.startswith("##") and (x.find("repository|") != -1))]
2326 content = [x for x in content if x not in repolines]
2327
2328 repolines = [x for x in repolines if (len(x.split("|")) == 5)]
2329 repolines_data = {}
2330 repocount = 0
2331 for x in repolines:
2332 repolines_data[repocount] = {}
2333 repolines_data[repocount]['repoid'] = x.split("|")[1]
2334 repolines_data[repocount]['line'] = x
2335 if disable and x.split("|")[1] == repodata['repoid']:
2336 if not x.startswith("#"):
2337 x = "#"+x
2338 repolines_data[repocount]['line'] = x
2339 elif enable and x.split("|")[1] == repodata['repoid'] and x.startswith("#"):
2340 repolines_data[repocount]['line'] = x[1:]
2341 repocount += 1
2342
2343 if not disable and not enable:
2344
2345 line = "repository|%s|%s|%s|%s#%s#%s,%s" % ( repodata['repoid'],
2346 repodata['description'],
2347 ' '.join(repodata['plain_packages']),
2348 repodata['plain_database'],
2349 repodata['dbcformat'],
2350 repodata['service_port'],
2351 repodata['ssl_service_port'],
2352 )
2353
2354
2355 to_remove = set()
2356 for cc in repolines_data:
2357 if repolines_data[cc]['line'].startswith("#") and \
2358 (repolines_data[cc]['line'].find("repository|"+repodata['repoid']) != -1):
2359
2360 to_remove.add(cc)
2361 for x in to_remove:
2362 del repolines_data[x]
2363
2364 repolines_data[repocount] = {}
2365 repolines_data[repocount]['repoid'] = repodata['repoid']
2366 repolines_data[repocount]['line'] = line
2367
2368
2369 keys = repolines_data.keys()
2370 keys.sort()
2371 for cc in keys:
2372
2373
2374 line = repolines_data[cc]['line']
2375 content.append(line)
2376
2377 _save_repositories_content(content)
2378
2380 if os.path.isfile(etpConst['repositoriesconf']):
2381 if os.path.isfile(etpConst['repositoriesconf']+".old"):
2382 os.remove(etpConst['repositoriesconf']+".old")
2383 shutil.copy2(etpConst['repositoriesconf'],etpConst['repositoriesconf']+".old")
2384 f = open(etpConst['repositoriesconf'],"w")
2385 for x in content:
2386 f.write(x+"\n")
2387 f.flush()
2388 f.close()
2389
2391
2392
2393 if not os.access(os.path.dirname(config_file),os.W_OK):
2394 return False
2395
2396 content = []
2397 if os.path.isfile(config_file):
2398 f = open(config_file,"r")
2399 content = [x.strip() for x in f.readlines()]
2400 f.close()
2401
2402
2403 config_file_tmp = config_file+".tmp"
2404 f = open(config_file_tmp,"w")
2405 param_found = False
2406 if data:
2407 proposed_line = "%s|%s" % (name,data,)
2408 myreg = re.compile('^(%s)?[|].*$' % (name,))
2409 else:
2410 proposed_line = "# %s|" % (name,)
2411 myreg_rem = re.compile('^(%s)?[|].*$' % (name,))
2412 myreg = re.compile('^#([ \t]+?)?(%s)?[|].*$' % (name,))
2413 new_content = []
2414 for line in content:
2415 if myreg_rem.match(line):
2416 continue
2417 new_content.append(line)
2418 content = new_content
2419
2420 for line in content:
2421 if myreg.match(line):
2422 param_found = True
2423 line = proposed_line
2424 f.write(line+"\n")
2425 if not param_found:
2426 f.write(proposed_line+"\n")
2427 f.flush()
2428 f.close()
2429 shutil.move(config_file_tmp,config_file)
2430 return True
2431
2434
2436 if not os.path.exists(tbz2file):
2437 return False
2438 try:
2439 obj = open(tbz2file, "r")
2440 entry_point = locate_edb(obj)
2441 if entry_point is None:
2442 obj.close()
2443 return False
2444 obj.close()
2445 return True
2446 except (IOError, OSError,):
2447 return False
2448
2450 invalid = [ord(x) for x in string if ord(x) not in xrange(32,127)]
2451 if invalid: return False
2452 return True
2453
2455 try:
2456 os.stat(path)
2457 except OSError:
2458 return False
2459 return True
2460
2462 if re.findall(r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])', myhash):
2463 return True
2464 return False
2465
2467 try:
2468 import cStringIO as stringio
2469 except ImportError:
2470 import StringIO as stringio
2471 return stringio.StringIO()
2472
2474 count = 0
2475 f.seek(count, os.SEEK_END)
2476 size = f.tell()
2477 while count > (size*-1):
2478 count -= 1
2479 f.seek(count, os.SEEK_END)
2480 myc = f.read(1)
2481 if myc == "\n":
2482 break
2483 f.seek(count+1, os.SEEK_END)
2484 pos = f.tell()
2485 f.truncate(pos)
2486
2488 import struct
2489 f = open(elf_file,"rb")
2490 f.seek(4)
2491 elf_class = f.read(1)
2492 f.close()
2493 elf_class = struct.unpack('B',elf_class)[0]
2494 return elf_class
2495
2497 import struct
2498 f = open(elf_file,"rb")
2499 data = f.read(4)
2500 f.close()
2501 try:
2502 data = struct.unpack('BBBB',data)
2503 except struct.error:
2504 return False
2505 if data == (127, 69, 76, 70):
2506 return True
2507 return False
2508
2510 """
2511 Resolve given library name (as contained into ELF metadata) to
2512 a library path.
2513
2514 @param library: library name (as contained into ELF metadata)
2515 @type library: string
2516 @param requiring_executable: path to ELF object that contains the given
2517 library name
2518 @type requiring_executable: string
2519 @return: resolved library path
2520 @rtype: string
2521 """
2522 def do_resolve(mypaths):
2523 found_path = None
2524 for mypath in mypaths:
2525 mypath = os.path.join(etpConst['systemroot']+mypath, library)
2526 if not os.access(mypath, os.R_OK):
2527 continue
2528 if os.path.isdir(mypath):
2529 continue
2530 if not is_elf_file(mypath):
2531 continue
2532 found_path = mypath
2533 break
2534 return found_path
2535
2536 mypaths = collect_linker_paths()
2537 found_path = do_resolve(mypaths)
2538
2539 if not found_path:
2540 mypaths = read_elf_linker_paths(requiring_executable)
2541 found_path = do_resolve(mypaths)
2542
2543 return found_path
2544
2545 readelf_avail_check = False
2546 ldd_avail_check = False
2554
2562
2564 global readelf_avail_check
2565 if not readelf_avail_check:
2566 if not os.access(etpConst['systemroot']+"/usr/bin/readelf",os.X_OK):
2567 raise FileNotFound('FileNotFound: no readelf')
2568 readelf_avail_check = True
2569 data = [x.strip().split()[-1][1:-1].split(":") for x in getstatusoutput('readelf -d %s' % (elf_file,))[1].split("\n") if not ((x.find("(RPATH)") == -1) and (x.find("(RUNPATH)") == -1))]
2570 mypaths = []
2571 for mypath in data:
2572 for xpath in mypath:
2573 xpath = xpath.replace("$ORIGIN",os.path.dirname(elf_file))
2574 mypaths.append(xpath)
2575 return mypaths
2576
2578 from xml.dom import minidom
2579 doc = minidom.Document()
2580 ugc = doc.createElement("entropy")
2581 for key, value in dictionary.items():
2582 item = doc.createElement('item')
2583 item.setAttribute('value',key)
2584 if isinstance(value,str):
2585 mytype = "str"
2586 elif isinstance(value,unicode):
2587 mytype = "unicode"
2588 elif isinstance(value,list):
2589 mytype = "list"
2590 elif isinstance(value,set):
2591 mytype = "set"
2592 elif isinstance(value,frozenset):
2593 mytype = "frozenset"
2594 elif isinstance(value,dict):
2595 mytype = "dict"
2596 elif isinstance(value,tuple):
2597 mytype = "tuple"
2598 elif isinstance(value,int):
2599 mytype = "int"
2600 elif isinstance(value,float):
2601 mytype = "float"
2602 elif value == None:
2603 mytype = "None"
2604 value = "None"
2605 else: raise TypeError
2606 item.setAttribute('type',mytype)
2607 item_value = doc.createTextNode("%s" % (value,))
2608 item.appendChild(item_value)
2609 ugc.appendChild(item)
2610 doc.appendChild(ugc)
2611 return doc.toxml()
2612
2614 from xml.dom import minidom
2615 doc = minidom.parseString(xml_string)
2616 entropies = doc.getElementsByTagName("entropy")
2617 if not entropies: return {}
2618 entropy = entropies[0]
2619 items = entropy.getElementsByTagName('item')
2620
2621 my_map = {
2622 "str": str,
2623 "unicode": unicode,
2624 "list": list,
2625 "set": set,
2626 "frozenset": frozenset,
2627 "dict": dict,
2628 "tuple": tuple,
2629 "int": int,
2630 "float": float,
2631 "None": None,
2632 }
2633
2634 mydict = {}
2635 for item in items:
2636 key = item.getAttribute('value')
2637 if not key: continue
2638 mytype = item.getAttribute('type')
2639 mytype_m = my_map.get(mytype)
2640 if mytype_m == None: raise TypeError
2641 try:
2642 data = item.firstChild.data
2643 except AttributeError:
2644 data = ''
2645 if mytype in ("list","set","frozenset","dict","tuple",):
2646 if data:
2647 if data[0] not in ("(","[","s","{",): data = ''
2648 mydict[key] = eval(data)
2649 elif mytype == "None":
2650 mydict[key] = None
2651 else:
2652 mydict[key] = mytype_m(data)
2653 return mydict
2654
2656 from xml.dom import minidom
2657 doc = minidom.Document()
2658 ugc = doc.createElement("entropy")
2659 for key, value in dictionary.items():
2660 item = doc.createElement('item')
2661 item.setAttribute('value',key)
2662 item_value = doc.createTextNode(value)
2663 item.appendChild(item_value)
2664 ugc.appendChild(item)
2665 doc.appendChild(ugc)
2666 return doc.toxml()
2667
2669 from xml.dom import minidom
2670 doc = minidom.parseString(xml_string)
2671 entropies = doc.getElementsByTagName("entropy")
2672 if not entropies:
2673 return {}
2674 entropy = entropies[0]
2675 items = entropy.getElementsByTagName('item')
2676 mydict = {}
2677 for item in items:
2678 key = item.getAttribute('value')
2679 if not key: continue
2680 try:
2681 data = item.firstChild.data
2682 except AttributeError:
2683 data = ''
2684 mydict[key] = data
2685 return mydict
2686
2688 if package_tag:
2689 package_tag = "#%s" % (package_tag,)
2690 else:
2691 package_tag = ''
2692
2693 package_name = "%s:%s-%s" % (category, name, version,)
2694 package_name += package_tag
2695 package_name += etpConst['packagesext']
2696 return package_name
2697
2699 if package_tag:
2700 package_tag = "#%s" % (package_tag,)
2701 else:
2702 package_tag = ''
2703 package_name = "%s/%s-%s" % (category,name,version,)
2704 package_name += package_tag
2705 return package_name
2706
2708 f = open(filepath,"r")
2709 items = set()
2710 line = f.readline()
2711 while line:
2712 x = line.strip().rsplit("#",1)[0]
2713 if x and (not x.startswith('#')):
2714 items.add(x)
2715 line = f.readline()
2716 f.close()
2717 return items
2718
2720
2721 ldpaths = []
2722 try:
2723 f = open(etpConst['systemroot']+"/etc/ld.so.conf","r")
2724 paths = f.readlines()
2725 for path in paths:
2726 path = path.strip()
2727 if path:
2728 if path[0] == "/":
2729 ldpaths.append(os.path.normpath(path))
2730 f.close()
2731 except (IOError,OSError,TypeError,ValueError,IndexError,):
2732 pass
2733
2734
2735 if "/lib" not in ldpaths:
2736 ldpaths.append("/lib")
2737 if "/usr/lib" not in ldpaths:
2738 ldpaths.append("/usr/lib")
2739
2740 return ldpaths
2741
2743 path = set()
2744 paths = os.getenv("PATH")
2745 if paths != None:
2746 paths = set(paths.split(":"))
2747 path |= paths
2748 return path
2749
2751 mynewlist = []
2752 for item in mylist:
2753 try:
2754 mynewlist.append(item.decode("utf-8"))
2755 except UnicodeDecodeError:
2756 try:
2757 mynewlist.append(item.decode("latin1").decode("utf-8"))
2758 except:
2759 raise
2760 return mynewlist
2761