entropy.xpak

1 # Copyright 2001-2004 Gentoo Foundation 2 # Distributed under the terms of the GNU General Public License v2 3 # $Id: xpak.py 5595 2007-01-12 08:08:03Z antarus $ 4 5 6 # The format for a tbz2/xpak: 7 # 8 # tbz2: tar.bz2 + xpak + (xpak_offset) + "STOP" 9 # xpak: "XPAKPACK" + (index_len) + (data_len) + index + data + "XPAKSTOP" 10 # index: (pathname_len) + pathname + (data_offset) + (data_len) 11 # index entries are concatenated end-to-end. 12 # data: concatenated data chunks, end-to-end. 13 # 14 # [tarball]XPAKPACKIIIIDDDD[index][data]XPAKSTOPOOOOSTOP 15 # 16 # (integer) == encodeint(integer) ===> 4 characters (big-endian copy) 17 # '+' means concatenate the fields ===> All chunks are strings 18 19 import os, shutil, errno 20 from stat import * 21

22 -def addtolist(mylist,curdir):

23 """(list, dir) --- Takes an array(list) and appends all files from dir down 24 the directory tree. Returns nothing. list is modified.""" 25 for x in os.listdir("."): 26 if os.path.isdir(x): 27 os.chdir(x) 28 addtolist(mylist,curdir+x+"/") 29 os.chdir("..") 30 else: 31 if curdir+x not in mylist: 32 mylist.append(curdir+x)

33

34 -def encodeint(myint):

35 """Takes a 4 byte integer and converts it into a string of 4 characters. 36 Returns the characters in a string.""" 37 part1=chr((myint >> 24 ) & 0x000000ff) 38 part2=chr((myint >> 16 ) & 0x000000ff) 39 part3=chr((myint >> 8 ) & 0x000000ff) 40 part4=chr(myint & 0x000000ff) 41 return part1+part2+part3+part4

42

43 -def decodeint(mystring):

44 """Takes a 4 byte string and converts it into a 4 byte integer. 45 Returns an integer.""" 46 myint=0 47 myint=myint+ord(mystring[3]) 48 myint=myint+(ord(mystring[2]) << 8) 49 myint=myint+(ord(mystring[1]) << 16) 50 myint=myint+(ord(mystring[0]) << 24) 51 return myint

52

53 -def xpak(rootdir,outfile=None):

54 """(rootdir,outfile) -- creates an xpak segment of the directory 'rootdir' 55 and under the name 'outfile' if it is specified. Otherwise it returns the 56 xpak segment.""" 57 try: 58 origdir=os.getcwd() 59 except SystemExit: 60 raise 61 except: 62 os.chdir("/") 63 origdir="/" 64 os.chdir(rootdir) 65 mylist=[] 66 67 addtolist(mylist,"") 68 mylist.sort() 69 mydata = {} 70 for x in mylist: 71 a = open(x, "r") 72 mydata[x] = a.read() 73 a.close() 74 os.chdir(origdir) 75 76 xpak_segment = xpak_mem(mydata) 77 if outfile: 78 outf = open(outfile, "w") 79 outf.write(xpak_segment) 80 outf.close() 81 else: 82 return xpak_segment

83

84 -def xpak_mem(mydata):

85 """Create an xpack segement from a map object.""" 86 indexglob="" 87 indexpos=0 88 dataglob="" 89 datapos=0 90 for x, newglob in mydata.iteritems(): 91 mydatasize=len(newglob) 92 indexglob=indexglob+encodeint(len(x))+x+encodeint(datapos)+encodeint(mydatasize) 93 indexpos=indexpos+4+len(x)+4+4 94 dataglob=dataglob+newglob 95 datapos=datapos+mydatasize 96 return "XPAKPACK" \ 97 + encodeint(len(indexglob)) \ 98 + encodeint(len(dataglob)) \ 99 + indexglob \ 100 + dataglob \ 101 + "XPAKSTOP"

102

103 -def xsplit(infile):

104 """(infile) -- Splits the infile into two files. 105 'infile.index' contains the index segment. 106 'infile.dat' contails the data segment.""" 107 myfile=open(infile,"r") 108 mydat=myfile.read() 109 myfile.close() 110 111 splits = xsplit_mem(mydat) 112 if not splits: 113 return False 114 115 myfile=open(infile+".index","w") 116 myfile.write(splits[0]) 117 myfile.close() 118 myfile=open(infile+".dat","w") 119 myfile.write(splits[1]) 120 myfile.close() 121 return True

122

123 -def xsplit_mem(mydat):

124 if mydat[0:8]!="XPAKPACK": 125 return None 126 if mydat[-8:]!="XPAKSTOP": 127 return None 128 indexsize=decodeint(mydat[8:12]) 129 #datasize=decodeint(mydat[12:16]) not used 130 return (mydat[16:indexsize+16], mydat[indexsize+16:-8])

131

132 -def getindex(infile):

133 """(infile) -- grabs the index segment from the infile and returns it.""" 134 myfile=open(infile,"r") 135 myheader=myfile.read(16) 136 if myheader[0:8]!="XPAKPACK": 137 myfile.close() 138 return 139 indexsize=decodeint(myheader[8:12]) 140 myindex=myfile.read(indexsize) 141 myfile.close() 142 return myindex

143

144 -def getboth(infile):

145 """(infile) -- grabs the index and data segments from the infile. 146 Returns an array [indexSegment,dataSegment]""" 147 myfile=open(infile,"r") 148 myheader=myfile.read(16) 149 if myheader[0:8]!="XPAKPACK": 150 myfile.close() 151 return 152 indexsize=decodeint(myheader[8:12]) 153 datasize=decodeint(myheader[12:16]) 154 myindex=myfile.read(indexsize) 155 mydata=myfile.read(datasize) 156 myfile.close() 157 return myindex, mydata

158

159 -def listindex(myindex):

160 """Print to the terminal the filenames listed in the indexglob passed in.""" 161 for x in getindex_mem(myindex): 162 print x

163

164 -def getindex_mem(myindex):

165 """Returns the filenames listed in the indexglob passed in.""" 166 myindexlen=len(myindex) 167 startpos=0 168 myret=[] 169 while ((startpos+8)<myindexlen): 170 mytestlen=decodeint(myindex[startpos:startpos+4]) 171 myret=myret+[myindex[startpos+4:startpos+4+mytestlen]] 172 startpos=startpos+mytestlen+12 173 return myret

174

175 -def searchindex(myindex,myitem):

176 """(index,item) -- Finds the offset and length of the file 'item' in the 177 datasegment via the index 'index' provided.""" 178 mylen=len(myitem) 179 myindexlen=len(myindex) 180 startpos=0 181 while ((startpos+8)<myindexlen): 182 mytestlen=decodeint(myindex[startpos:startpos+4]) 183 if mytestlen==mylen: 184 if myitem==myindex[startpos+4:startpos+4+mytestlen]: 185 #found 186 datapos=decodeint(myindex[startpos+4+mytestlen:startpos+8+mytestlen]); 187 datalen=decodeint(myindex[startpos+8+mytestlen:startpos+12+mytestlen]); 188 return datapos, datalen 189 startpos=startpos+mytestlen+12

190

191 -def getitem(myid,myitem):

192 myindex=myid[0] 193 mydata=myid[1] 194 myloc=searchindex(myindex,myitem) 195 if not myloc: 196 return None 197 return mydata[myloc[0]:myloc[0]+myloc[1]]

198

199 -def xpand(myid,mydest):

200 myindex=myid[0] 201 mydata=myid[1] 202 try: 203 origdir=os.getcwd() 204 except SystemExit: 205 raise 206 except: 207 os.chdir("/") 208 origdir="/" 209 os.chdir(mydest) 210 myindexlen=len(myindex) 211 startpos=0 212 while ((startpos+8)<myindexlen): 213 namelen=decodeint(myindex[startpos:startpos+4]) 214 datapos=decodeint(myindex[startpos+4+namelen:startpos+8+namelen]); 215 datalen=decodeint(myindex[startpos+8+namelen:startpos+12+namelen]); 216 myname=myindex[startpos+4:startpos+4+namelen] 217 dirname=os.path.dirname(myname) 218 if dirname: 219 if not os.path.exists(dirname): 220 os.makedirs(dirname) 221 mydat=open(myname,"w") 222 mydat.write(mydata[datapos:datapos+datalen]) 223 mydat.close() 224 startpos=startpos+namelen+12 225 os.chdir(origdir)

226

227 -class tbz2:

228 - def __init__(self,myfile):

229 self.file=myfile 230 self.filestat=None 231 self.index="" 232 self.infosize=0 233 self.xpaksize=0 234 self.indexsize=None 235 self.datasize=None 236 self.indexpos=None 237 self.datapos=None 238 self.scan()

239

240 - def decompose(self,datadir,cleanup=1):

241 """Alias for unpackinfo() --- Complement to recompose() but optionally 242 deletes the destination directory. Extracts the xpak from the tbz2 into 243 the directory provided. Raises IOError if scan() fails. 244 Returns result of upackinfo().""" 245 if not self.scan(): 246 raise IOError 247 if cleanup: 248 self.cleanup(datadir) 249 if not os.path.exists(datadir): 250 os.makedirs(datadir) 251 return self.unpackinfo(datadir)

252 - def compose(self,datadir,cleanup=0):

253 """Alias for recompose().""" 254 return self.recompose(datadir,cleanup)

255 - def recompose(self,datadir,cleanup=0):

256 """Creates an xpak segment from the datadir provided, truncates the tbz2 257 to the end of regular data if an xpak segment already exists, and adds 258 the new segment to the file with terminating info.""" 259 xpdata = xpak(datadir) 260 self.recompose_mem(xpdata) 261 if cleanup: 262 self.cleanup(datadir)

263

264 - def recompose_mem(self, xpdata):

265 self.scan() # Don't care about condition... We'll rewrite the data anyway. 266 myfile=open(self.file,"a+") 267 if not myfile: 268 raise IOError 269 myfile.seek(-self.xpaksize,os.SEEK_END) # 0,2 or -0,2 just mean EOF. 270 myfile.truncate() 271 myfile.write(xpdata+encodeint(len(xpdata))+"STOP") 272 myfile.flush() 273 myfile.close() 274 return 1

275

276 - def cleanup(self, datadir):

277 datadir_split = os.path.split(datadir) 278 if len(datadir_split) >= 2 and len(datadir_split[1]) > 0: 279 # This is potentially dangerous, 280 # thus the above sanity check. 281 try: 282 shutil.rmtree(datadir) 283 except OSError, oe: 284 if oe.errno == errno.ENOENT: 285 pass 286 else: 287 raise oe

288

289 - def scan(self):

290 """Scans the tbz2 to locate the xpak segment and setup internal values. 291 This function is called by relevant functions already.""" 292 try: 293 mystat=os.stat(self.file) 294 if self.filestat: 295 changed=0 296 for x in [ST_SIZE, ST_MTIME, ST_CTIME]: 297 if mystat[x] != self.filestat[x]: 298 changed=1 299 if not changed: 300 return 1 301 self.filestat=mystat 302 a=open(self.file,"r") 303 a.seek(-16,os.SEEK_END) 304 trailer=a.read() 305 self.infosize=0 306 self.xpaksize=0 307 if trailer[-4:]!="STOP": 308 a.close() 309 return 0 310 if trailer[0:8]!="XPAKSTOP": 311 a.close() 312 return 0 313 self.infosize=decodeint(trailer[8:12]) 314 self.xpaksize=self.infosize+8 315 a.seek(-(self.xpaksize),os.SEEK_END) 316 header=a.read(16) 317 if header[0:8]!="XPAKPACK": 318 a.close() 319 return 0 320 self.indexsize=decodeint(header[8:12]) 321 self.datasize=decodeint(header[12:16]) 322 self.indexpos=a.tell() 323 self.index=a.read(self.indexsize) 324 self.datapos=a.tell() 325 a.close() 326 return 2 327 except SystemExit: 328 raise 329 except: 330 return 0

331

332 - def filelist(self):

333 """Return an array of each file listed in the index.""" 334 if not self.scan(): 335 return None 336 return getindex_mem(self.index)

337

338 - def getfile(self,myfile,mydefault=None):

339 """Finds 'myfile' in the data segment and returns it.""" 340 if not self.scan(): 341 return None 342 myresult=searchindex(self.index,myfile) 343 if not myresult: 344 return mydefault 345 a=open(self.file,"r") 346 a.seek(self.datapos+myresult[0],0) 347 myreturn=a.read(myresult[1]) 348 a.close() 349 return myreturn

350

351 - def getelements(self,myfile):

352 """A split/array representation of tbz2.getfile()""" 353 mydat=self.getfile(myfile) 354 if not mydat: 355 return [] 356 return mydat.split()

357

358 - def unpackinfo(self,mydest):

359 """Unpacks all the files from the dataSegment into 'mydest'.""" 360 if not self.scan(): 361 return 0 362 try: 363 origdir=os.getcwd() 364 except SystemExit: 365 raise 366 except: 367 os.chdir("/") 368 origdir="/" 369 a=open(self.file,"r") 370 if not os.path.exists(mydest): 371 os.makedirs(mydest) 372 os.chdir(mydest) 373 startpos=0 374 while ((startpos+8)<self.indexsize): 375 namelen=decodeint(self.index[startpos:startpos+4]) 376 datapos=decodeint(self.index[startpos+4+namelen:startpos+8+namelen]); 377 datalen=decodeint(self.index[startpos+8+namelen:startpos+12+namelen]); 378 myname=self.index[startpos+4:startpos+4+namelen] 379 dirname=os.path.dirname(myname) 380 if dirname: 381 if not os.path.exists(dirname): 382 os.makedirs(dirname) 383 mydat=open(myname,"w") 384 a.seek(self.datapos+datapos) 385 mydat.write(a.read(datalen)) 386 mydat.close() 387 startpos=startpos+namelen+12 388 a.close() 389 os.chdir(origdir) 390 return 1

391

392 - def get_data(self):

393 """Returns all the files from the dataSegment as a map object.""" 394 if not self.scan(): 395 return 0 396 a = open(self.file, "r") 397 mydata = {} 398 startpos=0 399 while ((startpos+8)<self.indexsize): 400 namelen=decodeint(self.index[startpos:startpos+4]) 401 datapos=decodeint(self.index[startpos+4+namelen:startpos+8+namelen]); 402 datalen=decodeint(self.index[startpos+8+namelen:startpos+12+namelen]); 403 myname=self.index[startpos+4:startpos+4+namelen] 404 a.seek(self.datapos+datapos) 405 mydata[myname] = a.read(datalen) 406 startpos=startpos+namelen+12 407 a.close() 408 return mydata

409

410 - def getboth(self):

411 """Returns an array [indexSegment,dataSegment]""" 412 if not self.scan(): 413 return None 414 415 a = open(self.file,"r") 416 a.seek(self.datapos) 417 mydata =a.read(self.datasize) 418 a.close() 419 420 return self.index, mydata

421

Source Code for Module entropy.xpak