310 lines
9.8 KiB
Python
310 lines
9.8 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""
|
|
|
|
@author: Fabio Erculiani <lxnay@sabayon.org>
|
|
@contact: lxnay@sabayon.org
|
|
@copyright: Fabio Erculiani
|
|
@license: GPL-2
|
|
|
|
B{Entropy Framework object disk serializer module}.
|
|
|
|
This module contains Entropy Python object serialization functions and
|
|
disk dumpers.
|
|
|
|
Serialized objects are stored to disk with proper permissions by default
|
|
into path given by entropy.const's etpConst['dumpstoragedir'].
|
|
|
|
Permissions are set using entropy.const's const_setup_perms and
|
|
const_setup_file functions.
|
|
|
|
Objects are serialized using Python's cPickle/pickle modules, thus
|
|
they must be "pickable". Please read Python Library reference for
|
|
more information.
|
|
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import time
|
|
|
|
from entropy.const import etpConst, const_setup_file, const_is_python3
|
|
# Always use MAX pickle protocol to <=2, to allow Python 2 and 3 support
|
|
COMPAT_PICKLE_PROTOCOL = 0
|
|
|
|
if const_is_python3():
|
|
import pickle
|
|
else:
|
|
try:
|
|
import cPickle as pickle
|
|
except ImportError:
|
|
import pickle
|
|
|
|
pickle.HIGHEST_PROTOCOL = COMPAT_PICKLE_PROTOCOL
|
|
pickle.DEFAULT_PROTOCOL = COMPAT_PICKLE_PROTOCOL
|
|
|
|
D_EXT = etpConst['cachedumpext']
|
|
D_DIR = etpConst['dumpstoragedir']
|
|
E_GID = etpConst['entropygid']
|
|
if E_GID == None:
|
|
E_GID = 0
|
|
|
|
|
|
def dumpobj(name, my_object, complete_path = False, ignore_exceptions = True,
|
|
dump_dir = None, custom_permissions = None):
|
|
"""
|
|
Dump pickable object to file
|
|
|
|
@param name: name of the object
|
|
@type name: string
|
|
@param my_object: object to dump
|
|
@type my_object: any Python "pickable" object
|
|
@keyword complete_path: consider "name" argument as
|
|
a complete path (this overrides the default dump
|
|
path given by etpConst['dumpstoragedir'])
|
|
@type complete_path: bool
|
|
@keyword ignore_exceptions: ignore any possible exception
|
|
(EOFError, IOError, OSError,)
|
|
@type ignore_exceptions: bool
|
|
@keyword dump_dir: alternative dump directory
|
|
@type dump_dir: string
|
|
@keyword custom_permissions: give custom permission bits
|
|
@type custom_permissions: octal
|
|
@return: None
|
|
@rtype: None
|
|
@raise EOFError: could be caused by pickle.dump, ignored if
|
|
ignore_exceptions is True
|
|
@raise IOError: could be caused by pickle.dump, ignored if
|
|
ignore_exceptions is True
|
|
@raise OSError: could be caused by pickle.dump, ignored if
|
|
ignore_exceptions is True
|
|
"""
|
|
if dump_dir is None:
|
|
dump_dir = D_DIR
|
|
if custom_permissions is None:
|
|
custom_permissions = 0o664
|
|
|
|
while True: # trap ctrl+C
|
|
try:
|
|
if complete_path:
|
|
dmpfile = name
|
|
c_dump_dir = os.path.dirname(name)
|
|
else:
|
|
_dmp_path = os.path.join(dump_dir, name)
|
|
dmpfile = _dmp_path+D_EXT
|
|
c_dump_dir = os.path.dirname(_dmp_path)
|
|
|
|
my_dump_dir = c_dump_dir
|
|
d_paths = []
|
|
while not os.path.isdir(my_dump_dir):
|
|
d_paths.append(my_dump_dir)
|
|
my_dump_dir = os.path.dirname(my_dump_dir)
|
|
if d_paths:
|
|
d_paths = sorted(d_paths)
|
|
for d_path in d_paths:
|
|
os.mkdir(d_path)
|
|
const_setup_file(d_path, E_GID, 0o775)
|
|
|
|
tmp_dmpfile = dmpfile + ".tmp"
|
|
with open(tmp_dmpfile, "wb") as dmp_f:
|
|
if const_is_python3():
|
|
pickle.dump(my_object, dmp_f,
|
|
protocol = COMPAT_PICKLE_PROTOCOL, fix_imports = True)
|
|
else:
|
|
pickle.dump(my_object, dmp_f)
|
|
dmp_f.flush()
|
|
const_setup_file(tmp_dmpfile, E_GID, custom_permissions)
|
|
os.rename(tmp_dmpfile, dmpfile)
|
|
|
|
except RuntimeError:
|
|
try:
|
|
os.remove(dmpfile)
|
|
except OSError:
|
|
pass
|
|
except (EOFError, IOError, OSError):
|
|
if not ignore_exceptions:
|
|
raise
|
|
break
|
|
|
|
def serialize(myobj, ser_f, do_seek = True):
|
|
"""
|
|
Serialize object to ser_f (file)
|
|
|
|
@param myobj: Python object to serialize
|
|
@type myobj: any Python picklable object
|
|
@param ser_f: file object to write to
|
|
@type ser_f: file object
|
|
@keyword do_seek: move file cursor back to the beginning
|
|
of ser_f
|
|
@type do_seek: bool
|
|
@return: file object where data has been written
|
|
@rtype: file object
|
|
@raise RuntimeError: caused by pickle.dump in case of
|
|
system errors
|
|
@raise EOFError: caused by pickle.dump in case of
|
|
race conditions on multi-processing or multi-threading
|
|
@raise IOError: caused by pickle.dump in case of
|
|
race conditions on multi-processing or multi-threading
|
|
@raise pickle.PicklingError: when object cannot be recreated
|
|
"""
|
|
if const_is_python3():
|
|
pickle.dump(myobj, ser_f, protocol = COMPAT_PICKLE_PROTOCOL,
|
|
fix_imports = True)
|
|
else:
|
|
pickle.dump(myobj, ser_f)
|
|
ser_f.flush()
|
|
if do_seek:
|
|
ser_f.seek(0)
|
|
return ser_f
|
|
|
|
def unserialize(serial_f):
|
|
"""
|
|
Unserialize file to object (file)
|
|
|
|
@param serial_f: file object which data will be read from
|
|
@type serial_f: file object
|
|
@return: rebuilt object
|
|
@rtype: any Python pickable object
|
|
@raise pickle.UnpicklingError: when object cannot be recreated
|
|
"""
|
|
if const_is_python3():
|
|
return pickle.load(serial_f, fix_imports = True,
|
|
encoding = etpConst['conf_raw_encoding'])
|
|
else:
|
|
return pickle.load(serial_f)
|
|
|
|
def unserialize_string(mystring):
|
|
"""
|
|
Unserialize pickle string to object
|
|
|
|
@param mystring: data stream in string form to reconstruct
|
|
@type mystring: string
|
|
@return: reconstructed object
|
|
@rtype: any Python pickable object
|
|
@raise pickle.UnpicklingError: when object cannot be recreated
|
|
"""
|
|
if const_is_python3():
|
|
return pickle.loads(mystring, fix_imports = True,
|
|
encoding = etpConst['conf_raw_encoding'])
|
|
else:
|
|
return pickle.loads(mystring)
|
|
|
|
def serialize_string(myobj):
|
|
"""
|
|
Serialize object to string
|
|
|
|
@param myobj: object to serialize
|
|
@type myobj: any Python picklable object
|
|
@return: serialized string
|
|
@rtype: string
|
|
@raise pickle.PicklingError: when object cannot be recreated
|
|
"""
|
|
if const_is_python3():
|
|
return pickle.dumps(myobj, protocol = COMPAT_PICKLE_PROTOCOL,
|
|
fix_imports = True, encoding = etpConst['conf_raw_encoding'])
|
|
else:
|
|
return pickle.dumps(myobj)
|
|
|
|
def loadobj(name, complete_path = False, dump_dir = None, aging_days = None):
|
|
"""
|
|
Load object from a file
|
|
@param name: name of the object to load
|
|
@type name: string
|
|
@keyword complete_path: determine whether name argument
|
|
is a complete disk path to serialized object
|
|
@type complete_path: bool
|
|
@keyword dump_dir: alternative dump directory
|
|
@type dump_dir: string
|
|
@keyword aging_days: if int, consider the cached file invalid
|
|
if older than aging_days.
|
|
@type aging_days: int
|
|
@return: object or None
|
|
@rtype: any Python pickable object or None
|
|
"""
|
|
if dump_dir is None:
|
|
dump_dir = D_DIR
|
|
|
|
while True:
|
|
if complete_path:
|
|
dmpfile = name
|
|
else:
|
|
dump_path = os.path.join(dump_dir, name)
|
|
dmpfile = dump_path + D_EXT
|
|
|
|
if os.path.isfile(dmpfile) and os.access(dmpfile, os.R_OK):
|
|
if aging_days is not None:
|
|
cur_t = time.time()
|
|
try:
|
|
mtime = os.path.getmtime(dmpfile)
|
|
except (IOError, OSError,):
|
|
mtime = 0.0
|
|
if abs(cur_t - mtime) > (aging_days * 86400):
|
|
# do not unlink since other consumers might
|
|
# have different aging settings.
|
|
#try:
|
|
# os.remove(dmpfile)
|
|
#except (OSError, IOError):
|
|
# # did my best
|
|
# pass
|
|
return None
|
|
|
|
try:
|
|
with open(dmpfile, "rb") as dmp_f:
|
|
obj = None
|
|
try:
|
|
if const_is_python3():
|
|
obj = pickle.load(dmp_f, fix_imports = True,
|
|
encoding = etpConst['conf_raw_encoding'])
|
|
else:
|
|
obj = pickle.load(dmp_f)
|
|
except (ValueError, EOFError, IOError,
|
|
OSError, pickle.UnpicklingError, TypeError,
|
|
AttributeError, ImportError, SystemError,):
|
|
pass
|
|
return obj
|
|
except (IOError, OSError,):
|
|
pass
|
|
break
|
|
|
|
def getobjmtime(name, dump_dir = None):
|
|
"""
|
|
Get dumped object mtime
|
|
|
|
@param name: object name
|
|
@type name: string
|
|
@keyword dump_dir: alternative dump directory
|
|
@type dump_dir: string
|
|
@return: mtime of the file containing the serialized object or 0
|
|
if not found
|
|
@rtype: int
|
|
"""
|
|
if dump_dir is None:
|
|
dump_dir = D_DIR
|
|
mtime = 0
|
|
dump_path = os.path.join(dump_dir, name+D_EXT)
|
|
try:
|
|
mtime = os.path.getmtime(dump_path)
|
|
except (IOError, OSError):
|
|
mtime = 0
|
|
return int(mtime)
|
|
|
|
def removeobj(name, dump_dir = None):
|
|
"""
|
|
Remove cached object referenced by its object name
|
|
|
|
@param name: object name
|
|
@type name: string
|
|
@keyword dump_dir: alternative dump directory
|
|
@type dump_dir: string
|
|
@return: bool representing whether object has been
|
|
removed or not
|
|
@rtype: bool
|
|
@raise OSError: in case of troubles with os.remove()
|
|
"""
|
|
if dump_dir is None:
|
|
dump_dir = D_DIR
|
|
filepath = dump_dir + os.path.sep + name + D_EXT
|
|
if os.path.isfile(filepath) and os.access(filepath, os.W_OK):
|
|
os.remove(filepath)
|
|
return True
|
|
return False
|