entropy.cache

24 """ 25 Entropy asynchronous and synchronous cache writer 26 and reader. This class is a Singleton and contains 27 a thread doing the cache writes asynchronously, thus 28 it must be stopped before your application is terminated 29 calling the stop() method. 30 31 Sample code: 32 33 >>> # import module 34 >>> from entropy.cache import EntropyCacher 35 ... 36 >>> # first EntropyCacher load, start it 37 >>> cacher = EntropyCacher() 38 >>> cacher.start() 39 ... 40 >>> # now store something into its cache 41 >>> cacher.push('my_identifier1', [1, 2, 3]) 42 >>> # now store something synchronously 43 >>> cacher.push('my_identifier2', [1, 2, 3], async = False) 44 ... 45 >>> # now flush all the caches to disk, and make sure all 46 >>> # is written 47 >>> cacher.sync(wait = True) 48 ... 49 >>> # now fetch something from the cache 50 >>> data = cacher.pop('my_identifier1') 51 [1, 2, 3] 52 ... 53 >>> # now discard all the cached (async) writes 54 >>> cacher.discard() 55 ... 56 >>> # and stop EntropyCacher 57 >>> cacher.stop() 58 59 """ 60 61 import entropy.dump as dumpTools 62 import entropy.tools as entropyTools 63 import copy 64

65 - def init_singleton(self):

66 """ 67 Singleton overloaded method. Equals to __init__. 68 This is the place where all the properties initialization 69 takes place. 70 """ 71 import threading 72 self.__alive = False 73 self.__cache_writer = None 74 self.__cache_buffer = Lifo() 75 self.__cache_lock = threading.Lock()

76

77 - def __copy_obj(self, obj):

78 """ 79 Return a copy of an object done by the standard 80 library "copy" module. 81 82 @param obj: object to copy 83 @type obj: any Python object 84 @rtype: copied object 85 @return: copied object 86 """ 87 return self.copy.deepcopy(obj)

88

89 - def __cacher(self):

90 """ 91 This is where the actual asynchronous copy takes 92 place. __cacher runs on a different threads and 93 all the operations done by this are atomic and 94 thread-safe. It just loops over and over until 95 __alive becomes False. 96 """ 97 while True: 98 if not self.__alive: 99 break 100 with self.__cache_lock: 101 try: 102 data = self.__cache_buffer.pop() 103 except (ValueError, TypeError,): 104 # TypeError is when objects are being destroyed 105 break # stack empty 106 key, data = data 107 d_o = self.dumpTools.dumpobj 108 if not d_o: 109 break 110 d_o(key, data)

111

112 - def __del__(self):

113 self.stop()

114

115 - def start(self):

116 """ 117 This is the method used to start the asynchronous cache 118 writer but also the whole cacher. If this method is not 119 called, the instance will always trash and cache write 120 request. 121 122 @return: None 123 """ 124 with self.__cache_lock: 125 self.__cache_buffer.clear() 126 self.__cache_writer = TimeScheduled(1, self.__cacher) 127 self.__cache_writer.set_delay_before(True) 128 self.__cache_writer.start() 129 while not self.__cache_writer.isAlive(): 130 continue 131 self.__alive = True

132

133 - def is_started(self):

134 """ 135 Return whether start is called or not. This equals to 136 checking if the cacher is running, thus is writing cache 137 to disk. 138 139 @return: None 140 """ 141 return self.__alive

142

143 - def stop(self):

144 """ 145 This method stops the execution of the cacher, which won't 146 accept cache writes anymore. The thread responsible of writing 147 to disk is stopped here and the Cacher will be back to being 148 inactive. A watchdog will avoid the thread to freeze the 149 call if the write buffer is overloaded. 150 151 @return: None 152 """ 153 154 watch_dog = 80 155 while self.__cache_buffer.is_filled() and (watch_dog > 0): 156 watch_dog -= 1 157 time.sleep(0.125) 158 self.__alive = False 159 160 with self.__cache_lock: 161 self.__cache_buffer.clear() 162 if self.__cache_writer != None: 163 self.__cache_writer.kill() 164 self.__cache_writer.join() 165 self.__cache_writer = None

166

167 - def sync(self, wait = False):

168 """ 169 This method can be called anytime and forces the instance 170 to flush all the cache writes queued to disk. If wait == False 171 a watchdog prevents this call to get stuck in case of write 172 buffer overloads. 173 174 @keyword wait: indicates if waiting until done (synchronous mode) or not 175 @type wait: bool 176 @rtype: None 177 @return: None 178 """ 179 if not self.__alive: 180 self.__cache_buffer.clear() 181 return 182 183 watch_dog = 40 184 while self.__cache_buffer.is_filled() and ((watch_dog > 0) or wait) \ 185 and self.__alive: 186 187 if not wait: 188 watch_dog -= 1 189 time.sleep(0.125) 190 191 self.__cache_buffer.clear()

192

193 - def discard(self):

194 """ 195 This method makes buffered cache to be discarded synchronously. 196 197 @return: None 198 """ 199 self.__cache_buffer.clear() 200 with self.__cache_lock: 201 self.__cache_buffer.clear() # make sure twice

202

203 - def push(self, key, data, async = True):

204 """ 205 This is the place where data is either added 206 to the write queue or written to disk (if async == False) 207 only and only if start() method has been called. 208 209 @param key: cache data identifier 210 @type key: string 211 @param data: picklable object 212 @type data: any picklable object 213 @keyword async: store cache asynchronously or not 214 @type async: bool 215 @rtype: None 216 @return: None 217 """ 218 if not self.__alive: 219 return 220 if async: 221 with self.__cache_lock: 222 try: 223 self.__cache_buffer.push((key, self.__copy_obj(data),)) 224 except TypeError: 225 # sometimes, very rarely, copy.deepcopy() is unable 226 # to properly copy an object (blame Python bug) 227 sys.stdout.write("!!! cannot cache object with key %s\n" % ( 228 key,)) 229 sys.stdout.flush() 230 else: 231 self.dumpTools.dumpobj(key, data)

232

233 - def pop(self, key):

234 """ 235 This is the place where data is retrieved from cache. 236 You must know the cache identifier used when push() 237 was called. 238 239 @param key: cache data identifier 240 @type key: string 241 @rtype: Python object 242 @return: object stored into the stack or None (if stack is empty) 243 """ 244 with self.__cache_lock: 245 l_o = self.dumpTools.loadobj 246 if not l_o: 247 return 248 return l_o(key)

Source Code for Module entropy.cache