diff --git a/lib/entropy/client/interfaces/package.py b/lib/entropy/client/interfaces/package.py index e4e1e7279..be8d224f1 100644 --- a/lib/entropy/client/interfaces/package.py +++ b/lib/entropy/client/interfaces/package.py @@ -35,6 +35,80 @@ import entropy.tools class Package: + class ExtendedFileContentIter: + + def __init__(self, _cpath): + self._cpath = _cpath + self._f = None + + def _open_f(self): + self._f = codecs.open( + self._cpath, "r", + etpConst['conf_encoding']) + + def __iter__(self): + return self + + def __enter__(self): + if self._f is None: + self._open_f() + return self + + def __exit__(self, exc_type, exc_val, tb): + if self._f is not None: + self._f.close() + + def next(self): + if self._f is None: + self._open_f() + line = self._f.readline() + if not line: + raise StopIteration() + _package_id, _ftype, _path = line[:-1].split("|", 2) + # must be legal or die! + _package_id = int(_package_id) + return _package_id, _path, _ftype + + def close(self): + if self._f is not None: + self._f.close() + + class SimpleFileContentIter: + + def __init__(self, _cpath): + self._cpath = _cpath + self._f = None + + def _open_f(self): + self._f = codecs.open( + self._cpath, "r", + etpConst['conf_encoding']) + + def __iter__(self): + return self + + def __enter__(self): + if self._f is None: + self._open_f() + return self + + def __exit__(self, exc_type, exc_val, tb): + if self._f is not None: + self._f.close() + + def next(self): + if self._f is None: + self._open_f() + line = self._f.readline() + if not line: + raise StopIteration() + _nothing, _ftype, _path = line[:-1].split("|", 2) + return _path + + def close(self): + if self._f is not None: + self._f.close() + def __init__(self, entropy_client): if not isinstance(entropy_client, Client): @@ -64,6 +138,17 @@ class Package: return unicode(repr(self)) def kill(self): + + # remove temporary content files + # created by __generate_content_file() + content_files = self.pkgmeta.get( + '__content_files__', []) + for content_file in content_files: + try: + os.remove(content_file) + except (OSError, IOError): + pass + self.pkgmeta.clear() self._package_match = () @@ -1960,7 +2045,13 @@ class Package: if smart_pkg or self.pkgmeta['merge_from']: data = dbconn.getPackageData(self.pkgmeta['idpackage'], - content_insert_formatted = True, get_changelog = False) + content_insert_formatted = True, + get_changelog = False, get_content = False) + + content = dbconn.retrieveContentIter( + self.pkgmeta['idpackage']) + data['content_file'] = self.__generate_content_file( + content, package_id = self.pkgmeta['idpackage']) if self.pkgmeta['removeidpackage'] != -1: self.pkgmeta['removecontent'].update( @@ -1991,14 +2082,10 @@ class Package: # contain only one entry pkg_idpackage = sorted(pkg_dbconn.listAllPackageIds(), reverse = True)[0] - content = pkg_dbconn.retrieveContent( - pkg_idpackage, extended = True, - formatted = True, insert_formatted = True - ) - - real_idpk = self.pkgmeta['idpackage'] - content = [(real_idpk, x, y,) for orig_idpk, x, y in content] - data['content'] = content + content = pkg_dbconn.retrieveContentIter( + pkg_idpackage) + data['content_file'] = self.__generate_content_file( + content, package_id = self.pkgmeta['idpackage']) # setup content safety metadata, get from package data['content_safety'] = pkg_dbconn.retrieveContentSafety( @@ -2022,13 +2109,16 @@ class Package: self.pkgmeta['removecontent'] -= my_remove_content # filter out files not installed from content metadata - self.__filter_out_items_not_installed_from_content(data) + self.__filter_out_items_not_installed_from_content( + data['content_file']) - # this is needed to make postinstall trigger to work properly - trigger_content = set((x[1] for x in data['content'])) - self.pkgmeta['triggers']['install']['content'] = trigger_content + # this is needed to make postinstall trigger work properly + content_iter_class = Package.ExtendedFileContentIter + self.pkgmeta['triggers']['install']['content_iter_class'] = \ + content_iter_class + self.pkgmeta['triggers']['install']['content_file'] = \ + data['content_file'] - # open client db # always set data['injected'] to False # installed packages database SHOULD never have more # than one package for scope (key+slot) @@ -2055,8 +2145,21 @@ class Package: inst_repo = self._entropy.installed_repository() - idpackage = inst_repo.handlePackage(data, - forcedRevision = data['revision'], formattedContent = True) + data['content'] = None + try: + # now we are ready to craft a 'content' iter object + data['content'] = content_iter_class( + data['content_file']) + idpackage = inst_repo.handlePackage( + data, forcedRevision = data['revision'], + formattedContent = True) + finally: + if data['content'] is not None: + try: + data['content'].close() + data['content'] = None + except (OSError, IOError): + data['content'] = None # update datecreation ctime = time.time() @@ -2074,17 +2177,34 @@ class Package: inst_repo.commit() return idpackage - def __filter_out_items_not_installed_from_content(self, pkg_data): + def __filter_out_items_not_installed_from_content(self, content_file): - items_not_installed = self.pkgmeta.get('items_not_installed', set()) + items_not_installed = self.pkgmeta.get( + 'items_not_installed', set()) + if not items_not_installed: + # nothing to dos + return - # CONTENT metadata getting here is always already formatted - # for EntropyRepository insertion - def filter_content(content_item): - pkg_id, pkg_path, path_type = content_item - return pkg_path not in items_not_installed - - pkg_data['content'] = filter(filter_content, pkg_data['content']) + # rewrite content_file + tmp_content_file = content_file + "__filter_tmp" + enc = etpConst['conf_encoding'] + try: + with codecs.open(tmp_content_file, "w", enc) as tmp_w: + with codecs.open(content_file, "r", enc) as tmp_r: + line = tmp_r.readline() + while line: + # strip \n safely + _pkg_id, ftype, _path = line[:-1].split("|", 2) + if _path not in items_not_installed: + tmp_w.write(line) + line = tmp_r.readline() + os.rename(tmp_content_file, content_file) + except Exception as exc: + try: + os.remove(tmp_content_file) + except OSError: + pass + raise exc def __fill_image_dir(self, merge_from, image_dir): @@ -3828,6 +3948,7 @@ class Package: metadata = { 'splitdebug': splitdebug, 'splitdebug_dirs': splitdebug_dirs, + '__content_files__': [], } return metadata @@ -3853,6 +3974,66 @@ class Package: self.__prepared = True + @staticmethod + def _generate_content_file(content, package_id = None): + """ + Generate a file containing the "content" metadata, + reading by content list or iterator. Each item + of "content" must contain (path, ftype). + Each item shall be written to file, one per line, + in the following form: "[|]|". + The order of the element in "content" will be kept. + """ + tmp_dir = os.path.join( + etpConst['entropyunpackdir'], + "__generate_content_file_f") + try: + os.makedirs(tmp_dir, 0o755) + except OSError as err: + if err.errno != errno.EEXIST: + raise + + tmp_fd, tmp_path = None, None + try: + tmp_fd, tmp_path = tempfile.mkstemp( + prefix="PackageContent", + dir=tmp_dir) + with entropy.tools.codecs_fdopen( + tmp_fd, "w", etpConst['conf_encoding']) as tmp_f: + for path, ftype in content: + if package_id is not None: + tmp_f.write(str(package_id)) + tmp_f.write("|") + tmp_f.write(ftype) + tmp_f.write("|") + tmp_f.write(path) + tmp_f.write("\n") + return tmp_path + except Exception as exc: + if tmp_path is not None: + try: + os.remove(tmp_path) + except (OSError, IOError): + pass + raise exc + finally: + if tmp_fd is None: + try: + os.close(tmp_fd) + except OSError: + pass + + def __generate_content_file(self, content, package_id = None): + content_path = None + try: + content_path = Package._generate_content_file( + content, package_id = package_id) + return content_path + finally: + if content_path is not None: + self.pkgmeta['__content_files__'].append( + content_path) + def __generate_remove_metadata(self): idpackage = self._package_match[0] diff --git a/lib/entropy/client/interfaces/trigger.py b/lib/entropy/client/interfaces/trigger.py index f74a6dd8b..03a65e645 100644 --- a/lib/entropy/client/interfaces/trigger.py +++ b/lib/entropy/client/interfaces/trigger.py @@ -132,7 +132,14 @@ class Trigger: functions.append(self._trigger_spm_postinstall) break - cont_dirs = set((os.path.dirname(x) for x in self._pkgdata['content'])) + _cont_path = self._pkgdata['content_file'] + content_iter_class = self._pkgdata['content_iter_class'] + cont_dirs = set() + with content_iter_class(_cont_path) as cont_it: + for _pkg_id, _path, _ftype in cont_it: + if _ftype == "dir": + cont_dirs.add(_path) + ldpaths = entropy.tools.collect_linker_paths() if len(cont_dirs) != len(cont_dirs - set(ldpaths)): functions.insert(0, self._trigger_env_update) diff --git a/lib/tests/client.py b/lib/tests/client.py index fc7e417bf..1529659c6 100644 --- a/lib/tests/client.py +++ b/lib/tests/client.py @@ -220,16 +220,30 @@ class EntropyClientTest(unittest.TestCase): data = self.Spm.extract_package_metadata(test_pkg) idpackage = dbconn.addPackage(data) pkgdata = dbconn.getTriggerData(idpackage) - pkgdata['trigger'] = """\ + content_file = None + trigger = None + try: + pkgdata['content_iter_class'] = Package.ExtendedFileContentIter + content_file = Package._generate_content_file( + dbconn.retrieveContentIter(idpackage), + package_id = idpackage) + pkgdata['content_file'] = content_file + pkgdata['trigger'] = """\ #!%s echo $@ exit 42 """ % (etpConst['trigger_sh_interpreter'],) - trigger = self.Client.Triggers("install", 'postinstall', pkgdata, - pkgdata) - trigger.prepare() - exit_st = trigger._do_trigger_call_ext_generic() - trigger.kill() + trigger = self.Client.Triggers( + "install", 'postinstall', pkgdata, pkgdata) + trigger.prepare() + exit_st = trigger._do_trigger_call_ext_generic() + trigger.kill() + finally: + if trigger is not None: + trigger.kill() + if content_file is not None: + os.remove(content_file) + self.assertEqual(exit_st, 42) def test_python_trigger(self): @@ -239,16 +253,29 @@ exit 42 data = self.Spm.extract_package_metadata(test_pkg) idpackage = dbconn.addPackage(data) pkgdata = dbconn.getTriggerData(idpackage) - pkgdata['trigger'] = """\ + content_file = None + trigger = None + try: + pkgdata['content_iter_class'] = Package.ExtendedFileContentIter + content_file = Package._generate_content_file( + dbconn.retrieveContentIter(idpackage), + package_id = idpackage) + pkgdata['content_file'] = content_file + + pkgdata['trigger'] = """\ import os os.system("echo hello") my_ext_status = 42 """ - trigger = self.Client.Triggers("install", 'postinstall', pkgdata, - pkgdata) - trigger.prepare() - exit_st = trigger._do_trigger_call_ext_generic() - trigger.kill() + trigger = self.Client.Triggers( + "install", 'postinstall', pkgdata, pkgdata) + trigger.prepare() + exit_st = trigger._do_trigger_call_ext_generic() + finally: + if trigger is not None: + trigger.kill() + if content_file is not None: + os.remove(content_file) self.assertEqual(exit_st, 42) def test_python_trigger2(self): @@ -258,7 +285,15 @@ my_ext_status = 42 data = self.Spm.extract_package_metadata(test_pkg) idpackage = dbconn.addPackage(data) pkgdata = dbconn.getTriggerData(idpackage) - pkgdata['trigger'] = """\ + trigger = None + content_file = None + try: + pkgdata['content_iter_class'] = Package.ExtendedFileContentIter + content_file = Package._generate_content_file( + dbconn.retrieveContentIter(idpackage), + package_id = idpackage) + pkgdata['content_file'] = content_file + pkgdata['trigger'] = """\ import os import subprocess from entropy.const import etpConst @@ -284,11 +319,16 @@ if stage == "postinstall": else: my_ext_status = 0 """ - trigger = self.Client.Triggers("install", 'postinstall', pkgdata, - pkgdata) - trigger.prepare() - exit_st = trigger._do_trigger_call_ext_generic() - trigger.kill() + trigger = self.Client.Triggers( + "install", 'postinstall', pkgdata, pkgdata) + trigger.prepare() + exit_st = trigger._do_trigger_call_ext_generic() + finally: + if trigger is not None: + trigger.kill() + if content_file is not None: + os.remove(content_file) + self.assertEqual(exit_st, 42) def _do_pkg_test(self, pkg_path, pkg_atom):