From 3ab1d8db4b28fca71480736de24ea7ec45f07e55 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Tue, 20 Jul 2010 18:24:06 +0200 Subject: [PATCH] [entropy.client] complete implementation of older entropy downloaded packages cleanup --- conf/client.conf | 4 +- .../entropy/client/interfaces/methods.py | 80 ++++++++++++++++ services/client-updates-daemon | 95 ++----------------- 3 files changed, 92 insertions(+), 87 deletions(-) diff --git a/conf/client.conf b/conf/client.conf index dd8482046..3efafda6e 100644 --- a/conf/client.conf +++ b/conf/client.conf @@ -70,8 +70,8 @@ package-hashes|sha1 gpg # Default parameter if unset: enable # forced-updates|enable -# Number of days that should pass before old package files not availabile -# in repos anymore get removed from cache (freeing up space). +# Number of days that should pass before package files +# get removed from cache automatically. # Note that this feature should be disabled in server-environments where # storing packages cache is subject to different policies. # The daemon in charge of this is client-updates-daemon available via diff --git a/libraries/entropy/client/interfaces/methods.py b/libraries/entropy/client/interfaces/methods.py index 66657d2cd..c711564a5 100644 --- a/libraries/entropy/client/interfaces/methods.py +++ b/libraries/entropy/client/interfaces/methods.py @@ -818,6 +818,86 @@ class RepositoryMixin: os.access(os.path.join(client_dbdir, x), os.R_OK) ] + def clean_downloaded_packages(self): + """ + Clean Entropy Client downloaded packages older than the setting + specified by "packages-autoprune-days" in /etc/entropy/client.conf. + If setting is not set or invalid, this method will do nothing. + Otherwise, files older than given settings (representing time delta in + days) will be removed. + + @return: list of removed package file paths. + @rtype: list + """ + client_settings = self._settings[self.sys_settings_client_plugin_id] + misc_settings = client_settings['misc'] + autoprune_days = misc_settings.get('autoprune_days', None) + if autoprune_days is None: + # sorry, feature disabled or not available + return [] + + def filter_expired_pkg(pkg_path): + + if not os.path.isfile(pkg_path): + return False + if not os.access(pkg_path, os.R_OK | os.W_OK): + return False + try: + mtime = os.path.getmtime(pkg_path) + except (OSError, IOError): + return False + if (mtime + (autoprune_days*24*3600)) > time.time(): + return False + return True + + repo_pkgs_dirs = [os.path.join(etpConst['entropyworkdir'], x, + etpConst['currentarch']) for x in \ + etpConst['packagesrelativepaths']] + + def get_removable_packages(): + removable_pkgs = set() + for pkg_dir in repo_pkgs_dirs: + if not os.path.isdir(pkg_dir): + continue + for branch in os.listdir(pkg_dir): + branch_dir = os.path.join(pkg_dir, branch) + dir_repo_pkgs = set((os.path.join(branch_dir, x) \ + for x in os.listdir(branch_dir))) + # filter out hostile paths + dir_repo_pkgs = set((x for x in dir_repo_pkgs \ + if os.path.realpath(x).startswith(branch_dir) \ + and os.path.realpath(x).endswith( + etpConst['packagesext']))) + removable_pkgs |= dir_repo_pkgs + return removable_pkgs + + removable_pkgs = get_removable_packages() + removable_pkgs = sorted(filter(filter_expired_pkg, + removable_pkgs)) + + if not removable_pkgs: + return [] + + successfully_removed = [] + for repo_pkg in removable_pkgs: + try: + os.remove(repo_pkg) + successfully_removed.append(repo_pkg) + except OSError: + pass + try: + os.remove(repo_pkg + etpConst['packagesmd5fileext']) + except OSError: + pass + try: + os.remove(repo_pkg + \ + etpConst['packagemtimefileext']) + except OSError: + # KeyError is for backward compatibility + pass + + return successfully_removed + def _run_repositories_post_branch_switch_hooks(self, old_branch, new_branch): """ This method is called whenever branch is successfully switched by user. diff --git a/services/client-updates-daemon b/services/client-updates-daemon index 13bf8278a..f6930f96a 100644 --- a/services/client-updates-daemon +++ b/services/client-updates-daemon @@ -244,98 +244,23 @@ class UpdatesDaemon(dbus.service.Object): def cleanup_entropy_cache(self): with self.__is_working_mutex: - entropy = Entropy() - acquired = self.__acquire_entropy_locks(entropy) - if not acquired: - entropy.shutdown() - return True # respawn later + entropy = None try: - sys_set = SysSet() - sys_set_plg_id = \ - etpConst['system_settings_plugins_ids']['client_plugin'] - client_settings = sys_set[sys_set_plg_id] - misc_settings = client_settings['misc'] - autoprune_days = misc_settings.get('autoprune_days', None) - if autoprune_days is None: - # sorry, feature disabled or not available - # (upgrade entropy?) - return False - - def filter_expired_pkg(pkg_path): - - pkg_path = os.path.abspath(pkg_path) - # security check - if not pkg_path.startswith(etpConst['entropyworkdir']): - return False - if not pkg_path.endswith(etpConst['packagesext']): - return False - - if not os.path.isfile(pkg_path): - return False - if not os.access(pkg_path, os.R_OK | os.W_OK): - return False - try: - mtime = os.path.getmtime(pkg_path) - except (OSError, IOError): - return False - if (mtime + (autoprune_days*24*3600)) > time.time(): - return False - return True - - # only check against available repositories - # skip disabled and corrupted - for repo_id in sys_set['repositories']['available']: - - try: - repo_db = entropy.open_repository(repo_id) - except RepositoryError as err: - write_output( - "cleanup_entropy_cache: open_repository() error: %s" % (err,)) - continue - except Exception as err: - write_output( - "cleanup_entropy_cache: open_repository() general error: %s" % (err,)) - continue - - repo_pkgs = set(repo_db.listAllDownloads(do_sort = False, - full_path = True)) - repo_pkgs = [os.path.join(etpConst['entropyworkdir'], x) \ - for x in repo_pkgs] - repo_pkgs = sorted(filter(filter_expired_pkg, repo_pkgs)) - if not repo_pkgs: - if DAEMON_DEBUG: - write_output( - "cleanup_entropy_cache: removing from %s: nothing" % ( - repo_id,)) - continue - - if DAEMON_DEBUG: - write_output( - "cleanup_entropy_cache: removing from %s: %s" % ( - repo_id, ', '.join(repo_pkgs),)) - for repo_pkg in repo_pkgs: - try: - os.remove(repo_pkg) - except OSError: - pass - try: - os.remove(repo_pkg + etpConst['packagesmd5fileext']) - except OSError: - pass - try: - os.remove(repo_pkg + \ - etpConst['packagemtimefileext']) - except (OSError, KeyError,): - # KeyError is for backward compatibility - continue + entropy = Entropy() + acquired = self.__acquire_entropy_locks(entropy) + if not acquired: + return True # respawn later + if hasattr(entropy, 'clean_downloaded_packages'): + entropy.clean_downloaded_packages() return False finally: - self.__release_entropy_locks(entropy) - entropy.shutdown() + if entropy is not None: + self.__release_entropy_locks(entropy) + entropy.shutdown() def check_system_changes(self): if self.__trigger_oncall_updater: