[entropy.client] complete implementation of older entropy downloaded packages cleanup

conf/client.conf

@@ -70,8 +70,8 @@ package-hashes|sha1 gpg
 # Default parameter if unset: enable
 # forced-updates|enable
 
-# Number of days that should pass before old package files not availabile
-# in repos anymore get removed from cache (freeing up space).
+# Number of days that should pass before package files
+# get removed from cache automatically.
 # Note that this feature should be disabled in server-environments where
 # storing packages cache is subject to different policies.
 # The daemon in charge of this is client-updates-daemon available via

libraries/entropy/client/interfaces/methods.py

@@ -818,6 +818,86 @@ class RepositoryMixin:
                     os.access(os.path.join(client_dbdir, x), os.R_OK)
         ]
 
+    def clean_downloaded_packages(self):
+        """
+        Clean Entropy Client downloaded packages older than the setting
+        specified by "packages-autoprune-days" in /etc/entropy/client.conf.
+        If setting is not set or invalid, this method will do nothing.
+        Otherwise, files older than given settings (representing time delta in
+        days) will be removed.
+
+        @return: list of removed package file paths.
+        @rtype: list
+        """
+        client_settings = self._settings[self.sys_settings_client_plugin_id]
+        misc_settings = client_settings['misc']
+        autoprune_days = misc_settings.get('autoprune_days', None)
+        if autoprune_days is None:
+            # sorry, feature disabled or not available
+            return []
+
+        def filter_expired_pkg(pkg_path):
+
+            if not os.path.isfile(pkg_path):
+                return False
+            if not os.access(pkg_path, os.R_OK | os.W_OK):
+                return False
+            try:
+                mtime = os.path.getmtime(pkg_path)
+            except (OSError, IOError):
+                return False
+            if (mtime + (autoprune_days*24*3600)) > time.time():
+                return False
+            return True
+
+        repo_pkgs_dirs = [os.path.join(etpConst['entropyworkdir'], x,
+            etpConst['currentarch']) for x in \
+                etpConst['packagesrelativepaths']]
+
+        def get_removable_packages():
+            removable_pkgs = set()
+            for pkg_dir in repo_pkgs_dirs:
+                if not os.path.isdir(pkg_dir):
+                    continue
+                for branch in os.listdir(pkg_dir):
+                    branch_dir = os.path.join(pkg_dir, branch)
+                    dir_repo_pkgs = set((os.path.join(branch_dir, x) \
+                        for x in os.listdir(branch_dir)))
+                    # filter out hostile paths
+                    dir_repo_pkgs = set((x for x in dir_repo_pkgs \
+                        if os.path.realpath(x).startswith(branch_dir) \
+                        and os.path.realpath(x).endswith(
+                            etpConst['packagesext'])))
+                    removable_pkgs |= dir_repo_pkgs
+            return removable_pkgs
+
+        removable_pkgs = get_removable_packages()
+        removable_pkgs = sorted(filter(filter_expired_pkg,
+            removable_pkgs))
+
+        if not removable_pkgs:
+            return []
+
+        successfully_removed = []
+        for repo_pkg in removable_pkgs:
+            try:
+                os.remove(repo_pkg)
+                successfully_removed.append(repo_pkg)
+            except OSError:
+                pass
+            try:
+                os.remove(repo_pkg + etpConst['packagesmd5fileext'])
+            except OSError:
+                pass
+            try:
+                os.remove(repo_pkg + \
+                    etpConst['packagemtimefileext'])
+            except OSError:
+                # KeyError is for backward compatibility
+                pass
+
+        return successfully_removed
+
     def _run_repositories_post_branch_switch_hooks(self, old_branch, new_branch):
         """
         This method is called whenever branch is successfully switched by user.

services/client-updates-daemon

@@ -244,98 +244,23 @@ class UpdatesDaemon(dbus.service.Object):
     def cleanup_entropy_cache(self):
 
         with self.__is_working_mutex:
-            entropy = Entropy()
-            acquired = self.__acquire_entropy_locks(entropy)
-            if not acquired:
-                entropy.shutdown()
-                return True # respawn later
 
+            entropy = None
             try:
 
-                sys_set = SysSet()
-                sys_set_plg_id = \
-                    etpConst['system_settings_plugins_ids']['client_plugin']
-                client_settings = sys_set[sys_set_plg_id]
-                misc_settings = client_settings['misc']
-                autoprune_days = misc_settings.get('autoprune_days', None)
-                if autoprune_days is None:
-                    # sorry, feature disabled or not available
-                    # (upgrade entropy?)
-                    return False
-
-                def filter_expired_pkg(pkg_path):
-
-                    pkg_path = os.path.abspath(pkg_path)
-                    # security check
-                    if not pkg_path.startswith(etpConst['entropyworkdir']):
-                        return False
-                    if not pkg_path.endswith(etpConst['packagesext']):
-                        return False
-
-                    if not os.path.isfile(pkg_path):
-                        return False
-                    if not os.access(pkg_path, os.R_OK | os.W_OK):
-                        return False
-                    try:
-                        mtime = os.path.getmtime(pkg_path)
-                    except (OSError, IOError):
-                        return False
-                    if (mtime + (autoprune_days*24*3600)) > time.time():
-                        return False
-                    return True
-
-                # only check against available repositories
-                # skip disabled and corrupted
-                for repo_id in sys_set['repositories']['available']:
-
-                    try:
-                        repo_db = entropy.open_repository(repo_id)
-                    except RepositoryError as err:
-                        write_output(
-                            "cleanup_entropy_cache: open_repository() error: %s" % (err,))
-                        continue
-                    except Exception as err:
-                        write_output(
-                            "cleanup_entropy_cache: open_repository() general error: %s" % (err,))
-                        continue
-
-                    repo_pkgs = set(repo_db.listAllDownloads(do_sort = False,
-                        full_path = True))
-                    repo_pkgs = [os.path.join(etpConst['entropyworkdir'], x) \
-                        for x in repo_pkgs]
-                    repo_pkgs = sorted(filter(filter_expired_pkg, repo_pkgs))
-                    if not repo_pkgs:
-                        if DAEMON_DEBUG:
-                            write_output(
-                                "cleanup_entropy_cache: removing from %s: nothing" % (
-                                    repo_id,))
-                        continue
-
-                    if DAEMON_DEBUG:
-                        write_output(
-                            "cleanup_entropy_cache: removing from %s: %s" % (
-                                repo_id, ', '.join(repo_pkgs),))
-                    for repo_pkg in repo_pkgs:
-                        try:
-                            os.remove(repo_pkg)
-                        except OSError:
-                            pass
-                        try:
-                            os.remove(repo_pkg + etpConst['packagesmd5fileext'])
-                        except OSError:
-                            pass
-                        try:
-                            os.remove(repo_pkg + \
-                                etpConst['packagemtimefileext'])
-                        except (OSError, KeyError,):
-                            # KeyError is for backward compatibility
-                            continue
+                entropy = Entropy()
+                acquired = self.__acquire_entropy_locks(entropy)
+                if not acquired:
+                    return True # respawn later
 
+                if hasattr(entropy, 'clean_downloaded_packages'):
+                    entropy.clean_downloaded_packages()
                 return False
 
             finally:
-                self.__release_entropy_locks(entropy)
-                entropy.shutdown()
+                if entropy is not None:
+                    self.__release_entropy_locks(entropy)
+                    entropy.shutdown()
 
     def check_system_changes(self):
         if self.__trigger_oncall_updater: