From d9dfe06b0aa29a30bfaac8c34ec88c7574eae185 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Mon, 29 Aug 2011 06:35:40 +0200 Subject: [PATCH] [entropy.server] make possible to properly handle expiration of extra package files. Enforce file name constraints on them. --- libraries/entropy/const.py | 4 +- libraries/entropy/db/skel.py | 4 + libraries/entropy/server/interfaces/main.py | 4 +- .../entropy/server/interfaces/mirrors.py | 75 +++++++++++-------- libraries/entropy/spm/plugins/skel.py | 4 +- 5 files changed, 54 insertions(+), 37 deletions(-) diff --git a/libraries/entropy/const.py b/libraries/entropy/const.py index 40cf85772..c31d04512 100644 --- a/libraries/entropy/const.py +++ b/libraries/entropy/const.py @@ -431,7 +431,9 @@ def const_default_settings(rootdir): 'userpackagesetsid': "__user__", 'cachedumpext': ".dmp", 'packagesext': ".tbz2", - 'packagesdebugext': ".debug.tar.bz2", + # extra download package file extension (mandatory) + 'packagesextraext': ".tar.bz2", + 'packagesdebugext': ".debug.tar.bz2", # .tar.bz2 'packagesext_webinstall': ".etp", # entropy package files binary delta extension 'packagesdeltaext': ".edelta", diff --git a/libraries/entropy/db/skel.py b/libraries/entropy/db/skel.py index 80cc050bc..2e27b06cd 100644 --- a/libraries/entropy/db/skel.py +++ b/libraries/entropy/db/skel.py @@ -2258,6 +2258,8 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore): Retrieve a list of extra package file URLs for package identifier. These URLs usually contain extra files that can be optionally installed by Entropy Client, for example: debug files. + All the extra download file names must end with etpConst['packagesextraext'] + extension. @param package_id: package indentifier @type package_id: int @@ -3508,6 +3510,8 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore): def listAllExtraDownloads(self, do_sort = True): """ List all package extra download URLs stored in repository. + All the extra download file names must end with etpConst['packagesextraext'] + extension. @keyword do_sort: sort by name @type do_sort: bool diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py index 09d01546c..42c79426d 100644 --- a/libraries/entropy/server/interfaces/main.py +++ b/libraries/entropy/server/interfaces/main.py @@ -1187,14 +1187,14 @@ class Server(Client): return db_download_uri.replace("/%s/" % (cur_branch,), "/%s/" % (new_branch,)) - def _get_basedir_pkg_listing(self, base_dir, branch = None): + def _get_basedir_pkg_listing(self, base_dir, extension, branch = None): pkgs_dir_types = set(self._get_pkg_dir_names()) basedir_raw_content = [] entropy.tools.recursive_directory_relative_listing( basedir_raw_content, base_dir) - pkg_ext = etpConst['packagesext'] + pkg_ext = extension pkg_list = [x for x in basedir_raw_content if x.endswith(pkg_ext)] pkg_list = [x for x in pkg_list if \ x.split(os.path.sep)[0] in pkgs_dir_types] diff --git a/libraries/entropy/server/interfaces/mirrors.py b/libraries/entropy/server/interfaces/mirrors.py index 1400f3b49..62d634f2b 100644 --- a/libraries/entropy/server/interfaces/mirrors.py +++ b/libraries/entropy/server/interfaces/mirrors.py @@ -899,7 +899,7 @@ class Server(object): branch = self._settings['repositories']['branch'] upload_pkgs = self._entropy._get_basedir_pkg_listing(upload_dir, - branch = branch) + etpConst['packagesext'], branch = branch) pkg_ext = etpConst['packagesext'] for package in upload_pkgs: @@ -921,7 +921,7 @@ class Server(object): branch = self._settings['repositories']['branch'] pkg_files = self._entropy._get_basedir_pkg_listing(base_dir, - branch = branch) + etpConst['packagesext'], branch = branch) pkg_ext = etpConst['packagesext'] for package in pkg_files: @@ -1938,8 +1938,8 @@ class Server(object): upload_dir = self._entropy._get_local_upload_directory(repository_id) basedir_list = [] - entropy.tools.recursive_directory_relative_listing( - basedir_list, base_dir) + entropy.tools.recursive_directory_relative_listing(basedir_list, + upload_dir) for pkg_rel in basedir_list: @@ -1992,23 +1992,24 @@ class Server(object): def _collect_expiring_packages(self, repository_id, branch): - dbconn = self._entropy.open_server_repository(repository_id, - just_reading = True) + dbconn = self._entropy.open_repository(repository_id) database_bins = set(dbconn.listAllDownloads(do_sort = False, full_path = True)) + extra_database_bins = set(dbconn.listAllExtraDownloads(do_sort = False)) repo_basedir = self._entropy._get_local_repository_base_directory( repository_id) - repo_bins = self._entropy._get_basedir_pkg_listing(repo_basedir, - branch = branch) + repo_bins = set(self._entropy._get_basedir_pkg_listing(repo_basedir, + etpConst['packagesext'], branch = branch)) + extra_repo_bins = set(self._entropy._get_basedir_pkg_listing( + repo_basedir, etpConst['packagesextraext'], branch = branch)) # convert to set, so that we can do fast thingszzsd - repo_bins = set(repo_bins) repo_bins -= database_bins - return repo_bins - + extra_repo_bins -= extra_database_bins + return repo_bins, extra_repo_bins def tidy_mirrors(self, repository_id, ask = True, pretend = False, expiration_days = None): @@ -2067,8 +2068,8 @@ class Server(object): ) # collect removed packages - expiring_packages = self._collect_expiring_packages(repository_id, - branch) + expiring_packages, extra_expiring_packages = \ + self._collect_expiring_packages(repository_id, branch) if expiring_packages: # filter expired packages used by other branches @@ -2080,13 +2081,31 @@ class Server(object): repository_id, etpConst['etpdatabasepkglist'], excluded_branches = [branch]) # format data - for key, val in branch_pkglist_data.items(): + for key, val in list(branch_pkglist_data.items()): branch_pkglist_data[key] = val.split("\n") for other_branch in branch_pkglist_data: branch_pkglist = set(branch_pkglist_data[other_branch]) expiring_packages -= branch_pkglist + if extra_expiring_packages: + + # filter expired packages used by other branches + # this is done for the sake of consistency + # --- read packages.db.extra_pkglist, make sure your repository + # has been ported to latest Entropy + + branch_extra_pkglist_data = self._read_remote_file_in_branches( + repository_id, etpConst['etpdatabaseextrapkglist'], + excluded_branches = [branch]) + # format data + for key, val in list(branch_extra_pkglist_data.items()): + branch_extra_pkglist_data[key] = val.split("\n") + + for other_branch in branch_extra_pkglist_data: + branch_pkglist = set(branch_extra_pkglist_data[other_branch]) + extra_expiring_packages -= branch_pkglist + removal = [] for package_rel in expiring_packages: expired = self._is_package_expired(repository_id, package_rel, @@ -2097,6 +2116,15 @@ class Server(object): self._create_expiration_file(repository_id, package_rel, gentle = True) + for extra_package_rel in extra_expiring_packages: + expired = self._is_package_expired(repository_id, extra_package_rel, + expiration_days) + if expired: + removal.append(extra_package_rel) + else: + self._create_expiration_file(repository_id, extra_package_rel, + gentle = True) + if not removal: self._entropy.output( "[%s] %s" % ( @@ -2143,32 +2171,13 @@ class Server(object): removal_map = {} dbconn = self._entropy.open_server_repository(repository_id, just_reading = True) - extra_removal = [] for package_rel in removal: - rel_path = self._entropy.complete_remote_package_relative_path( package_rel, repository_id) rel_dir = os.path.dirname(rel_path) obj = removal_map.setdefault(rel_dir, []) base_pkg = os.path.basename(package_rel) obj.append(base_pkg) - package_id = dbconn.getPackageIdFromDownload(package_rel) - if package_id == -1: - # wtf? - continue - - extra_downloads = dbconn.retrieveExtraDownload(package_id) - for extra_download in extra_downloads: - extra_rel = extra_download['download'] - extra_removal.append(extra_rel) - rel_path = self._entropy.complete_remote_package_relative_path( - extra_rel, repository_id) - rel_dir = os.path.dirname(rel_path) - obj = removal_map.setdefault(rel_dir, []) - base_pkg = os.path.basename(extra_rel) - obj.append(base_pkg) - - removal.extend(extra_removal) for uri in self._entropy.remote_packages_mirrors(repository_id): diff --git a/libraries/entropy/spm/plugins/skel.py b/libraries/entropy/spm/plugins/skel.py index a52827ac9..f9e395ec1 100644 --- a/libraries/entropy/spm/plugins/skel.py +++ b/libraries/entropy/spm/plugins/skel.py @@ -370,7 +370,9 @@ class SpmPlugin(Singleton): the return list. @type builtin_debug: bool @return: list of package file paths, the first is the main one, the - second in list, if available, is the debug package. + second in list, if available, is the debug package. All these + extra package files must end with etpConst['packagesextraext'] + extension. @rtype: list @raise entropy.exception.SPMError: if unable to satisfy the request """