sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[entropy.server] make possible to properly handle expiration of extra package files. Enforce file name constraints on them.

Browse Source
This commit is contained in:
Fabio Erculiani
2011-08-29 06:35:40 +02:00
parent 294b1a25e4
commit d9dfe06b0a
5 changed files with 54 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/entropy/const.py
+3 -1
View File
@@ -431,7 +431,9 @@ def const_default_settings(rootdir):
'userpackagesetsid': "__user__",
'cachedumpext': ".dmp",
'packagesext': ".tbz2",
'packagesdebugext': ".debug.tar.bz2",
# extra download package file extension (mandatory)
'packagesextraext': ".tar.bz2",
'packagesdebugext': ".debug.tar.bz2", # .tar.bz2
'packagesext_webinstall': ".etp",
# entropy package files binary delta extension
'packagesdeltaext': ".edelta",
libraries/entropy/db/skel.py
+4
View File
@@ -2258,6 +2258,8 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore):
Retrieve a list of extra package file URLs for package identifier.
These URLs usually contain extra files that can be optionally installed
by Entropy Client, for example: debug files.
All the extra download file names must end with etpConst['packagesextraext']
extension.
@param package_id: package indentifier
@type package_id: int
@@ -3508,6 +3510,8 @@ class EntropyRepositoryBase(TextInterface, EntropyRepositoryPluginStore):
def listAllExtraDownloads(self, do_sort = True):
"""
List all package extra download URLs stored in repository.
All the extra download file names must end with etpConst['packagesextraext']
extension.
@keyword do_sort: sort by name
@type do_sort: bool
libraries/entropy/server/interfaces/main.py
+2 -2
View File
@@ -1187,14 +1187,14 @@ class Server(Client):
return db_download_uri.replace("/%s/" % (cur_branch,),
"/%s/" % (new_branch,))
def _get_basedir_pkg_listing(self, base_dir, branch = None):
def _get_basedir_pkg_listing(self, base_dir, extension, branch = None):
pkgs_dir_types = set(self._get_pkg_dir_names())
basedir_raw_content = []
entropy.tools.recursive_directory_relative_listing(
basedir_raw_content, base_dir)
pkg_ext = etpConst['packagesext']
pkg_ext = extension
pkg_list = [x for x in basedir_raw_content if x.endswith(pkg_ext)]
pkg_list = [x for x in pkg_list if \
x.split(os.path.sep)[0] in pkgs_dir_types]
libraries/entropy/server/interfaces/mirrors.py
+42 -33
View File
@@ -899,7 +899,7 @@ class Server(object):
branch = self._settings['repositories']['branch']
upload_pkgs = self._entropy._get_basedir_pkg_listing(upload_dir,
branch = branch)
etpConst['packagesext'], branch = branch)
pkg_ext = etpConst['packagesext']
for package in upload_pkgs:
@@ -921,7 +921,7 @@ class Server(object):
branch = self._settings['repositories']['branch']
pkg_files = self._entropy._get_basedir_pkg_listing(base_dir,
branch = branch)
etpConst['packagesext'], branch = branch)
pkg_ext = etpConst['packagesext']
for package in pkg_files:
@@ -1938,8 +1938,8 @@ class Server(object):
upload_dir = self._entropy._get_local_upload_directory(repository_id)
basedir_list = []
entropy.tools.recursive_directory_relative_listing(
basedir_list, base_dir)
entropy.tools.recursive_directory_relative_listing(basedir_list,
upload_dir)
for pkg_rel in basedir_list:
@@ -1992,23 +1992,24 @@ class Server(object):
def _collect_expiring_packages(self, repository_id, branch):
dbconn = self._entropy.open_server_repository(repository_id,
just_reading = True)
dbconn = self._entropy.open_repository(repository_id)
database_bins = set(dbconn.listAllDownloads(do_sort = False,
full_path = True))
extra_database_bins = set(dbconn.listAllExtraDownloads(do_sort = False))
repo_basedir = self._entropy._get_local_repository_base_directory(
repository_id)
repo_bins = self._entropy._get_basedir_pkg_listing(repo_basedir,
branch = branch)
repo_bins = set(self._entropy._get_basedir_pkg_listing(repo_basedir,
etpConst['packagesext'], branch = branch))
extra_repo_bins = set(self._entropy._get_basedir_pkg_listing(
repo_basedir, etpConst['packagesextraext'], branch = branch))
# convert to set, so that we can do fast thingszzsd
repo_bins = set(repo_bins)
repo_bins -= database_bins
return repo_bins
extra_repo_bins -= extra_database_bins
return repo_bins, extra_repo_bins
def tidy_mirrors(self, repository_id, ask = True, pretend = False,
expiration_days = None):
@@ -2067,8 +2068,8 @@ class Server(object):
)
# collect removed packages
expiring_packages = self._collect_expiring_packages(repository_id,
branch)
expiring_packages, extra_expiring_packages = \
self._collect_expiring_packages(repository_id, branch)
if expiring_packages:
# filter expired packages used by other branches
@@ -2080,13 +2081,31 @@ class Server(object):
repository_id, etpConst['etpdatabasepkglist'],
excluded_branches = [branch])
# format data
for key, val in branch_pkglist_data.items():
for key, val in list(branch_pkglist_data.items()):
branch_pkglist_data[key] = val.split("\n")
for other_branch in branch_pkglist_data:
branch_pkglist = set(branch_pkglist_data[other_branch])
expiring_packages -= branch_pkglist
if extra_expiring_packages:
# filter expired packages used by other branches
# this is done for the sake of consistency
# --- read packages.db.extra_pkglist, make sure your repository
# has been ported to latest Entropy
branch_extra_pkglist_data = self._read_remote_file_in_branches(
repository_id, etpConst['etpdatabaseextrapkglist'],
excluded_branches = [branch])
# format data
for key, val in list(branch_extra_pkglist_data.items()):
branch_extra_pkglist_data[key] = val.split("\n")
for other_branch in branch_extra_pkglist_data:
branch_pkglist = set(branch_extra_pkglist_data[other_branch])
extra_expiring_packages -= branch_pkglist
removal = []
for package_rel in expiring_packages:
expired = self._is_package_expired(repository_id, package_rel,
@@ -2097,6 +2116,15 @@ class Server(object):
self._create_expiration_file(repository_id, package_rel,
gentle = True)
for extra_package_rel in extra_expiring_packages:
expired = self._is_package_expired(repository_id, extra_package_rel,
expiration_days)
if expired:
removal.append(extra_package_rel)
else:
self._create_expiration_file(repository_id, extra_package_rel,
gentle = True)
if not removal:
self._entropy.output(
"[%s] %s" % (
@@ -2143,32 +2171,13 @@ class Server(object):
removal_map = {}
dbconn = self._entropy.open_server_repository(repository_id,
just_reading = True)
extra_removal = []
for package_rel in removal:
rel_path = self._entropy.complete_remote_package_relative_path(
package_rel, repository_id)
rel_dir = os.path.dirname(rel_path)
obj = removal_map.setdefault(rel_dir, [])
base_pkg = os.path.basename(package_rel)
obj.append(base_pkg)
package_id = dbconn.getPackageIdFromDownload(package_rel)
if package_id == -1:
# wtf?
continue
extra_downloads = dbconn.retrieveExtraDownload(package_id)
for extra_download in extra_downloads:
extra_rel = extra_download['download']
extra_removal.append(extra_rel)
rel_path = self._entropy.complete_remote_package_relative_path(
extra_rel, repository_id)
rel_dir = os.path.dirname(rel_path)
obj = removal_map.setdefault(rel_dir, [])
base_pkg = os.path.basename(extra_rel)
obj.append(base_pkg)
removal.extend(extra_removal)
for uri in self._entropy.remote_packages_mirrors(repository_id):
libraries/entropy/spm/plugins/skel.py
+3 -1
View File
@@ -370,7 +370,9 @@ class SpmPlugin(Singleton):
the return list.
@type builtin_debug: bool
@return: list of package file paths, the first is the main one, the
second in list, if available, is the debug package.
second in list, if available, is the debug package. All these
extra package files must end with etpConst['packagesextraext']
extension.
@rtype: list
@raise entropy.exception.SPMError: if unable to satisfy the request
"""