sablink/entropy
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[entropy.server] add support for multiple packages directories (to host nonfree pkgs)

Browse Source
This commit is contained in:
Fabio Erculiani
2010-01-27 18:37:01 +01:00
parent 6c61cda7a5
commit a0245b84ca
7 changed files with 482 additions and 568 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/man/man1/reagent.1
View File

@@ -129,7 +129,7 @@
.\" ========================================================================
.\"
.IX Title "REAGENT 1"
.TH REAGENT 1 "2010-01-23" "perl v5.8.8" "Entropy"
.TH REAGENT 1 "2010-01-27" "perl v5.8.8" "Entropy"
.SH "NAME"
\&\fBreagent\fR \- Official Sabayon Linux Server\-Side Package Management tool
.SH "SYNOPSIS"
@@ -258,9 +258,6 @@ repository database functions
.IX Item "--initialize"
(re)initialize the current repository database
.RS 4
.IP "\fB\-\-empty\fR" 4
.IX Item "--empty"
do not refill database using packages on mirrors
.IP "\fB\-\-repo=<repo\fR>" 4
.IX Item "--repo=<repo>"
(re)create the database for the specified repository

4
docs/man/reagent.pod
View File

@@ -174,10 +174,6 @@ repository database functions
=over
=item B<--empty>
do not refill database using packages on mirrors
=item B<--repo=<repo>>
(re)create the database for the specified repository

449
libraries/entropy/server/interfaces/main.py
View File

@@ -620,59 +620,55 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin):
# expand paths
for repoid in data['repositories']:
data['repositories'][repoid]['repo_basedir'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid
)
data['repositories'][repoid]['packages_dir'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid,
etpConst['packagesrelativepath_basedir'],
etpSys['arch']
etpConst['currentarch']
)
data['repositories'][repoid]['packages_dir_nonfree'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid,
etpConst['packagesrelativepath_basedir_nonfree'],
etpSys['arch']
etpConst['currentarch']
)
data['repositories'][repoid]['store_dir'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid,
"store",
etpSys['arch']
etpConst['currentarch']
)
data['repositories'][repoid]['upload_dir'] = \
data['repositories'][repoid]['upload_basedir'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid,
"upload",
etpSys['arch']
"upload" # consider this a base dir
)
data['repositories'][repoid]['database_dir'] = \
os.path.join( etpConst['entropyworkdir'],
"server",
repoid,
"database",
etpSys['arch']
etpConst['currentarch']
)
data['repositories'][repoid]['packages_relative_path'] = \
data['repositories'][repoid]['remote_repo_basedir'] = \
os.path.join( sys_set['repositories']['product'],
repoid,
etpConst['packagesrelativepath_basedir'],
etpSys['arch']
)+"/"
data['repositories'][repoid]['packages_relative_path_nonfree'] = \
os.path.join( sys_set['repositories']['product'],
repoid,
etpConst['packagesrelativepath_basedir_nonfree'],
etpSys['arch']
)+"/"
repoid
)
data['repositories'][repoid]['database_relative_path'] = \
os.path.join( sys_set['repositories']['product'],
repoid,
"database",
etpSys['arch']
)+"/"
etpConst['currentarch']
)
# Support for shell variables
shell_repoid = os.getenv('ETP_REPO')
@@ -755,17 +751,28 @@ class ServerQAInterfacePlugin(QAInterfacePlugin):
class ServerSettingsMixin:
def _get_remote_packages_relative_path(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return srv_set['repositories'][repo]['packages_relative_path']
def _get_basedir_pkg_listing(self, base_dir, repo = None, branch = None):
def _get_remote_packages_nonfree_relative_path(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return srv_set['repositories'][repo]['packages_relative_path_nonfree']
pkgs_dir_types = self.Entropy._get_pkg_dir_names()
basedir_raw_content = []
entropy.tools.recursive_directory_relative_listing(
basedir_raw_content, base_dir)
pkg_ext = etpConst['packagesext']
pkg_list = [x for x in basedir_raw_content if \
x.split(os.path.sep)[0] in pkgs_dir_types and \
x.endswith(pkg_ext)]
if branch is not None:
branch_extractor = \
self.Entropy.Client.get_branch_from_download_relative_uri
pkg_list = [x for x in pkg_list if branch_extractor(x) == branch]
return pkg_list
def _get_pkg_dir_names(self):
return [etpConst['packagesrelativepath_basedir'],
etpConst['packagesrelativepath_basedir_nonfree']]
def _get_remote_database_relative_path(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
@@ -789,19 +796,13 @@ class ServerSettingsMixin:
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return srv_set['repositories'][repo]['upload_dir']
return srv_set['repositories'][repo]['upload_basedir']
def _get_local_packages_directory(self, repo = None):
def _get_local_repository_base_directory(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return srv_set['repositories'][repo]['packages_dir']
def _get_local_packages_nonfree_directory(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return srv_set['repositories'][repo]['packages_dir_nonfree']
return srv_set['repositories'][repo]['repo_basedir']
def _get_local_database_taint_file(self, repo = None, branch = None):
if repo is None:
@@ -1018,6 +1019,27 @@ class ServerSettingsMixin:
if os.path.isfile(lock_file):
os.remove(lock_file)
def complete_remote_package_relative_path(self, pkg_rel_url, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return os.path.join(
srv_set['repositories'][repo]['remote_repo_basedir'], pkg_rel_url)
def complete_local_upload_package_path(self, pkg_rel_url, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return os.path.join(srv_set['repositories'][repo]['upload_basedir'],
pkg_rel_url)
def complete_local_package_path(self, pkg_rel_url, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
repo = self.default_repository
return os.path.join(srv_set['repositories'][repo]['repo_basedir'],
pkg_rel_url)
def get_remote_mirrors(self, repo = None):
srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
if repo is None:
@@ -1168,8 +1190,7 @@ class ServerPackageDepsMixin:
class ServerPackagesHandlingMixin:
def initialize_server_repository(self, empty = True, repo = None,
warnings = True):
def initialize_server_repository(self, repo = None, show_warnings = True):
if repo is None:
repo = self.default_repository
@@ -1191,7 +1212,7 @@ class ServerPackagesHandlingMixin:
if os.path.isfile(self._get_local_database_file(repo)):
dbconn = self.open_server_repository(read_only = True,
no_upload = True, repo = repo, warnings = warnings)
no_upload = True, repo = repo, warnings = show_warnings)
try:
dbconn.validateDatabase()
@@ -1247,151 +1268,6 @@ class ServerPackagesHandlingMixin:
no_upload = True, repo = repo, is_new = True)
dbconn.initializeDatabase()
if not empty:
revisions_file = "/entropy-revisions-dump.txt"
# dump revisions - as a backup
if revisions_match:
self.output(
"%s: %s" % (
red(_("Dumping current revisions to file")),
darkgreen(revisions_file),
),
importance = 1,
type = "info",
header = darkgreen(" * ")
)
f_rev = open(revisions_file, "w")
f_rev.write(str(revisions_match))
f_rev.flush()
f_rev.close()
# dump treeupdates - as a backup
treeupdates_file = "/entropy-treeupdates-dump.txt"
if treeupdates_actions:
self.output(
"%s: %s" % (
# do not translate treeupdates
red(_("Dumping current 'treeupdates' actions to file")),
bold(treeupdates_file),
),
importance = 1,
type = "info",
header = darkgreen(" * ")
)
f_tree = open(treeupdates_file, "w")
f_tree.write(str(treeupdates_actions))
f_tree.flush()
f_tree.close()
rc_question = self.ask_question(
_("Would you like to sync packages first (important!) ?"))
if rc_question == _("Yes"):
self.Mirrors.sync_packages(repo = repo)
# fill tree updates actions
if treeupdates_actions:
dbconn.bumpTreeUpdatesActions(treeupdates_actions)
# now fill the database
pkg_branch_dir = os.path.join(
self._get_local_packages_directory(repo),
self.SystemSettings['repositories']['branch'])
pkglist = os.listdir(pkg_branch_dir)
# filter .md5 and .expired packages
pkg_ext_len = len(etpConst['packagesext'])
pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
etpConst['packagesext']) and not \
os.path.isfile(os.path.join(pkg_branch_dir,
x + etpConst['packagesexpirationfileext']))]
if pkglist:
self.output(
"%s '%s' %s %s" % (
red(_("Reinitializing Entropy database for branch")),
bold(self.SystemSettings['repositories']['branch']),
red(_("using Packages in the repository")),
red("..."),
),
importance = 1,
type = "info",
header = darkgreen(" * ")
)
counter = 0
maxcount = len(pkglist)
branch = self.SystemSettings['repositories']['branch']
for pkg in pkglist:
counter += 1
self.output(
"[repo:%s|%s] %s: %s" % (
darkgreen(repo),
brown(branch),
blue(_("analyzing")),
bold(pkg),
),
importance = 1,
type = "info",
header = " ",
back = True,
count = (counter, maxcount,)
)
doinject = False
if pkg in injected_packages:
doinject = True
pkg_path = os.path.join(self._get_local_packages_directory(repo),
branch, pkg)
mydata = self.Spm().extract_package_metadata(pkg_path)
self._pump_extracted_package_metadata(mydata, repo,
{'injected': doinject,})
# get previous revision
revision_avail = revisions_match.get(pkg)
add_revision = 0
if (revision_avail != None):
if branch == revision_avail[0]:
add_revision = revision_avail[1]
idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
revision = add_revision)
idpackages_added.add(idpackage)
self.output(
"[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
repo,
brown(branch),
darkgreen(str(counter)),
blue(str(maxcount)),
red(_("added package")),
darkgreen(pkg),
red(_("revision")),
brown(str(revision)),
),
importance = 1,
type = "info",
header = " ",
back = True
)
self.generate_reverse_dependencies_metadata(repo)
my_qa = self.QA()
if idpackages_added:
dbconn = self.open_server_repository(read_only = False,
no_upload = True, repo = repo)
my_qa.test_missing_dependencies(
idpackages_added, dbconn, ask = True,
repo = repo, self_check = True,
black_list = \
self._get_missing_dependencies_blacklist(repo = repo),
black_list_adder = \
self._add_missing_dependencies_blacklist_items
)
dbconn.commitChanges()
self.close_repositories()
@@ -1515,8 +1391,6 @@ class ServerPackagesHandlingMixin:
tmp_down_dir = tempfile.mkdtemp()
download_queue = {}
local_up_dir = self._get_local_upload_directory(repo)
local_basedir = os.path.join(local_up_dir, branch)
dbconn = self.open_server_repository(read_only = False,
no_upload = True, repo = repo)
@@ -1566,26 +1440,41 @@ class ServerPackagesHandlingMixin:
if rc_question == _("No"):
continue
remote_relative_path = self._get_remote_packages_relative_path(repo)
for uri in self.get_remote_mirrors(repo):
crippled_uri = EntropyTransceiver.get_uri_name(uri)
basedir = os.path.join(remote_relative_path, from_branch)
downloader_queue = [x[0] for x in download_queue[from_branch]]
downloader = self.Mirrors.TransceiverServerHandler(
self,
[uri],
downloader_queue,
critical_files = downloader_queue,
txc_basedir = basedir,
local_basedir = tmp_down_dir,
download = True,
repo = repo
)
queue_map = {}
errors, m_fine_uris, m_broken_uris = downloader.go()
for pkg_fp, idpackage in download_queue[from_branch]:
down_url = dbconn.retrieveDownloadURL(idpackage)
down_rel = self.complete_remote_package_relative_path(
down_url, repo = repo)
down_rel_dir = os.path.dirname(down_rel)
obj = queue_map.setdefault(down_rel_dir, [])
obj.append(pkg_fp)
errors = False
m_fine_uris = set()
m_broken_uris = set()
for down_rel_dir, downloader_queue in queue_map.items():
downloader = self.Mirrors.TransceiverServerHandler(
self,
[uri],
downloader_queue,
critical_files = downloader_queue,
txc_basedir = down_rel_dir,
local_basedir = tmp_down_dir,
download = True,
repo = repo
)
xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
if xerrors:
errors = True
m_fine_uris.update(xm_fine_uris)
m_broken_uris.update(xm_broken_uris)
if not errors:
for downloaded_path, idpackage in \
@@ -1710,9 +1599,17 @@ class ServerPackagesHandlingMixin:
back = True
)
# build new download url
download_url = dbconn.retrieveDownloadURL(idpackage)
download_url = \
self.Client.swap_branch_in_download_relative_uri(
branch, download_url)
# move files to upload
package_name = os.path.basename(package_path)
new_package_path = os.path.join(local_basedir, package_name)
new_package_path = self.complete_local_upload_package_path(
download_url, repo = repo)
self._ensure_dir_path(os.path.dirname(new_package_path))
try:
os.rename(package_path, new_package_path)
except OSError:
@@ -1722,10 +1619,6 @@ class ServerPackagesHandlingMixin:
entropy.tools.create_md5_file(new_package_path)
# update database
download_url = dbconn.retrieveDownloadURL(idpackage)
download_url = \
self.Client.swap_branch_in_download_relative_uri(
branch, download_url)
dbconn.setDownloadURL(idpackage, download_url)
dbconn.switchBranch(idpackage, branch)
dbconn.commitChanges()
@@ -1903,19 +1796,20 @@ class ServerPackagesHandlingMixin:
return switched
for idpackage, repo in my_matches:
dbconn = self.open_server_repository(read_only = False,
no_upload = True, repo = repo)
match_branch = dbconn.retrieveBranch(idpackage)
match_atom = dbconn.retrieveAtom(idpackage)
package_filename = os.path.basename(
dbconn.retrieveDownloadURL(idpackage))
package_rel_path = dbconn.retrieveDownloadURL(idpackage)
self.output(
"[%s=>%s|%s] %s: %s" % (
darkgreen(repo),
darkred(to_repo),
brown(branch),
blue(_("switching")),
darkgreen(match_atom),
darkgreen(repo),
darkred(to_repo),
brown(branch),
blue(_("switching")),
darkgreen(match_atom),
),
importance = 0,
type = "info",
@@ -1923,11 +1817,11 @@ class ServerPackagesHandlingMixin:
back = True
)
# move binary file
from_file = os.path.join(self._get_local_packages_directory(repo),
match_branch, package_filename)
from_file = self.complete_local_package_path(package_rel_path,
repo = repo)
if not os.path.isfile(from_file):
from_file = os.path.join(self._get_local_upload_directory(repo),
match_branch, package_filename)
from_file = self.complete_local_upload_package_path(
package_rel_path, repo = repo)
if not os.path.isfile(from_file):
self.output(
"[%s=>%s|%s] %s: %s -> %s" % (
@@ -1944,20 +1838,25 @@ class ServerPackagesHandlingMixin:
)
continue
to_file = self.complete_local_upload_package_path(package_rel_path,
repo = to_repo)
if new_tag != None:
match_category = dbconn.retrieveCategory(idpackage)
match_name = dbconn.retrieveName(idpackage)
match_version = dbconn.retrieveVersion(idpackage)
tagged_package_filename = \
entropy.tools.create_package_filename(
match_category, match_name, match_version, new_tag)
to_file = os.path.join(self._get_local_upload_directory(to_repo),
match_branch, tagged_package_filename)
else:
to_file = os.path.join(self._get_local_upload_directory(to_repo),
match_branch, package_filename)
if not os.path.isdir(os.path.dirname(to_file)):
os.makedirs(os.path.dirname(to_file))
to_file = self.complete_local_upload_package_path(
package_rel_path, repo = to_repo)
# directly move to correct place, tag changed, so file name
to_file = os.path.join(os.path.dirname(to_file),
tagged_package_filename)
self._ensure_dir_path(os.path.dirname(to_file))
copy_data = [
(from_file, to_file,),
@@ -1969,17 +1868,17 @@ class ServerPackagesHandlingMixin:
for from_item, to_item in copy_data:
self.output(
"[%s=>%s|%s] %s: %s" % (
darkgreen(repo),
darkred(to_repo),
brown(branch),
blue(_("moving file")),
darkgreen(os.path.basename(from_item)),
),
importance = 0,
type = "info",
header = red(" @@ "),
back = True
"[%s=>%s|%s] %s: %s" % (
darkgreen(repo),
darkred(to_repo),
brown(branch),
blue(_("moving file")),
darkgreen(os.path.basename(from_item)),
),
importance = 0,
type = "info",
header = red(" @@ "),
back = True
)
if os.path.isfile(from_item):
shutil.copy2(from_item, to_item)
@@ -2765,7 +2664,6 @@ class ServerPackagesHandlingMixin:
uploaddir_path = self._get_upload_package_path(repo, dbconn,
idpackage)
pkg_path = dbconn.retrieveDownloadURL(idpackage)
pkg_rel_path = self._get_common_pkg_relative_path(pkg_path)
pkgatom = dbconn.retrieveAtom(idpackage)
if os.path.isfile(bindir_path):
@@ -2773,7 +2671,7 @@ class ServerPackagesHandlingMixin:
"[%s] %s :: %s" % (
darkgreen(_("available")),
blue(pkgatom),
darkgreen(pkg_rel_path),
darkgreen(pkg_path),
),
importance = 0,
type = "info",
@@ -2785,7 +2683,7 @@ class ServerPackagesHandlingMixin:
"[%s] %s :: %s" % (
darkred(_("upload/ignored")),
blue(pkgatom),
darkgreen(pkg_rel_path),
darkgreen(pkg_path),
),
importance = 0,
type = "info",
@@ -2796,7 +2694,7 @@ class ServerPackagesHandlingMixin:
"[%s] %s :: %s" % (
brown(_("download")),
blue(pkgatom),
darkgreen(pkg_rel_path),
darkgreen(pkg_path),
),
importance = 0,
type = "info",
@@ -3414,8 +3312,8 @@ class ServerRepositoryMixin:
dbfile = self._get_local_database_file(repoid)
if os.path.isfile(dbfile):
shutil.move(dbfile, dbfile+".backup")
self.initialize_server_repository(empty = True,
repo = repoid, warnings = False)
self.initialize_server_repository(repo = repoid,
show_warnings = False)
def _save_default_repository(self, repoid):
@@ -3694,8 +3592,7 @@ class ServerRepositoryMixin:
return cached
local_dbfile_dir = os.path.dirname(local_dbfile)
if not os.path.isdir(local_dbfile_dir):
os.makedirs(local_dbfile_dir)
self._ensure_dir_path(local_dbfile_dir)
if (not read_only) and (lock_remote) and \
(repo not in self._sync_lock_cache):
@@ -3830,11 +3727,6 @@ class ServerRepositoryMixin:
if repo is None:
repo = self.default_repository
upload_dir = os.path.join(self._get_local_upload_directory(repo),
self.SystemSettings['repositories']['branch'])
if not os.path.isdir(upload_dir):
os.makedirs(upload_dir)
dbconn = self.open_server_repository(read_only = False,
no_upload = True, repo = repo)
self.output(
@@ -3943,8 +3835,11 @@ class ServerRepositoryMixin:
download_url = self._setup_repository_package_filename(idpackage,
repo = repo)
downloadfile = os.path.basename(download_url)
destination_path = os.path.join(upload_dir, downloadfile)
destination_path = self.complete_local_upload_package_path(
download_url, repo = repo)
destination_dir = os.path.dirname(destination_path)
self._ensure_dir_path(destination_dir)
try:
os.rename(package_file, destination_path)
except OSError:
@@ -4103,14 +3998,18 @@ class ServerRepositoryMixin:
class ServerMiscMixin:
def _ensure_paths(self, repo):
upload_dir = os.path.join(self._get_local_upload_directory(repo),
self.SystemSettings['repositories']['branch'])
upload_dir = self._get_local_upload_directory(repo)
db_dir = self._get_local_database_dir(repo)
for mydir in [upload_dir, db_dir]:
if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
os.makedirs(mydir)
os.makedirs(mydir, 0o775)
const_setup_perms(mydir, etpConst['entropygid'])
def _ensure_dir_path(self, dir_path):
if not os.path.isdir(dir_path):
os.makedirs(dir_path, 0o775)
const_setup_perms(dir_path, etpConst['entropygid'])
def _setup_services(self):
self._setup_entropy_settings()
cs_name = 'Client'
@@ -4306,34 +4205,24 @@ class ServerMiscMixin:
def get_branch_from_download_relative_uri(self, mypath):
return self.Client.get_branch_from_download_relative_uri(mypath)
def _get_common_pkg_relative_path(self, pkg_path, branch = None):
my_path = '/'.join(pkg_path.split("/")[2:])
if branch is None:
return my_path
head, tail = os.path.split(my_path)
return os.path.join(branch, tail)
def _get_package_path(self, repo, dbconn, idpackage):
"""
Given EntropyRepository instance and package identifier, return local
path of package. This method does not check for path validity though.
"""
pkg_rel_url = dbconn.retrieveDownloadURL(idpackage)
complete_path = self.complete_local_package_path(pkg_rel_url,
repo = repo)
return complete_path
def _get_package_path(self, repo, dbconn, idpackage, branch = None):
def _get_upload_package_path(self, repo, dbconn, idpackage):
"""
Given EntropyRepository instance and package identifier, return local
path of package. This method does not check for path validity though.
"""
pkg_path = dbconn.retrieveDownloadURL(idpackage)
pkg_rel_path = self._get_common_pkg_relative_path(pkg_path,
branch = branch)
return os.path.join(self._get_local_packages_directory(repo),
pkg_rel_path)
def _get_upload_package_path(self, repo, dbconn, idpackage, branch = None):
"""
Given EntropyRepository instance and package identifier, return local
path of package. This method does not check for path validity though.
"""
pkg_path = dbconn.retrieveDownloadURL(idpackage)
pkg_rel_path = self._get_common_pkg_relative_path(pkg_path,
branch = branch)
return os.path.join(self._get_local_upload_directory(repo),
pkg_rel_path)
return os.path.join(self._get_local_upload_directory(repo = repo),
pkg_path)
def scan_package_changes(self):

581
libraries/entropy/server/interfaces/mirrors.py
View File

@@ -18,7 +18,7 @@ from entropy.exceptions import OnlineMirrorError, ConnectionError, \
EntropyPackageException, TransceiverError
from entropy.output import red, darkgreen, bold, brown, blue, darkred, \
darkblue, purple
from entropy.const import etpConst, etpSys
from entropy.const import etpConst, const_setup_perms
from entropy.i18n import _
from entropy.misc import RSS
from entropy.server.interfaces.rss import ServerRssMetadata
@@ -634,7 +634,6 @@ class Server(ServerNoticeBoardMixin):
if repo is None:
repo = self.Entropy.default_repository
pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:])
pkgfile = os.path.basename(pkg_relative_path)
crippled_uri = EntropyTransceiver.get_uri_name(uri)
@@ -651,7 +650,7 @@ class Server(ServerNoticeBoardMixin):
darkgreen(crippled_uri),
brown(str(tries)),
blue(_("connecting to download package")),
darkgreen(pkg_to_join_path),
darkgreen(pkg_relative_path),
),
importance = 1,
type = "info",
@@ -659,12 +658,12 @@ class Server(ServerNoticeBoardMixin):
back = True
)
remote_path = os.path.join(
self.Entropy._get_remote_packages_relative_path(repo),
pkg_to_join_path)
download_path = os.path.join(
self.Entropy._get_local_packages_directory(repo),
pkg_to_join_path)
remote_path = \
self.Entropy.complete_remote_package_relative_path(
pkg_relative_path, repo = repo)
download_path = self.Entropy.complete_local_package_path(
pkg_relative_path, repo = repo)
download_dir = os.path.dirname(download_path)
self.Entropy.output(
@@ -692,7 +691,7 @@ class Server(ServerNoticeBoardMixin):
darkgreen(crippled_uri),
brown(str(tries)),
blue(_("package")),
darkgreen(pkg_to_join_path),
darkgreen(pkg_relative_path),
blue(_("does not exist")),
),
importance = 1,
@@ -2184,20 +2183,22 @@ class Server(ServerNoticeBoardMixin):
self.lock_mirrors(False, repo = repo)
return 0, set(), set()
def _calculate_local_upload_files(self, branch, repo = None):
def _calculate_local_upload_files(self, repo = None):
upload_files = 0
upload_packages = set()
upload_dir = os.path.join(self.Entropy._get_local_upload_directory(repo),
branch)
upload_dir = self.Entropy._get_local_upload_directory(repo = repo)
# check if it exists
if not os.path.isdir(upload_dir):
return upload_files, upload_packages
branch = self.SystemSettings['repositories']['branch']
upload_pkgs = self.Entropy._get_basedir_pkg_listing(upload_dir,
repo = repo, branch = branch)
pkg_ext = etpConst['packagesext']
pkg_md5_ext = etpConst['packagesmd5fileext']
for package in os.listdir(upload_dir):
for package in upload_pkgs:
if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext):
upload_packages.add(package)
if package.endswith(pkg_ext):
@@ -2205,18 +2206,23 @@ class Server(ServerNoticeBoardMixin):
return upload_files, upload_packages
def _calculate_local_package_files(self, branch, repo = None):
def _calculate_local_package_files(self, repo = None):
local_files = 0
local_packages = set()
packages_dir = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch)
base_dir = self.Entropy._get_local_repository_base_directory(
repo = repo)
if not os.path.isdir(packages_dir):
os.makedirs(packages_dir)
# check if it exists
if not os.path.isdir(base_dir):
return local_files, local_packages
branch = self.SystemSettings['repositories']['branch']
pkg_files = self.Entropy._get_basedir_pkg_listing(base_dir,
repo = repo, branch = branch)
pkg_ext = etpConst['packagesext']
pkg_md5_ext = etpConst['packagesmd5fileext']
for package in os.listdir(packages_dir):
for package in pkg_files:
if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext):
local_packages.add(package)
if package.endswith(pkg_ext):
@@ -2257,12 +2263,13 @@ class Server(ServerNoticeBoardMixin):
header = red(" @@ ")
)
def _show_sync_queues(self, upload, download, removal, copy, metainfo,
branch):
def _show_sync_queues(self, upload, download, removal, copy, metainfo):
branch = self.SystemSettings['repositories']['branch']
# show stats
for package, size in upload:
package = darkgreen(os.path.basename(package))
for package, rel_pkg, size in upload:
package = darkgreen(rel_pkg)
size = blue(entropy.tools.bytes_into_human(size))
self.Entropy.output(
"[branch:%s|%s] %s [%s]" % (
@@ -2275,8 +2282,8 @@ class Server(ServerNoticeBoardMixin):
type = "info",
header = red(" # ")
)
for package, size in download:
package = darkred(os.path.basename(package))
for package, rel_pkg, size in download:
package = darkred(rel_pkg)
size = blue(entropy.tools.bytes_into_human(size))
self.Entropy.output(
"[branch:%s|%s] %s [%s]" % (
@@ -2289,8 +2296,8 @@ class Server(ServerNoticeBoardMixin):
type = "info",
header = red(" # ")
)
for package, size in copy:
package = darkblue(os.path.basename(package))
for package, rel_pkg, size in copy:
package = darkblue(rel_pkg)
size = blue(entropy.tools.bytes_into_human(size))
self.Entropy.output(
"[branch:%s|%s] %s [%s]" % (
@@ -2303,8 +2310,8 @@ class Server(ServerNoticeBoardMixin):
type = "info",
header = red(" # ")
)
for package, size in removal:
package = brown(os.path.basename(package))
for package, rel_pkg, size in removal:
package = brown(rel_pkg)
size = blue(entropy.tools.bytes_into_human(size))
self.Entropy.output(
"[branch:%s|%s] %s [%s]" % (
@@ -2377,40 +2384,50 @@ class Server(ServerNoticeBoardMixin):
header = blue(" @@ ")
)
def _calculate_remote_package_files(self, uri, branch, txc_handler,
repo = None):
remote_dir = os.path.join(
self.Entropy._get_remote_packages_relative_path(repo), branch)
# create path to lock file if it doesn't exist
if not txc_handler.is_dir(remote_dir):
txc_handler.makedirs(remote_dir)
remote_packages_info = txc_handler.list_content_metadata(remote_dir)
remote_packages = [x[0] for x in remote_packages_info]
def _calculate_remote_package_files(self, uri, txc_handler, repo = None):
remote_files = 0
for pkg in remote_packages:
if pkg.endswith(etpConst['packagesext']):
remote_files += 1
remote_packages_data = {}
for pkg in remote_packages_info:
remote_packages_data[pkg[0]] = int(pkg[1])
remote_packages = []
branch = self.SystemSettings['repositories']['branch']
pkgs_dir_types = self.Entropy._get_pkg_dir_names()
for pkg_dir_type in pkgs_dir_types:
remote_dir = self.Entropy.complete_remote_package_relative_path(
pkg_dir_type, repo = repo)
remote_dir = os.path.join(remote_dir, etpConst['currentarch'],
branch)
only_dir = self.Entropy.complete_remote_package_relative_path("",
repo = repo)
db_url_dir = remote_dir[len(only_dir):]
# create path to lock file if it doesn't exist
if not txc_handler.is_dir(remote_dir):
txc_handler.makedirs(remote_dir)
remote_packages_info = txc_handler.list_content_metadata(remote_dir)
remote_packages += [os.path.join(db_url_dir, x[0]) for x \
in remote_packages_info]
for pkg in remote_packages:
if pkg.endswith(etpConst['packagesext']):
remote_files += 1
my_remote_pkg_data = dict((os.path.join(db_url_dir, x[0]),
int(x[1])) for x in remote_packages_info)
remote_packages_data.update(my_remote_pkg_data)
return remote_files, remote_packages, remote_packages_data
def calculate_packages_to_sync(self, uri, branch, repo = None):
def calculate_packages_to_sync(self, uri, repo = None):
if repo is None:
repo = self.Entropy.default_repository
crippled_uri = EntropyTransceiver.get_uri_name(uri)
upload_files, upload_packages = self._calculate_local_upload_files(
branch, repo)
local_files, local_packages = self._calculate_local_package_files(
branch, repo)
upload_files, upload_packages = self._calculate_local_upload_files(repo)
local_files, local_packages = self._calculate_local_package_files(repo)
self._show_local_sync_stats(upload_files, local_files)
self.Entropy.output(
@@ -2423,8 +2440,7 @@ class Server(ServerNoticeBoardMixin):
txc = self.Entropy.Transceiver(uri)
with txc as handler:
remote_files, remote_packages, remote_packages_data = \
self._calculate_remote_package_files(uri, branch, handler,
repo = repo)
self._calculate_remote_package_files(uri, handler, repo = repo)
self.Entropy.output(
"%s:\t\t\t%s %s" % (
@@ -2449,35 +2465,37 @@ class Server(ServerNoticeBoardMixin):
upload_queue, download_queue, removal_queue, fine_queue = \
self._calculate_sync_queues(upload_packages, local_packages,
remote_packages, remote_packages_data, branch, repo)
remote_packages, remote_packages_data, repo)
return upload_queue, download_queue, removal_queue, fine_queue, \
remote_packages_data
def _calculate_sync_queues(self, upload_packages, local_packages,
remote_packages, remote_packages_data, branch, repo = None):
remote_packages, remote_packages_data, repo = None):
upload_queue = set()
download_queue = set()
removal_queue = set()
fine_queue = set()
branch = self.SystemSettings['repositories']['branch']
for local_package in upload_packages:
if local_package in remote_packages:
local_filepath = os.path.join(
self.Entropy._get_local_upload_directory(repo), branch,
local_package)
local_filepath = \
self.Entropy.complete_local_upload_package_path(
local_package, repo = repo)
local_size = entropy.tools.get_file_size(local_filepath)
remote_size = remote_packages_data.get(local_package)
if remote_size is None:
remote_size = 0
if (local_size != remote_size):
if local_size != remote_size:
# size does not match, adding to the upload queue
upload_queue.add(local_package)
else:
# just move from upload to packages
fine_queue.add(local_package)
else:
# always force upload of packages in uploaddir
upload_queue.add(local_package)
@@ -2486,9 +2504,8 @@ class Server(ServerNoticeBoardMixin):
# we have to upload it we have local_packages and remote_packages
for local_package in local_packages:
if local_package in remote_packages:
local_filepath = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
local_package)
local_filepath = self.Entropy.complete_local_package_path(
local_package, repo = repo)
local_size = entropy.tools.get_file_size(local_filepath)
remote_size = remote_packages_data.get(local_package)
if remote_size is None:
@@ -2505,9 +2522,8 @@ class Server(ServerNoticeBoardMixin):
# Fill download_queue and removal_queue
for remote_package in remote_packages:
if remote_package in local_packages:
local_filepath = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
remote_package)
local_filepath = self.Entropy.complete_local_package_path(
remote_package, repo = repo)
local_size = entropy.tools.get_file_size(local_filepath)
remote_size = remote_packages_data.get(remote_package)
if remote_size is None:
@@ -2561,9 +2577,8 @@ class Server(ServerNoticeBoardMixin):
return upload_queue, download_queue, removal_queue, fine_queue
def _expand_queues(self, upload_queue, download_queue, removal_queue,
remote_packages_data, branch, repo):
remote_packages_data, repo):
metainfo = {
'removal': 0,
@@ -2578,59 +2593,62 @@ class Server(ServerNoticeBoardMixin):
for item in removal_queue:
if not item.endswith(etpConst['packagesext']):
continue
local_filepath = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch, item)
local_filepath = self.Entropy.complete_local_package_path(
item, repo = repo)
size = entropy.tools.get_file_size(local_filepath)
metainfo['removal'] += size
removal.append((local_filepath, size))
removal.append((local_filepath, item, size))
for item in download_queue:
if not item.endswith(etpConst['packagesext']):
continue
local_filepath = os.path.join(
self.Entropy._get_local_upload_directory(repo), branch, item)
local_filepath = self.Entropy.complete_local_upload_package_path(
item, repo = repo)
if not os.path.isfile(local_filepath):
size = remote_packages_data.get(item)
if size is None:
size = 0
size = int(size)
metainfo['removal'] += size
download.append((local_filepath, size))
download.append((local_filepath, item, size))
else:
size = entropy.tools.get_file_size(local_filepath)
do_copy.append((local_filepath, size))
do_copy.append((local_filepath, item, size))
for item in upload_queue:
if not item.endswith(etpConst['packagesext']):
continue
local_filepath = os.path.join(
self.Entropy._get_local_upload_directory(repo), branch, item)
local_filepath_pkgs = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch, item)
local_filepath = self.Entropy.complete_local_upload_package_path(
item, repo = repo)
local_filepath_pkgs = self.Entropy.complete_local_package_path(
item, repo = repo)
if os.path.isfile(local_filepath):
size = entropy.tools.get_file_size(local_filepath)
upload.append((local_filepath, size))
upload.append((local_filepath, item, size))
else:
size = entropy.tools.get_file_size(local_filepath_pkgs)
upload.append((local_filepath_pkgs, size))
upload.append((local_filepath_pkgs, item, size))
metainfo['upload'] += size
return upload, download, removal, do_copy, metainfo
def _sync_run_removal_queue(self, removal_queue, branch, repo = None):
def _sync_run_removal_queue(self, removal_queue, repo = None):
if repo is None:
repo = self.Entropy.default_repository
branch = self.SystemSettings['repositories']['branch']
for itemdata in removal_queue:
for remove_filepath, rel_path, size in removal_queue:
remove_filename = itemdata[0]
remove_filepath = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
remove_filename)
remove_filename = os.path.basename(remove_filepath)
remove_filepath_hash = remove_filepath + \
etpConst['packagesmd5fileext']
remove_filepath_exp = remove_filepath + \
etpConst['packagesexpirationfileext']
self.Entropy.output(
"[repo:%s|%s|%s] %s: %s [%s]" % (
brown(repo),
@@ -2638,7 +2656,7 @@ class Server(ServerNoticeBoardMixin):
brown(branch),
blue(_("removing package+hash")),
darkgreen(remove_filename),
blue(entropy.tools.bytes_into_human(itemdata[1])),
blue(entropy.tools.bytes_into_human(size)),
),
importance = 0,
type = "info",
@@ -2649,6 +2667,8 @@ class Server(ServerNoticeBoardMixin):
os.remove(remove_filepath)
if os.path.isfile(remove_filepath_hash):
os.remove(remove_filepath_hash)
if os.path.isfile(remove_filepath_exp):
os.remove(remove_filepath_exp)
self.Entropy.output(
"[repo:%s|%s|%s] %s" % (
@@ -2663,20 +2683,19 @@ class Server(ServerNoticeBoardMixin):
)
def _sync_run_copy_queue(self, copy_queue, branch, repo = None):
def _sync_run_copy_queue(self, copy_queue, repo = None):
if repo is None:
repo = self.Entropy.default_repository
branch = self.SystemSettings['repositories']['branch']
for itemdata in copy_queue:
from_file = itemdata[0]
for from_file, rel_file, size in copy_queue:
from_file_hash = from_file + etpConst['packagesmd5fileext']
to_file = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
os.path.basename(from_file))
to_file = self.Entropy.complete_local_package_path(rel_file)
to_file_hash = to_file+etpConst['packagesmd5fileext']
expiration_file = to_file+etpConst['packagesexpirationfileext']
self.Entropy.output(
"[repo:%s|%s|%s] %s: %s" % (
brown(repo),
@@ -2689,13 +2708,10 @@ class Server(ServerNoticeBoardMixin):
type = "info",
header = darkred(" * ")
)
if not os.path.isdir(os.path.dirname(to_file)):
os.makedirs(os.path.dirname(to_file))
self.Entropy._ensure_dir_path(os.path.dirname(to_file))
shutil.copy2(from_file, to_file)
if not os.path.isfile(from_file_hash):
self._create_file_checksum(from_file, from_file_hash)
self._create_file_checksum(from_file, from_file_hash)
shutil.copy2(from_file_hash, to_file_hash)
# clear expiration file
@@ -2703,30 +2719,46 @@ class Server(ServerNoticeBoardMixin):
os.remove(expiration_file)
def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):
def _sync_run_upload_queue(self, uri, upload_queue, repo = None):
if repo is None:
repo = self.Entropy.default_repository
branch = self.SystemSettings['repositories']['branch']
crippled_uri = EntropyTransceiver.get_uri_name(uri)
myqueue = []
for itemdata in upload_queue:
upload_item = itemdata[0]
hash_file = upload_item + etpConst['packagesmd5fileext']
queue_map = {}
for upload_path, rel_path, size in upload_queue:
hash_file = upload_path + etpConst['packagesmd5fileext']
if not os.path.isfile(hash_file):
entropy.tools.create_md5_file(upload_item)
myqueue.append(hash_file)
myqueue.append(upload_item)
entropy.tools.create_md5_file(upload_path)
remote_dir = os.path.join(
self.Entropy._get_remote_packages_relative_path(repo), branch)
rel_dir = os.path.dirname(rel_path)
obj = queue_map.setdefault(rel_dir, [])
obj.append(hash_file)
obj.append(upload_path)
uploader = self.TransceiverServerHandler(self.Entropy, [uri],
myqueue, critical_files = myqueue,
txc_basedir = remote_dir,
handlers_data = {'branch': branch }, repo = repo)
erorrs = False
m_fine_uris = set()
m_broken_uris = set()
for rel_path, myqueue in queue_map.items():
remote_dir = os.path.dirname(
self.Entropy.complete_remote_package_relative_path(
rel_path, repo = repo))
uploader = self.TransceiverServerHandler(self.Entropy, [uri],
myqueue, critical_files = myqueue,
txc_basedir = remote_dir,
handlers_data = {'branch': branch }, repo = repo)
xerrors, xm_fine_uris, xm_broken_uris = uploader.go()
if xerrors:
errors = True
m_fine_uris.update(xm_fine_uris)
m_broken_uris.update(xm_broken_uris)
errors, m_fine_uris, m_broken_uris = uploader.go()
if errors:
my_broken_uris = [
(EntropyTransceiver.get_uri_name(x[0]), x[1]) for \
@@ -2759,30 +2791,50 @@ class Server(ServerNoticeBoardMixin):
return errors, m_fine_uris, m_broken_uris
def _sync_run_download_queue(self, uri, download_queue, branch,
repo = None):
def _sync_run_download_queue(self, uri, download_queue, repo = None):
if repo is None:
repo = self.Entropy.default_repository
branch = self.SystemSettings['repositories']['branch']
crippled_uri = EntropyTransceiver.get_uri_name(uri)
myqueue = []
for package in download_queue:
hash_file = package + etpConst['packagesmd5fileext']
myqueue.append(package)
myqueue.append(hash_file)
remote_dir = os.path.join(
self.Entropy._get_remote_packages_relative_path(repo), branch)
local_basedir = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch)
downloader = self.TransceiverServerHandler(
self.Entropy, [uri], myqueue,
critical_files = myqueue,
txc_basedir = remote_dir, local_basedir = local_basedir,
handlers_data = {'branch': branch }, download = True, repo = repo)
queue_map = {}
for download_path, rel_path, size in download_queue:
hash_file = download_path + etpConst['packagesmd5fileext']
rel_dir = os.path.dirname(rel_path)
obj = queue_map.setdefault(rel_dir, [])
obj.append(hash_file)
obj.append(download_path)
erorrs = False
m_fine_uris = set()
m_broken_uris = set()
for rel_path, myqueue in queue_map.items():
remote_dir = os.path.dirname(
self.Entropy.complete_remote_package_relative_path(
rel_path, repo = repo))
local_basedir = os.path.dirname(
self.Entropy.complete_local_package_path(rel_path,
repo = repo))
downloader = self.TransceiverServerHandler(
self.Entropy, [uri], myqueue,
critical_files = myqueue,
txc_basedir = remote_dir, local_basedir = local_basedir,
handlers_data = {'branch': branch }, download = True,
repo = repo)
xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
if xerrors:
errors = True
m_fine_uris.update(xm_fine_uris)
m_broken_uris.update(xm_broken_uris)
errors, m_fine_uris, m_broken_uris = downloader.go()
if errors:
my_broken_uris = [
(EntropyTransceiver.get_uri_name(x), y,) \
@@ -2915,7 +2967,7 @@ class Server(ServerNoticeBoardMixin):
try:
upload_queue, download_queue, removal_queue, fine_queue, \
remote_packages_data = self.calculate_packages_to_sync(uri,
self.SystemSettings['repositories']['branch'], repo)
repo)
except self.socket.error as err:
self.Entropy.output(
"[repo:%s|%s|branch:%s] %s: %s, %s %s" % (
@@ -2957,20 +3009,14 @@ class Server(ServerNoticeBoardMixin):
header = red(" ** ")
)
upload, download, removal, copy, metainfo = self._expand_queues(
upload_queue,
download_queue,
removal_queue,
remote_packages_data,
self.SystemSettings['repositories']['branch'],
repo
)
upload, download, removal, copy_q, metainfo = self._expand_queues(
upload_queue, download_queue, removal_queue,
remote_packages_data, repo)
del upload_queue, download_queue, removal_queue, \
remote_packages_data
self._show_sync_queues(upload, download, removal, copy, metainfo,
self.SystemSettings['repositories']['branch'])
self._show_sync_queues(upload, download, removal, copy_q, metainfo)
if not len(upload)+len(download)+len(removal)+len(copy):
if not len(upload)+len(download)+len(removal)+len(copy_q):
self.Entropy.output(
"[repo:%s|%s|branch:%s] %s %s" % (
@@ -3009,31 +3055,26 @@ class Server(ServerNoticeBoardMixin):
repo = repo)
if removal:
self._sync_run_removal_queue(removal,
self.SystemSettings['repositories']['branch'], repo)
self._sync_run_removal_queue(removal, repo)
if copy:
self._sync_run_copy_queue(copy,
self.SystemSettings['repositories']['branch'], repo)
if copy_q:
self._sync_run_copy_queue(copy_q, repo)
if upload or download:
mirrors_tainted = True
if upload:
d_errors, m_fine_uris, \
m_broken_uris = self._sync_run_upload_queue(
uri, upload,
self.SystemSettings['repositories']['branch'], repo)
m_broken_uris = self._sync_run_upload_queue(uri,
upload, repo)
if d_errors:
mirror_errors = True
if download:
my_downlist = [x[0] for x in download]
d_errors, m_fine_uris, \
m_broken_uris = self._sync_run_download_queue(
uri, my_downlist,
self.SystemSettings['repositories']['branch'], repo)
m_broken_uris = self._sync_run_download_queue(uri,
download, repo)
if d_errors:
mirror_errors = True
@@ -3126,8 +3167,7 @@ class Server(ServerNoticeBoardMixin):
# if at least one server has been synced successfully, move files
if (len(successfull_mirrors) > 0) and not pretend:
self._remove_expiration_files(
self.SystemSettings['repositories']['branch'], repo)
self._move_files_over_from_upload(repo)
if packages_check:
check_data = self.Entropy.verify_local_packages([], ask = ask,
@@ -3136,39 +3176,38 @@ class Server(ServerNoticeBoardMixin):
return mirrors_tainted, mirrors_errors, successfull_mirrors, \
broken_mirrors, check_data
def _remove_expiration_files(self, branch, repo = None):
def _move_files_over_from_upload(self, repo = None):
if repo is None:
repo = self.Entropy.default_repository
branch_dir = os.path.join(
self.Entropy._get_local_upload_directory(repo), branch)
upload_dir = self.Entropy._get_local_upload_directory(repo = repo)
basedir_list = self.Entropy._get_basedir_pkg_listing(upload_dir,
repo = repo)
# check if it exists
if not os.path.isdir(branch_dir):
return None
for pkg_rel in basedir_list:
source_pkg = self.Entropy.complete_local_upload_package_path(
pkg_rel, repo = repo)
dest_pkg = self.Entropy.complete_local_package_path(pkg_rel,
repo = repo)
dest_pkg_dir = os.path.dirname(dest_pkg)
self.Entropy._ensure_dir_path(dest_pkg_dir)
try:
os.rename(source_pkg, dest_pkg)
except OSError: # on different hard drives?
shutil.move(source_pkg, dest_pkg)
branchcontent = os.listdir(branch_dir)
for xfile in branchcontent:
source = os.path.join(self.Entropy._get_local_upload_directory(repo),
branch, xfile)
destdir = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch)
if not os.path.isdir(destdir):
os.makedirs(destdir)
dest = os.path.join(destdir, xfile)
shutil.move(source, dest)
# clear expiration file
dest_expiration = dest + etpConst['packagesexpirationfileext']
dest_expiration = dest_pkg + etpConst['packagesexpirationfileext']
if os.path.isfile(dest_expiration):
os.remove(dest_expiration)
def _is_package_expired(self, package_rel, repo = None):
def _is_package_expired(self, package_file, branch, repo = None):
pkg_path = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
package_file)
pkg_path = self.Entropy.complete_local_package_path(package_rel,
repo = repo)
pkg_path += etpConst['packagesexpirationfileext']
if not os.path.isfile(pkg_path):
return False
@@ -3184,12 +3223,10 @@ class Server(ServerNoticeBoardMixin):
return True
return False
def _create_expiration_file(self, package_file, branch, repo = None,
gentle = False):
def _create_expiration_file(self, package_rel, repo = None, gentle = False):
pkg_path = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch,
package_file)
pkg_path = self.Entropy.complete_local_package_path(package_rel,
repo = repo)
pkg_path += etpConst['packagesexpirationfileext']
if gentle and os.path.isfile(pkg_path):
return
@@ -3197,25 +3234,24 @@ class Server(ServerNoticeBoardMixin):
f_exp.flush()
f_exp.close()
def _collect_expiring_packages(self, branch, repo = None):
dbconn = self.Entropy.open_server_repository(just_reading = True,
repo = repo)
database_bins = dbconn.listAllDownloads(do_sort = False,
full_path = True)
bins_dir = os.path.join(
self.Entropy._get_local_packages_directory(repo), branch)
repo_bins = set()
if os.path.isdir(bins_dir):
repo_bins = os.listdir(bins_dir)
repo_bins = set([
os.path.join('packages', etpSys['arch'], branch, x) for x \
in repo_bins if x.endswith(etpConst['packagesext'])])
database_bins = set(dbconn.listAllDownloads(do_sort = False,
full_path = True))
repo_basedir = self.Entropy._get_local_repository_base_directory(
repo = repo)
repo_bins = self.Entropy._get_basedir_pkg_listing(repo_basedir,
repo = repo, branch = branch)
# convert to set, so that we can do fast thingszzsd
repo_bins = set(repo_bins)
repo_bins -= database_bins
return set([os.path.basename(x) for x in repo_bins])
return repo_bins
def tidy_mirrors(self, ask = True, pretend = False, repo = None):
@@ -3266,29 +3302,17 @@ class Server(ServerNoticeBoardMixin):
for key, val in branch_pkglist_data.items():
branch_pkglist_data[key] = val.split("\n")
rel_path_basedir = etpConst['packagesrelativepath_basedir']
rel_path_basename = etpConst['packagesrelativepath_basename']
remote_relpath = rel_path_basedir + "/" + rel_path_basename + "/" \
+ branch
my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \
expiring_packages])
for other_branch in branch_pkglist_data:
branch_pkglist = set(branch_pkglist_data[other_branch])
my_expiring_pkgs -= branch_pkglist
# fallback to normality, set new expiring packages var
expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs]
expiring_packages -= branch_pkglist
removal = []
for package in expiring_packages:
expired = self._is_package_expired(package, branch, repo)
for package_rel in expiring_packages:
expired = self._is_package_expired(package_rel, repo)
if expired:
removal.append(package)
removal.append(package_rel)
else:
self._create_expiration_file(package, branch, repo,
gentle = True)
self._create_expiration_file(package_rel, repo, gentle = True)
# fill returning data
branch_data['removal'] = removal[:]
@@ -3335,72 +3359,87 @@ class Server(ServerNoticeBoardMixin):
if rc_question == _("No"):
return errors, branch_data
myqueue = []
for package in removal:
myqueue.append(package+etpConst['packagesmd5fileext'])
myqueue.append(package)
# split queue by remote directories to work on
removal_map = {}
for package_rel in removal:
rel_path = self.Entropy.complete_remote_package_rel_path(
package_rel, repo = repo)
rel_dir = os.path.dirname(rel_path)
obj = removal_map.setdefault(rel_dir, [])
base_pkg = os.path.basename(package)
obj.append(base_pkg)
obj.append(base_pkg+etpConst['packagesmd5fileext'])
remote_dir = os.path.join(
self.Entropy._get_remote_packages_relative_path(repo), branch)
for uri in self.Entropy.get_remote_mirrors(repo):
self.Entropy.output(
"[branch:%s] %s..." % (
brown(branch),
blue(_("removing packages remotely")),
),
importance = 1,
type = "info",
header = blue(" @@ ")
)
##
# remove remotely
##
crippled_uri = EntropyTransceiver.get_uri_name(uri)
destroyer = self.TransceiverServerHandler(
self.Entropy,
[uri],
myqueue,
critical_files = [],
txc_basedir = remote_dir,
remove = True,
repo = repo
)
errors, m_fine_uris, m_broken_uris = destroyer.go()
if errors:
my_broken_uris = [
(EntropyTransceiver.get_uri_name(x[0]), x[1]) \
for x in m_broken_uris]
for remote_dir, myqueue in removal_map.items():
reason = my_broken_uris[0][1]
self.Entropy.output(
"[branch:%s] %s: %s, %s: %s" % (
"[branch:%s] %s..." % (
brown(branch),
blue(_("remove errors")),
red(crippled_uri),
blue(_("reason")),
reason,
blue(_("removing packages remotely")),
),
importance = 1,
type = "warning",
header = brown(" !!! ")
type = "info",
header = blue(" @@ ")
)
branch_data['errors'] = True
errors = True
self.Entropy.output(
"[branch:%s] %s..." % (
brown(branch),
blue(_("removing packages locally")),
),
importance = 1,
type = "info",
header = blue(" @@ ")
)
crippled_uri = EntropyTransceiver.get_uri_name(uri)
destroyer = self.TransceiverServerHandler(
self.Entropy,
[uri],
myqueue,
critical_files = [],
txc_basedir = remote_dir,
remove = True,
repo = repo
)
errors, m_fine_uris, m_broken_uris = destroyer.go()
if errors:
my_broken_uris = [
(EntropyTransceiver.get_uri_name(x[0]), x[1]) \
for x in m_broken_uris]
reason = my_broken_uris[0][1]
self.Entropy.output(
"[branch:%s] %s: %s, %s: %s" % (
brown(branch),
blue(_("remove errors")),
red(crippled_uri),
blue(_("reason")),
reason,
),
importance = 1,
type = "warning",
header = brown(" !!! ")
)
branch_data['errors'] = True
errors = True
self.Entropy.output(
"[branch:%s] %s..." % (
brown(branch),
blue(_("removing packages locally")),
),
importance = 1,
type = "info",
header = blue(" @@ ")
)
##
# remove locally
##
branch_data['removed'] = set()
for package in removal:
package_path = os.path.join(
self.Entropy._get_local_packages_directory(repo),
branch, package)
for package_rel in removal:
package_path = self.Entropy.complete_local_package_path(
package_rel, repo = repo)
package_path_hash = package_path + \
etpConst['packagesmd5fileext']
package_path_expired = package_path + \

3
libraries/entropy/services/system/executors.py
View File

@@ -808,8 +808,7 @@ class Base:
try:
upload_queue, download_queue, removal_queue, \
fine_queue, remote_packages_data = Entropy.Mirrors.calculate_packages_to_sync(
uri, Entropy.SystemSettings['repositories']['branch'],
repoid)
uri, repoid)
except socket.error:
entropy.tools.print_traceback(f = stdout_err)
stdout_err.write("\n"+_("Socket error, continuing...").encode('utf-8')+"\n")

1
server/reagent.py
View File

@@ -73,7 +73,6 @@ help_opts = [
None,
(1, 'database', 2, _('repository database functions')),
(2, '--initialize', 3, _('(re)initialize the current repository database')),
(3, '--empty', 2, _('do not refill database using packages on mirrors')),
(3, '--repo=<repo>', 2, _('(re)create the database for the specified repository')),
(2, 'bump', 4, _('manually force a revision bump for the current repository database')),
(3, '--sync', 3, _('synchronize the database')),

7
server/server_reagent.py
View File

@@ -594,7 +594,6 @@ def database(options):
d_request_noask = False
d_request_sync = False
d_request_empty = False
repo = None
_options = []
for opt in options:
@@ -602,11 +601,8 @@ def database(options):
d_request_noask = True
elif opt.startswith("--sync"):
d_request_sync = True
elif opt.startswith("--empty"):
d_request_empty = True
elif opt.startswith("--repo=") and len(opt.split("=")) == 2:
repo = opt.split("=")[1]
d_request_empty = True
else:
_options.append(opt)
options = _options
@@ -617,8 +613,7 @@ def database(options):
if (options[0] == "--initialize"):
rc = Entropy.initialize_server_repository(empty = d_request_empty,
repo = repo)
rc = Entropy.initialize_server_repository(repo = repo)
if rc == 0:
print_info(darkgreen(" * ")+red(_("Entropy database has been reinitialized using binary packages available")))