From a0245b84cab7c208d39acad6a783e87c75720c1e Mon Sep 17 00:00:00 2001
From: Fabio Erculiani <lxnay@sabayon.org>
Date: Wed, 27 Jan 2010 18:37:01 +0100
Subject: [PATCH] [entropy.server] add support for multiple packages
 directories (to host nonfree pkgs)

---
 docs/man/man1/reagent.1                       |   5 +-
 docs/man/reagent.pod                          |   4 -
 libraries/entropy/server/interfaces/main.py   | 449 +++++---------
 .../entropy/server/interfaces/mirrors.py      | 581 ++++++++++--------
 .../entropy/services/system/executors.py      |   3 +-
 server/reagent.py                             |   1 -
 server/server_reagent.py                      |   7 +-
 7 files changed, 482 insertions(+), 568 deletions(-)

diff --git a/docs/man/man1/reagent.1 b/docs/man/man1/reagent.1
index 0074ae0d5..de0a7a1cc 100644
--- a/docs/man/man1/reagent.1
+++ b/docs/man/man1/reagent.1
@@ -129,7 +129,7 @@
 .\" ========================================================================
 .\"
 .IX Title "REAGENT 1"
-.TH REAGENT 1 "2010-01-23" "perl v5.8.8" "Entropy"
+.TH REAGENT 1 "2010-01-27" "perl v5.8.8" "Entropy"
 .SH "NAME"
 \&\fBreagent\fR \- Official Sabayon Linux Server\-Side Package Management tool
 .SH "SYNOPSIS"
@@ -258,9 +258,6 @@ repository database functions
 .IX Item "--initialize"
 (re)initialize the current repository database
 .RS 4
-.IP "\fB\-\-empty\fR" 4
-.IX Item "--empty"
-do not refill database using packages on mirrors
 .IP "\fB\-\-repo=<repo\fR>" 4
 .IX Item "--repo=<repo>"
 (re)create the database for the specified repository
diff --git a/docs/man/reagent.pod b/docs/man/reagent.pod
index 915cf16d9..9bf523353 100644
--- a/docs/man/reagent.pod
+++ b/docs/man/reagent.pod
@@ -174,10 +174,6 @@ repository database functions
 =over
 
 
-=item B<--empty>
-
-do not refill database using packages on mirrors
-
 =item B<--repo=<repo>>
 
 (re)create the database for the specified repository
diff --git a/libraries/entropy/server/interfaces/main.py b/libraries/entropy/server/interfaces/main.py
index c21df769e..fb011438e 100644
--- a/libraries/entropy/server/interfaces/main.py
+++ b/libraries/entropy/server/interfaces/main.py
@@ -620,59 +620,55 @@ class ServerSystemSettingsPlugin(SystemSettingsPlugin):
 
         # expand paths
         for repoid in data['repositories']:
+            data['repositories'][repoid]['repo_basedir'] = \
+                os.path.join(   etpConst['entropyworkdir'],
+                                "server",
+                                repoid
+                            )
             data['repositories'][repoid]['packages_dir'] = \
                 os.path.join(   etpConst['entropyworkdir'],
                                 "server",
                                 repoid,
                                 etpConst['packagesrelativepath_basedir'],
-                                etpSys['arch']
+                                etpConst['currentarch']
                             )
             data['repositories'][repoid]['packages_dir_nonfree'] = \
                 os.path.join(   etpConst['entropyworkdir'],
                                 "server",
                                 repoid,
                                 etpConst['packagesrelativepath_basedir_nonfree'],
-                                etpSys['arch']
+                                etpConst['currentarch']
                             )
             data['repositories'][repoid]['store_dir'] = \
                 os.path.join(   etpConst['entropyworkdir'],
                                 "server",
                                 repoid,
                                 "store",
-                                etpSys['arch']
+                                etpConst['currentarch']
                             )
-            data['repositories'][repoid]['upload_dir'] = \
+            data['repositories'][repoid]['upload_basedir'] = \
                 os.path.join(   etpConst['entropyworkdir'],
                                 "server",
                                 repoid,
-                                "upload",
-                                etpSys['arch']
+                                "upload" # consider this a base dir
                             )
             data['repositories'][repoid]['database_dir'] = \
                 os.path.join(   etpConst['entropyworkdir'],
                                 "server",
                                 repoid,
                                 "database",
-                                etpSys['arch']
+                                etpConst['currentarch']
                             )
-            data['repositories'][repoid]['packages_relative_path'] = \
+            data['repositories'][repoid]['remote_repo_basedir'] = \
                 os.path.join(   sys_set['repositories']['product'],
-                                repoid,
-                                etpConst['packagesrelativepath_basedir'],
-                                etpSys['arch']
-                            )+"/"
-            data['repositories'][repoid]['packages_relative_path_nonfree'] = \
-                os.path.join(   sys_set['repositories']['product'],
-                                repoid,
-                                etpConst['packagesrelativepath_basedir_nonfree'],
-                                etpSys['arch']
-                            )+"/"
+                                repoid
+                            )
             data['repositories'][repoid]['database_relative_path'] = \
                 os.path.join(   sys_set['repositories']['product'],
                                 repoid,
                                 "database",
-                                etpSys['arch']
-                            )+"/"
+                                etpConst['currentarch']
+                            )
 
         # Support for shell variables
         shell_repoid = os.getenv('ETP_REPO')
@@ -755,17 +751,28 @@ class ServerQAInterfacePlugin(QAInterfacePlugin):
 
 class ServerSettingsMixin:
 
-    def _get_remote_packages_relative_path(self, repo = None):
-        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
-        if repo is None:
-            repo = self.default_repository
-        return srv_set['repositories'][repo]['packages_relative_path']
+    def _get_basedir_pkg_listing(self, base_dir, repo = None, branch = None):
 
-    def _get_remote_packages_nonfree_relative_path(self, repo = None):
-        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
-        if repo is None:
-            repo = self.default_repository
-        return srv_set['repositories'][repo]['packages_relative_path_nonfree']
+        pkgs_dir_types = self.Entropy._get_pkg_dir_names()
+        basedir_raw_content = []
+        entropy.tools.recursive_directory_relative_listing(
+            basedir_raw_content, base_dir)
+
+        pkg_ext = etpConst['packagesext']
+        pkg_list = [x for x in basedir_raw_content if \
+            x.split(os.path.sep)[0] in pkgs_dir_types and \
+                x.endswith(pkg_ext)]
+
+        if branch is not None:
+            branch_extractor = \
+                self.Entropy.Client.get_branch_from_download_relative_uri
+            pkg_list = [x for x in pkg_list if branch_extractor(x) == branch]
+
+        return pkg_list
+
+    def _get_pkg_dir_names(self):
+        return [etpConst['packagesrelativepath_basedir'],
+            etpConst['packagesrelativepath_basedir_nonfree']]
 
     def _get_remote_database_relative_path(self, repo = None):
         srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
@@ -789,19 +796,13 @@ class ServerSettingsMixin:
         srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
         if repo is None:
             repo = self.default_repository
-        return srv_set['repositories'][repo]['upload_dir']
+        return srv_set['repositories'][repo]['upload_basedir']
 
-    def _get_local_packages_directory(self, repo = None):
+    def _get_local_repository_base_directory(self, repo = None):
         srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
         if repo is None:
             repo = self.default_repository
-        return srv_set['repositories'][repo]['packages_dir']
-
-    def _get_local_packages_nonfree_directory(self, repo = None):
-        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
-        if repo is None:
-            repo = self.default_repository
-        return srv_set['repositories'][repo]['packages_dir_nonfree']
+        return srv_set['repositories'][repo]['repo_basedir']
 
     def _get_local_database_taint_file(self, repo = None, branch = None):
         if repo is None:
@@ -1018,6 +1019,27 @@ class ServerSettingsMixin:
         if os.path.isfile(lock_file):
             os.remove(lock_file)
 
+    def complete_remote_package_relative_path(self, pkg_rel_url, repo = None):
+        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
+        if repo is None:
+            repo = self.default_repository
+        return os.path.join(
+            srv_set['repositories'][repo]['remote_repo_basedir'], pkg_rel_url)
+
+    def complete_local_upload_package_path(self, pkg_rel_url, repo = None):
+        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
+        if repo is None:
+            repo = self.default_repository
+        return os.path.join(srv_set['repositories'][repo]['upload_basedir'],
+            pkg_rel_url)
+
+    def complete_local_package_path(self, pkg_rel_url, repo = None):
+        srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
+        if repo is None:
+            repo = self.default_repository
+        return os.path.join(srv_set['repositories'][repo]['repo_basedir'],
+            pkg_rel_url)
+
     def get_remote_mirrors(self, repo = None):
         srv_set = self.SystemSettings[self.sys_settings_plugin_id]['server']
         if repo is None:
@@ -1168,8 +1190,7 @@ class ServerPackageDepsMixin:
 
 class ServerPackagesHandlingMixin:
 
-    def initialize_server_repository(self, empty = True, repo = None,
-        warnings = True):
+    def initialize_server_repository(self, repo = None, show_warnings = True):
 
         if repo is None:
             repo = self.default_repository
@@ -1191,7 +1212,7 @@ class ServerPackagesHandlingMixin:
         if os.path.isfile(self._get_local_database_file(repo)):
 
             dbconn = self.open_server_repository(read_only = True,
-                no_upload = True, repo = repo, warnings = warnings)
+                no_upload = True, repo = repo, warnings = show_warnings)
 
             try:
                 dbconn.validateDatabase()
@@ -1247,151 +1268,6 @@ class ServerPackagesHandlingMixin:
             no_upload = True, repo = repo, is_new = True)
         dbconn.initializeDatabase()
 
-        if not empty:
-
-            revisions_file = "/entropy-revisions-dump.txt"
-            # dump revisions - as a backup
-            if revisions_match:
-                self.output(
-                    "%s: %s" % (
-                        red(_("Dumping current revisions to file")),
-                        darkgreen(revisions_file),
-                    ),
-                    importance = 1,
-                    type = "info",
-                    header = darkgreen(" * ")
-                )
-                f_rev = open(revisions_file, "w")
-                f_rev.write(str(revisions_match))
-                f_rev.flush()
-                f_rev.close()
-
-            # dump treeupdates - as a backup
-            treeupdates_file = "/entropy-treeupdates-dump.txt"
-            if treeupdates_actions:
-                self.output(
-                    "%s: %s" % (
-                        # do not translate treeupdates
-                        red(_("Dumping current 'treeupdates' actions to file")),
-                        bold(treeupdates_file),
-                    ),
-                    importance = 1,
-                    type = "info",
-                    header = darkgreen(" * ")
-                )
-                f_tree = open(treeupdates_file, "w")
-                f_tree.write(str(treeupdates_actions))
-                f_tree.flush()
-                f_tree.close()
-
-            rc_question = self.ask_question(
-                _("Would you like to sync packages first (important!) ?"))
-            if rc_question == _("Yes"):
-                self.Mirrors.sync_packages(repo = repo)
-
-            # fill tree updates actions
-            if treeupdates_actions:
-                dbconn.bumpTreeUpdatesActions(treeupdates_actions)
-
-            # now fill the database
-            pkg_branch_dir = os.path.join(
-                self._get_local_packages_directory(repo),
-                self.SystemSettings['repositories']['branch'])
-            pkglist = os.listdir(pkg_branch_dir)
-            # filter .md5 and .expired packages
-            pkg_ext_len = len(etpConst['packagesext'])
-            pkglist = [x for x in pkglist if (x[(pkg_ext_len*-1):] == \
-                etpConst['packagesext']) and not \
-                os.path.isfile(os.path.join(pkg_branch_dir,
-                    x + etpConst['packagesexpirationfileext']))]
-
-            if pkglist:
-                self.output(
-                    "%s '%s' %s %s" % (
-                        red(_("Reinitializing Entropy database for branch")),
-                        bold(self.SystemSettings['repositories']['branch']),
-                        red(_("using Packages in the repository")),
-                        red("..."),
-                    ),
-                    importance = 1,
-                    type = "info",
-                    header = darkgreen(" * ")
-                )
-
-            counter = 0
-            maxcount = len(pkglist)
-            branch = self.SystemSettings['repositories']['branch']
-            for pkg in pkglist:
-                counter += 1
-
-                self.output(
-                    "[repo:%s|%s] %s: %s" % (
-                        darkgreen(repo),
-                        brown(branch),
-                        blue(_("analyzing")),
-                        bold(pkg),
-                    ),
-                    importance = 1,
-                    type = "info",
-                    header = " ",
-                    back = True,
-                    count = (counter, maxcount,)
-                )
-
-                doinject = False
-                if pkg in injected_packages:
-                    doinject = True
-
-                pkg_path = os.path.join(self._get_local_packages_directory(repo),
-                    branch, pkg)
-                mydata = self.Spm().extract_package_metadata(pkg_path)
-                self._pump_extracted_package_metadata(mydata, repo,
-                    {'injected': doinject,})
-
-                # get previous revision
-                revision_avail = revisions_match.get(pkg)
-                add_revision = 0
-                if (revision_avail != None):
-                    if branch == revision_avail[0]:
-                        add_revision = revision_avail[1]
-
-                idpackage, revision, mydata_upd = dbconn.addPackage(mydata,
-                    revision = add_revision)
-                idpackages_added.add(idpackage)
-
-                self.output(
-                    "[repo:%s] [%s:%s/%s] %s: %s, %s: %s" % (
-                            repo,
-                            brown(branch),
-                            darkgreen(str(counter)),
-                            blue(str(maxcount)),
-                            red(_("added package")),
-                            darkgreen(pkg),
-                            red(_("revision")),
-                            brown(str(revision)),
-                    ),
-                    importance = 1,
-                    type = "info",
-                    header = " ",
-                    back = True
-                )
-
-            self.generate_reverse_dependencies_metadata(repo)
-
-            my_qa = self.QA()
-
-            if idpackages_added:
-                dbconn = self.open_server_repository(read_only = False,
-                    no_upload = True, repo = repo)
-                my_qa.test_missing_dependencies(
-                    idpackages_added, dbconn, ask = True,
-                    repo = repo, self_check = True,
-                    black_list = \
-                        self._get_missing_dependencies_blacklist(repo = repo),
-                    black_list_adder = \
-                        self._add_missing_dependencies_blacklist_items
-                )
-
         dbconn.commitChanges()
         self.close_repositories()
 
@@ -1515,8 +1391,6 @@ class ServerPackagesHandlingMixin:
         tmp_down_dir = tempfile.mkdtemp()
 
         download_queue = {}
-        local_up_dir = self._get_local_upload_directory(repo)
-        local_basedir = os.path.join(local_up_dir, branch)
         dbconn = self.open_server_repository(read_only = False,
             no_upload = True, repo = repo)
 
@@ -1566,26 +1440,41 @@ class ServerPackagesHandlingMixin:
                 if rc_question == _("No"):
                     continue
 
-            remote_relative_path = self._get_remote_packages_relative_path(repo)
-
             for uri in self.get_remote_mirrors(repo):
 
                 crippled_uri = EntropyTransceiver.get_uri_name(uri)
-                basedir = os.path.join(remote_relative_path, from_branch)
 
-                downloader_queue = [x[0] for x in download_queue[from_branch]]
-                downloader = self.Mirrors.TransceiverServerHandler(
-                    self,
-                    [uri],
-                    downloader_queue,
-                    critical_files = downloader_queue,
-                    txc_basedir = basedir,
-                    local_basedir = tmp_down_dir,
-                    download = True,
-                    repo = repo
-                )
+                queue_map = {}
 
-                errors, m_fine_uris, m_broken_uris = downloader.go()
+                for pkg_fp, idpackage in download_queue[from_branch]:
+                    down_url = dbconn.retrieveDownloadURL(idpackage)
+                    down_rel = self.complete_remote_package_relative_path(
+                        down_url, repo = repo)
+                    down_rel_dir = os.path.dirname(down_rel)
+                    obj = queue_map.setdefault(down_rel_dir, [])
+                    obj.append(pkg_fp)
+
+                errors = False
+                m_fine_uris = set()
+                m_broken_uris = set()
+
+                for down_rel_dir, downloader_queue in queue_map.items():
+
+                    downloader = self.Mirrors.TransceiverServerHandler(
+                        self,
+                        [uri],
+                        downloader_queue,
+                        critical_files = downloader_queue,
+                        txc_basedir = down_rel_dir,
+                        local_basedir = tmp_down_dir,
+                        download = True,
+                        repo = repo
+                    )
+                    xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
+                    if xerrors:
+                        errors = True
+                    m_fine_uris.update(xm_fine_uris)
+                    m_broken_uris.update(xm_broken_uris)
 
                 if not errors:
                     for downloaded_path, idpackage in \
@@ -1710,9 +1599,17 @@ class ServerPackagesHandlingMixin:
                     back = True
                 )
 
+                # build new download url
+                download_url = dbconn.retrieveDownloadURL(idpackage)
+                download_url = \
+                    self.Client.swap_branch_in_download_relative_uri(
+                        branch, download_url)
+
                 # move files to upload
-                package_name = os.path.basename(package_path)
-                new_package_path = os.path.join(local_basedir, package_name)
+                new_package_path = self.complete_local_upload_package_path(
+                    download_url, repo = repo)
+                self._ensure_dir_path(os.path.dirname(new_package_path))
+
                 try:
                     os.rename(package_path, new_package_path)
                 except OSError:
@@ -1722,10 +1619,6 @@ class ServerPackagesHandlingMixin:
                 entropy.tools.create_md5_file(new_package_path)
 
                 # update database
-                download_url = dbconn.retrieveDownloadURL(idpackage)
-                download_url = \
-                    self.Client.swap_branch_in_download_relative_uri(
-                        branch, download_url)
                 dbconn.setDownloadURL(idpackage, download_url)
                 dbconn.switchBranch(idpackage, branch)
                 dbconn.commitChanges()
@@ -1903,19 +1796,20 @@ class ServerPackagesHandlingMixin:
                 return switched
 
         for idpackage, repo in my_matches:
+
             dbconn = self.open_server_repository(read_only = False,
                 no_upload = True, repo = repo)
-            match_branch = dbconn.retrieveBranch(idpackage)
+
             match_atom = dbconn.retrieveAtom(idpackage)
-            package_filename = os.path.basename(
-                dbconn.retrieveDownloadURL(idpackage))
+            package_rel_path = dbconn.retrieveDownloadURL(idpackage)
+
             self.output(
                 "[%s=>%s|%s] %s: %s" % (
-                        darkgreen(repo),
-                        darkred(to_repo),
-                        brown(branch),
-                        blue(_("switching")),
-                        darkgreen(match_atom),
+                    darkgreen(repo),
+                    darkred(to_repo),
+                    brown(branch),
+                    blue(_("switching")),
+                    darkgreen(match_atom),
                 ),
                 importance = 0,
                 type = "info",
@@ -1923,11 +1817,11 @@ class ServerPackagesHandlingMixin:
                 back = True
             )
             # move binary file
-            from_file = os.path.join(self._get_local_packages_directory(repo),
-                match_branch, package_filename)
+            from_file = self.complete_local_package_path(package_rel_path,
+                repo = repo)
             if not os.path.isfile(from_file):
-                from_file = os.path.join(self._get_local_upload_directory(repo),
-                    match_branch, package_filename)
+                from_file = self.complete_local_upload_package_path(
+                    package_rel_path, repo = repo)
             if not os.path.isfile(from_file):
                 self.output(
                     "[%s=>%s|%s] %s: %s -> %s" % (
@@ -1944,20 +1838,25 @@ class ServerPackagesHandlingMixin:
                 )
                 continue
 
+            to_file = self.complete_local_upload_package_path(package_rel_path,
+                repo = to_repo)
+
             if new_tag != None:
+
                 match_category = dbconn.retrieveCategory(idpackage)
                 match_name = dbconn.retrieveName(idpackage)
                 match_version = dbconn.retrieveVersion(idpackage)
                 tagged_package_filename = \
                     entropy.tools.create_package_filename(
                         match_category, match_name, match_version, new_tag)
-                to_file = os.path.join(self._get_local_upload_directory(to_repo),
-                    match_branch, tagged_package_filename)
-            else:
-                to_file = os.path.join(self._get_local_upload_directory(to_repo),
-                    match_branch, package_filename)
-            if not os.path.isdir(os.path.dirname(to_file)):
-                os.makedirs(os.path.dirname(to_file))
+
+                to_file = self.complete_local_upload_package_path(
+                    package_rel_path, repo = to_repo)
+                # directly move to correct place, tag changed, so file name
+                to_file = os.path.join(os.path.dirname(to_file),
+                    tagged_package_filename)
+
+            self._ensure_dir_path(os.path.dirname(to_file))
 
             copy_data = [
                 (from_file, to_file,),
@@ -1969,17 +1868,17 @@ class ServerPackagesHandlingMixin:
 
             for from_item, to_item in copy_data:
                 self.output(
-                        "[%s=>%s|%s] %s: %s" % (
-                            darkgreen(repo),
-                            darkred(to_repo),
-                            brown(branch),
-                            blue(_("moving file")),
-                            darkgreen(os.path.basename(from_item)),
-                        ),
-                        importance = 0,
-                        type = "info",
-                        header = red(" @@ "),
-                        back = True
+                    "[%s=>%s|%s] %s: %s" % (
+                        darkgreen(repo),
+                        darkred(to_repo),
+                        brown(branch),
+                        blue(_("moving file")),
+                        darkgreen(os.path.basename(from_item)),
+                    ),
+                    importance = 0,
+                    type = "info",
+                    header = red(" @@ "),
+                    back = True
                 )
                 if os.path.isfile(from_item):
                     shutil.copy2(from_item, to_item)
@@ -2765,7 +2664,6 @@ class ServerPackagesHandlingMixin:
             uploaddir_path = self._get_upload_package_path(repo, dbconn,
                 idpackage)
             pkg_path = dbconn.retrieveDownloadURL(idpackage)
-            pkg_rel_path = self._get_common_pkg_relative_path(pkg_path)
 
             pkgatom = dbconn.retrieveAtom(idpackage)
             if os.path.isfile(bindir_path):
@@ -2773,7 +2671,7 @@ class ServerPackagesHandlingMixin:
                     "[%s] %s :: %s" % (
                         darkgreen(_("available")),
                         blue(pkgatom),
-                        darkgreen(pkg_rel_path),
+                        darkgreen(pkg_path),
                     ),
                     importance = 0,
                     type = "info",
@@ -2785,7 +2683,7 @@ class ServerPackagesHandlingMixin:
                     "[%s] %s :: %s" % (
                         darkred(_("upload/ignored")),
                         blue(pkgatom),
-                        darkgreen(pkg_rel_path),
+                        darkgreen(pkg_path),
                     ),
                     importance = 0,
                     type = "info",
@@ -2796,7 +2694,7 @@ class ServerPackagesHandlingMixin:
                     "[%s] %s :: %s" % (
                         brown(_("download")),
                         blue(pkgatom),
-                        darkgreen(pkg_rel_path),
+                        darkgreen(pkg_path),
                     ),
                     importance = 0,
                     type = "info",
@@ -3414,8 +3312,8 @@ class ServerRepositoryMixin:
                 dbfile = self._get_local_database_file(repoid)
                 if os.path.isfile(dbfile):
                     shutil.move(dbfile, dbfile+".backup")
-                self.initialize_server_repository(empty = True,
-                    repo = repoid, warnings = False)
+                self.initialize_server_repository(repo = repoid,
+                    show_warnings = False)
 
     def _save_default_repository(self, repoid):
 
@@ -3694,8 +3592,7 @@ class ServerRepositoryMixin:
                 return cached
 
         local_dbfile_dir = os.path.dirname(local_dbfile)
-        if not os.path.isdir(local_dbfile_dir):
-            os.makedirs(local_dbfile_dir)
+        self._ensure_dir_path(local_dbfile_dir)
 
         if (not read_only) and (lock_remote) and \
             (repo not in self._sync_lock_cache):
@@ -3830,11 +3727,6 @@ class ServerRepositoryMixin:
         if repo is None:
             repo = self.default_repository
 
-        upload_dir = os.path.join(self._get_local_upload_directory(repo),
-            self.SystemSettings['repositories']['branch'])
-        if not os.path.isdir(upload_dir):
-            os.makedirs(upload_dir)
-
         dbconn = self.open_server_repository(read_only = False,
             no_upload = True, repo = repo)
         self.output(
@@ -3943,8 +3835,11 @@ class ServerRepositoryMixin:
 
         download_url = self._setup_repository_package_filename(idpackage,
             repo = repo)
-        downloadfile = os.path.basename(download_url)
-        destination_path = os.path.join(upload_dir, downloadfile)
+        destination_path = self.complete_local_upload_package_path(
+            download_url, repo = repo)
+        destination_dir = os.path.dirname(destination_path)
+        self._ensure_dir_path(destination_dir)
+
         try:
             os.rename(package_file, destination_path)
         except OSError:
@@ -4103,14 +3998,18 @@ class ServerRepositoryMixin:
 class ServerMiscMixin:
 
     def _ensure_paths(self, repo):
-        upload_dir = os.path.join(self._get_local_upload_directory(repo),
-            self.SystemSettings['repositories']['branch'])
+        upload_dir = self._get_local_upload_directory(repo)
         db_dir = self._get_local_database_dir(repo)
         for mydir in [upload_dir, db_dir]:
             if (not os.path.isdir(mydir)) and (not os.path.lexists(mydir)):
-                os.makedirs(mydir)
+                os.makedirs(mydir, 0o775)
                 const_setup_perms(mydir, etpConst['entropygid'])
 
+    def _ensure_dir_path(self, dir_path):
+        if not os.path.isdir(dir_path):
+            os.makedirs(dir_path, 0o775)
+            const_setup_perms(dir_path, etpConst['entropygid'])
+
     def _setup_services(self):
         self._setup_entropy_settings()
         cs_name = 'Client'
@@ -4306,34 +4205,24 @@ class ServerMiscMixin:
     def get_branch_from_download_relative_uri(self, mypath):
         return self.Client.get_branch_from_download_relative_uri(mypath)
 
-    def _get_common_pkg_relative_path(self, pkg_path, branch = None):
-        my_path = '/'.join(pkg_path.split("/")[2:])
-        if branch is None:
-            return my_path
-        head, tail = os.path.split(my_path)
-        return os.path.join(branch, tail)
+    def _get_package_path(self, repo, dbconn, idpackage):
+        """
+        Given EntropyRepository instance and package identifier, return local
+        path of package. This method does not check for path validity though.
+        """
+        pkg_rel_url = dbconn.retrieveDownloadURL(idpackage)
+        complete_path = self.complete_local_package_path(pkg_rel_url,
+            repo = repo)
+        return complete_path
 
-    def _get_package_path(self, repo, dbconn, idpackage, branch = None):
+    def _get_upload_package_path(self, repo, dbconn, idpackage):
         """
         Given EntropyRepository instance and package identifier, return local
         path of package. This method does not check for path validity though.
         """
         pkg_path = dbconn.retrieveDownloadURL(idpackage)
-        pkg_rel_path = self._get_common_pkg_relative_path(pkg_path,
-            branch = branch)
-        return os.path.join(self._get_local_packages_directory(repo),
-            pkg_rel_path)
-
-    def _get_upload_package_path(self, repo, dbconn, idpackage, branch = None):
-        """
-        Given EntropyRepository instance and package identifier, return local
-        path of package. This method does not check for path validity though.
-        """
-        pkg_path = dbconn.retrieveDownloadURL(idpackage)
-        pkg_rel_path = self._get_common_pkg_relative_path(pkg_path,
-            branch = branch)
-        return os.path.join(self._get_local_upload_directory(repo),
-            pkg_rel_path)
+        return os.path.join(self._get_local_upload_directory(repo = repo),
+            pkg_path)
 
     def scan_package_changes(self):
 
diff --git a/libraries/entropy/server/interfaces/mirrors.py b/libraries/entropy/server/interfaces/mirrors.py
index 5436b4178..55c07207e 100644
--- a/libraries/entropy/server/interfaces/mirrors.py
+++ b/libraries/entropy/server/interfaces/mirrors.py
@@ -18,7 +18,7 @@ from entropy.exceptions import OnlineMirrorError, ConnectionError, \
     EntropyPackageException, TransceiverError
 from entropy.output import red, darkgreen, bold, brown, blue, darkred, \
     darkblue, purple
-from entropy.const import etpConst, etpSys
+from entropy.const import etpConst, const_setup_perms
 from entropy.i18n import _
 from entropy.misc import RSS
 from entropy.server.interfaces.rss import ServerRssMetadata
@@ -634,7 +634,6 @@ class Server(ServerNoticeBoardMixin):
         if repo is None:
             repo = self.Entropy.default_repository
 
-        pkg_to_join_path = '/'.join(pkg_relative_path.split('/')[2:])
         pkgfile = os.path.basename(pkg_relative_path)
         crippled_uri = EntropyTransceiver.get_uri_name(uri)
 
@@ -651,7 +650,7 @@ class Server(ServerNoticeBoardMixin):
                         darkgreen(crippled_uri),
                         brown(str(tries)),
                         blue(_("connecting to download package")),
-                        darkgreen(pkg_to_join_path),
+                        darkgreen(pkg_relative_path),
                     ),
                     importance = 1,
                     type = "info",
@@ -659,12 +658,12 @@ class Server(ServerNoticeBoardMixin):
                     back = True
                 )
 
-                remote_path = os.path.join(
-                    self.Entropy._get_remote_packages_relative_path(repo),
-                    pkg_to_join_path)
-                download_path = os.path.join(
-                    self.Entropy._get_local_packages_directory(repo),
-                    pkg_to_join_path)
+                remote_path = \
+                    self.Entropy.complete_remote_package_relative_path(
+                        pkg_relative_path, repo = repo)
+                download_path = self.Entropy.complete_local_package_path(
+                    pkg_relative_path, repo = repo)
+
                 download_dir = os.path.dirname(download_path)
 
                 self.Entropy.output(
@@ -692,7 +691,7 @@ class Server(ServerNoticeBoardMixin):
                             darkgreen(crippled_uri),
                             brown(str(tries)),
                             blue(_("package")),
-                            darkgreen(pkg_to_join_path),
+                            darkgreen(pkg_relative_path),
                             blue(_("does not exist")),
                         ),
                         importance = 1,
@@ -2184,20 +2183,22 @@ class Server(ServerNoticeBoardMixin):
             self.lock_mirrors(False, repo = repo)
         return 0, set(), set()
 
-
-    def _calculate_local_upload_files(self, branch, repo = None):
+    def _calculate_local_upload_files(self, repo = None):
         upload_files = 0
         upload_packages = set()
-        upload_dir = os.path.join(self.Entropy._get_local_upload_directory(repo),
-            branch)
+        upload_dir = self.Entropy._get_local_upload_directory(repo = repo)
 
         # check if it exists
         if not os.path.isdir(upload_dir):
             return upload_files, upload_packages
 
+        branch = self.SystemSettings['repositories']['branch']
+        upload_pkgs = self.Entropy._get_basedir_pkg_listing(upload_dir,
+            repo = repo, branch = branch)
+
         pkg_ext = etpConst['packagesext']
         pkg_md5_ext = etpConst['packagesmd5fileext']
-        for package in os.listdir(upload_dir):
+        for package in upload_pkgs:
             if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext):
                 upload_packages.add(package)
                 if package.endswith(pkg_ext):
@@ -2205,18 +2206,23 @@ class Server(ServerNoticeBoardMixin):
 
         return upload_files, upload_packages
 
-    def _calculate_local_package_files(self, branch, repo = None):
+    def _calculate_local_package_files(self, repo = None):
         local_files = 0
         local_packages = set()
-        packages_dir = os.path.join(
-            self.Entropy._get_local_packages_directory(repo), branch)
+        base_dir = self.Entropy._get_local_repository_base_directory(
+            repo = repo)
 
-        if not os.path.isdir(packages_dir):
-            os.makedirs(packages_dir)
+        # check if it exists
+        if not os.path.isdir(base_dir):
+            return local_files, local_packages
+
+        branch = self.SystemSettings['repositories']['branch']
+        pkg_files = self.Entropy._get_basedir_pkg_listing(base_dir,
+            repo = repo, branch = branch)
 
         pkg_ext = etpConst['packagesext']
         pkg_md5_ext = etpConst['packagesmd5fileext']
-        for package in os.listdir(packages_dir):
+        for package in pkg_files:
             if package.endswith(pkg_ext) or package.endswith(pkg_md5_ext):
                 local_packages.add(package)
                 if package.endswith(pkg_ext):
@@ -2257,12 +2263,13 @@ class Server(ServerNoticeBoardMixin):
             header = red(" @@ ")
         )
 
-    def _show_sync_queues(self, upload, download, removal, copy, metainfo,
-        branch):
+    def _show_sync_queues(self, upload, download, removal, copy, metainfo):
+
+        branch = self.SystemSettings['repositories']['branch']
 
         # show stats
-        for package, size in upload:
-            package = darkgreen(os.path.basename(package))
+        for package, rel_pkg, size in upload:
+            package = darkgreen(rel_pkg)
             size = blue(entropy.tools.bytes_into_human(size))
             self.Entropy.output(
                 "[branch:%s|%s] %s [%s]" % (
@@ -2275,8 +2282,8 @@ class Server(ServerNoticeBoardMixin):
                 type = "info",
                 header = red("    # ")
             )
-        for package, size in download:
-            package = darkred(os.path.basename(package))
+        for package, rel_pkg, size in download:
+            package = darkred(rel_pkg)
             size = blue(entropy.tools.bytes_into_human(size))
             self.Entropy.output(
                 "[branch:%s|%s] %s [%s]" % (
@@ -2289,8 +2296,8 @@ class Server(ServerNoticeBoardMixin):
                 type = "info",
                 header = red("    # ")
             )
-        for package, size in copy:
-            package = darkblue(os.path.basename(package))
+        for package, rel_pkg, size in copy:
+            package = darkblue(rel_pkg)
             size = blue(entropy.tools.bytes_into_human(size))
             self.Entropy.output(
                 "[branch:%s|%s] %s [%s]" % (
@@ -2303,8 +2310,8 @@ class Server(ServerNoticeBoardMixin):
                 type = "info",
                 header = red("    # ")
             )
-        for package, size in removal:
-            package = brown(os.path.basename(package))
+        for package, rel_pkg, size in removal:
+            package = brown(rel_pkg)
             size = blue(entropy.tools.bytes_into_human(size))
             self.Entropy.output(
                 "[branch:%s|%s] %s [%s]" % (
@@ -2377,40 +2384,50 @@ class Server(ServerNoticeBoardMixin):
             header = blue(" @@ ")
         )
 
-    def _calculate_remote_package_files(self, uri, branch, txc_handler,
-        repo = None):
-
-        remote_dir = os.path.join(
-            self.Entropy._get_remote_packages_relative_path(repo), branch)
-
-        # create path to lock file if it doesn't exist
-        if not txc_handler.is_dir(remote_dir):
-            txc_handler.makedirs(remote_dir)
-
-        remote_packages_info = txc_handler.list_content_metadata(remote_dir)
-        remote_packages = [x[0] for x in remote_packages_info]
+    def _calculate_remote_package_files(self, uri, txc_handler, repo = None):
 
         remote_files = 0
-        for pkg in remote_packages:
-            if pkg.endswith(etpConst['packagesext']):
-                remote_files += 1
-
         remote_packages_data = {}
-        for pkg in remote_packages_info:
-            remote_packages_data[pkg[0]] = int(pkg[1])
+        remote_packages = []
+        branch = self.SystemSettings['repositories']['branch']
+
+        pkgs_dir_types = self.Entropy._get_pkg_dir_names()
+        for pkg_dir_type in pkgs_dir_types:
+
+            remote_dir = self.Entropy.complete_remote_package_relative_path(
+                pkg_dir_type, repo = repo)
+            remote_dir = os.path.join(remote_dir, etpConst['currentarch'],
+                branch)
+            only_dir = self.Entropy.complete_remote_package_relative_path("",
+                repo = repo)
+            db_url_dir = remote_dir[len(only_dir):]
+
+            # create path to lock file if it doesn't exist
+            if not txc_handler.is_dir(remote_dir):
+                txc_handler.makedirs(remote_dir)
+
+            remote_packages_info = txc_handler.list_content_metadata(remote_dir)
+            remote_packages += [os.path.join(db_url_dir, x[0]) for x \
+                in remote_packages_info]
+
+            for pkg in remote_packages:
+                if pkg.endswith(etpConst['packagesext']):
+                    remote_files += 1
+
+            my_remote_pkg_data = dict((os.path.join(db_url_dir, x[0]),
+                int(x[1])) for x in remote_packages_info)
+            remote_packages_data.update(my_remote_pkg_data)
 
         return remote_files, remote_packages, remote_packages_data
 
-    def calculate_packages_to_sync(self, uri, branch, repo = None):
+    def calculate_packages_to_sync(self, uri, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
 
         crippled_uri = EntropyTransceiver.get_uri_name(uri)
-        upload_files, upload_packages = self._calculate_local_upload_files(
-            branch, repo)
-        local_files, local_packages = self._calculate_local_package_files(
-            branch, repo)
+        upload_files, upload_packages = self._calculate_local_upload_files(repo)
+        local_files, local_packages = self._calculate_local_package_files(repo)
         self._show_local_sync_stats(upload_files, local_files)
 
         self.Entropy.output(
@@ -2423,8 +2440,7 @@ class Server(ServerNoticeBoardMixin):
         txc = self.Entropy.Transceiver(uri)
         with txc as handler:
             remote_files, remote_packages, remote_packages_data = \
-                self._calculate_remote_package_files(uri, branch, handler,
-                    repo = repo)
+                self._calculate_remote_package_files(uri, handler, repo = repo)
 
         self.Entropy.output(
             "%s:\t\t\t%s %s" % (
@@ -2449,35 +2465,37 @@ class Server(ServerNoticeBoardMixin):
 
         upload_queue, download_queue, removal_queue, fine_queue = \
             self._calculate_sync_queues(upload_packages, local_packages,
-                remote_packages, remote_packages_data, branch, repo)
+                remote_packages, remote_packages_data, repo)
         return upload_queue, download_queue, removal_queue, fine_queue, \
             remote_packages_data
 
     def _calculate_sync_queues(self, upload_packages, local_packages,
-        remote_packages, remote_packages_data, branch, repo = None):
+        remote_packages, remote_packages_data, repo = None):
 
         upload_queue = set()
         download_queue = set()
         removal_queue = set()
         fine_queue = set()
+        branch = self.SystemSettings['repositories']['branch']
 
         for local_package in upload_packages:
             if local_package in remote_packages:
 
-                local_filepath = os.path.join(
-                    self.Entropy._get_local_upload_directory(repo), branch,
-                    local_package)
+                local_filepath = \
+                    self.Entropy.complete_local_upload_package_path(
+                        local_package, repo = repo)
 
                 local_size = entropy.tools.get_file_size(local_filepath)
                 remote_size = remote_packages_data.get(local_package)
                 if remote_size is None:
                     remote_size = 0
-                if (local_size != remote_size):
+                if local_size != remote_size:
                     # size does not match, adding to the upload queue
                     upload_queue.add(local_package)
                 else:
                     # just move from upload to packages
                     fine_queue.add(local_package)
+
             else:
                 # always force upload of packages in uploaddir
                 upload_queue.add(local_package)
@@ -2486,9 +2504,8 @@ class Server(ServerNoticeBoardMixin):
         # we have to upload it we have local_packages and remote_packages
         for local_package in local_packages:
             if local_package in remote_packages:
-                local_filepath = os.path.join(
-                    self.Entropy._get_local_packages_directory(repo), branch,
-                    local_package)
+                local_filepath = self.Entropy.complete_local_package_path(
+                    local_package, repo = repo)
                 local_size = entropy.tools.get_file_size(local_filepath)
                 remote_size = remote_packages_data.get(local_package)
                 if remote_size is None:
@@ -2505,9 +2522,8 @@ class Server(ServerNoticeBoardMixin):
         # Fill download_queue and removal_queue
         for remote_package in remote_packages:
             if remote_package in local_packages:
-                local_filepath = os.path.join(
-                    self.Entropy._get_local_packages_directory(repo), branch,
-                    remote_package)
+                local_filepath = self.Entropy.complete_local_package_path(
+                    remote_package, repo = repo)
                 local_size = entropy.tools.get_file_size(local_filepath)
                 remote_size = remote_packages_data.get(remote_package)
                 if remote_size is None:
@@ -2561,9 +2577,8 @@ class Server(ServerNoticeBoardMixin):
 
         return upload_queue, download_queue, removal_queue, fine_queue
 
-
     def _expand_queues(self, upload_queue, download_queue, removal_queue,
-        remote_packages_data, branch, repo):
+        remote_packages_data, repo):
 
         metainfo = {
             'removal': 0,
@@ -2578,59 +2593,62 @@ class Server(ServerNoticeBoardMixin):
         for item in removal_queue:
             if not item.endswith(etpConst['packagesext']):
                 continue
-            local_filepath = os.path.join(
-                self.Entropy._get_local_packages_directory(repo), branch, item)
+            local_filepath = self.Entropy.complete_local_package_path(
+                item, repo = repo)
             size = entropy.tools.get_file_size(local_filepath)
             metainfo['removal'] += size
-            removal.append((local_filepath, size))
+            removal.append((local_filepath, item, size))
 
         for item in download_queue:
             if not item.endswith(etpConst['packagesext']):
                 continue
-            local_filepath = os.path.join(
-                self.Entropy._get_local_upload_directory(repo), branch, item)
+
+            local_filepath = self.Entropy.complete_local_upload_package_path(
+                item, repo = repo)
             if not os.path.isfile(local_filepath):
                 size = remote_packages_data.get(item)
                 if size is None:
                     size = 0
                 size = int(size)
                 metainfo['removal'] += size
-                download.append((local_filepath, size))
+                download.append((local_filepath, item, size))
             else:
                 size = entropy.tools.get_file_size(local_filepath)
-                do_copy.append((local_filepath, size))
+                do_copy.append((local_filepath, item, size))
 
         for item in upload_queue:
             if not item.endswith(etpConst['packagesext']):
                 continue
-            local_filepath = os.path.join(
-                self.Entropy._get_local_upload_directory(repo), branch, item)
-            local_filepath_pkgs = os.path.join(
-                self.Entropy._get_local_packages_directory(repo), branch, item)
+
+            local_filepath = self.Entropy.complete_local_upload_package_path(
+                item, repo = repo)
+
+            local_filepath_pkgs = self.Entropy.complete_local_package_path(
+                item, repo = repo)
             if os.path.isfile(local_filepath):
                 size = entropy.tools.get_file_size(local_filepath)
-                upload.append((local_filepath, size))
+                upload.append((local_filepath, item, size))
             else:
                 size = entropy.tools.get_file_size(local_filepath_pkgs)
-                upload.append((local_filepath_pkgs, size))
+                upload.append((local_filepath_pkgs, item, size))
             metainfo['upload'] += size
 
         return upload, download, removal, do_copy, metainfo
 
-
-    def _sync_run_removal_queue(self, removal_queue, branch, repo = None):
+    def _sync_run_removal_queue(self, removal_queue, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
+        branch = self.SystemSettings['repositories']['branch']
 
-        for itemdata in removal_queue:
+        for remove_filepath, rel_path, size in removal_queue:
 
-            remove_filename = itemdata[0]
-            remove_filepath = os.path.join(
-                self.Entropy._get_local_packages_directory(repo), branch,
-                remove_filename)
+            remove_filename = os.path.basename(remove_filepath)
             remove_filepath_hash = remove_filepath + \
                 etpConst['packagesmd5fileext']
+            remove_filepath_exp = remove_filepath + \
+                etpConst['packagesexpirationfileext']
+
             self.Entropy.output(
                 "[repo:%s|%s|%s] %s: %s [%s]" % (
                     brown(repo),
@@ -2638,7 +2656,7 @@ class Server(ServerNoticeBoardMixin):
                     brown(branch),
                     blue(_("removing package+hash")),
                     darkgreen(remove_filename),
-                    blue(entropy.tools.bytes_into_human(itemdata[1])),
+                    blue(entropy.tools.bytes_into_human(size)),
                 ),
                 importance = 0,
                 type = "info",
@@ -2649,6 +2667,8 @@ class Server(ServerNoticeBoardMixin):
                 os.remove(remove_filepath)
             if os.path.isfile(remove_filepath_hash):
                 os.remove(remove_filepath_hash)
+            if os.path.isfile(remove_filepath_exp):
+                os.remove(remove_filepath_exp)
 
         self.Entropy.output(
             "[repo:%s|%s|%s] %s" % (
@@ -2663,20 +2683,19 @@ class Server(ServerNoticeBoardMixin):
         )
 
 
-    def _sync_run_copy_queue(self, copy_queue, branch, repo = None):
+    def _sync_run_copy_queue(self, copy_queue, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
+        branch = self.SystemSettings['repositories']['branch']
 
-        for itemdata in copy_queue:
-
-            from_file = itemdata[0]
+        for from_file, rel_file, size in copy_queue:
             from_file_hash = from_file + etpConst['packagesmd5fileext']
-            to_file = os.path.join(
-                self.Entropy._get_local_packages_directory(repo), branch,
-                os.path.basename(from_file))
+
+            to_file = self.Entropy.complete_local_package_path(rel_file)
             to_file_hash = to_file+etpConst['packagesmd5fileext']
             expiration_file = to_file+etpConst['packagesexpirationfileext']
+
             self.Entropy.output(
                 "[repo:%s|%s|%s] %s: %s" % (
                     brown(repo),
@@ -2689,13 +2708,10 @@ class Server(ServerNoticeBoardMixin):
                 type = "info",
                 header = darkred(" * ")
             )
-
-            if not os.path.isdir(os.path.dirname(to_file)):
-                os.makedirs(os.path.dirname(to_file))
+            self.Entropy._ensure_dir_path(os.path.dirname(to_file))
 
             shutil.copy2(from_file, to_file)
-            if not os.path.isfile(from_file_hash):
-                self._create_file_checksum(from_file, from_file_hash)
+            self._create_file_checksum(from_file, from_file_hash)
             shutil.copy2(from_file_hash, to_file_hash)
 
             # clear expiration file
@@ -2703,30 +2719,46 @@ class Server(ServerNoticeBoardMixin):
                 os.remove(expiration_file)
 
 
-    def _sync_run_upload_queue(self, uri, upload_queue, branch, repo = None):
+    def _sync_run_upload_queue(self, uri, upload_queue, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
+        branch = self.SystemSettings['repositories']['branch']
 
         crippled_uri = EntropyTransceiver.get_uri_name(uri)
-        myqueue = []
-        for itemdata in upload_queue:
-            upload_item = itemdata[0]
-            hash_file = upload_item + etpConst['packagesmd5fileext']
+        queue_map = {}
+
+        for upload_path, rel_path, size in upload_queue:
+
+            hash_file = upload_path + etpConst['packagesmd5fileext']
             if not os.path.isfile(hash_file):
-                entropy.tools.create_md5_file(upload_item)
-            myqueue.append(hash_file)
-            myqueue.append(upload_item)
+                entropy.tools.create_md5_file(upload_path)
 
-        remote_dir = os.path.join(
-            self.Entropy._get_remote_packages_relative_path(repo), branch)
+            rel_dir = os.path.dirname(rel_path)
+            obj = queue_map.setdefault(rel_dir, [])
+            obj.append(hash_file)
+            obj.append(upload_path)
 
-        uploader = self.TransceiverServerHandler(self.Entropy, [uri],
-            myqueue, critical_files = myqueue,
-            txc_basedir = remote_dir,
-            handlers_data = {'branch': branch }, repo = repo)
+        erorrs = False
+        m_fine_uris = set()
+        m_broken_uris = set()
+        for rel_path, myqueue in queue_map.items():
+
+            remote_dir = os.path.dirname(
+                self.Entropy.complete_remote_package_relative_path(
+                    rel_path, repo = repo))
+
+            uploader = self.TransceiverServerHandler(self.Entropy, [uri],
+                myqueue, critical_files = myqueue,
+                txc_basedir = remote_dir,
+                handlers_data = {'branch': branch }, repo = repo)
+
+            xerrors, xm_fine_uris, xm_broken_uris = uploader.go()
+            if xerrors:
+                errors = True
+            m_fine_uris.update(xm_fine_uris)
+            m_broken_uris.update(xm_broken_uris)
 
-        errors, m_fine_uris, m_broken_uris = uploader.go()
         if errors:
             my_broken_uris = [
                 (EntropyTransceiver.get_uri_name(x[0]), x[1]) for \
@@ -2759,30 +2791,50 @@ class Server(ServerNoticeBoardMixin):
         return errors, m_fine_uris, m_broken_uris
 
 
-    def _sync_run_download_queue(self, uri, download_queue, branch,
-            repo = None):
+    def _sync_run_download_queue(self, uri, download_queue, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
+        branch = self.SystemSettings['repositories']['branch']
 
         crippled_uri = EntropyTransceiver.get_uri_name(uri)
-        myqueue = []
-        for package in download_queue:
-            hash_file = package + etpConst['packagesmd5fileext']
-            myqueue.append(package)
-            myqueue.append(hash_file)
 
-        remote_dir = os.path.join(
-            self.Entropy._get_remote_packages_relative_path(repo), branch)
-        local_basedir = os.path.join(
-            self.Entropy._get_local_packages_directory(repo), branch)
-        downloader = self.TransceiverServerHandler(
-            self.Entropy, [uri], myqueue,
-            critical_files = myqueue,
-            txc_basedir = remote_dir, local_basedir = local_basedir,
-            handlers_data = {'branch': branch }, download = True, repo = repo)
+        queue_map = {}
+
+        for download_path, rel_path, size in download_queue:
+            hash_file = download_path + etpConst['packagesmd5fileext']
+
+            rel_dir = os.path.dirname(rel_path)
+            obj = queue_map.setdefault(rel_dir, [])
+            obj.append(hash_file)
+            obj.append(download_path)
+
+        erorrs = False
+        m_fine_uris = set()
+        m_broken_uris = set()
+        for rel_path, myqueue in queue_map.items():
+
+            remote_dir = os.path.dirname(
+                self.Entropy.complete_remote_package_relative_path(
+                    rel_path, repo = repo))
+
+            local_basedir = os.path.dirname(
+                self.Entropy.complete_local_package_path(rel_path,
+                    repo = repo))
+
+            downloader = self.TransceiverServerHandler(
+                self.Entropy, [uri], myqueue,
+                critical_files = myqueue,
+                txc_basedir = remote_dir, local_basedir = local_basedir,
+                handlers_data = {'branch': branch }, download = True,
+                repo = repo)
+
+            xerrors, xm_fine_uris, xm_broken_uris = downloader.go()
+            if xerrors:
+                errors = True
+            m_fine_uris.update(xm_fine_uris)
+            m_broken_uris.update(xm_broken_uris)
 
-        errors, m_fine_uris, m_broken_uris = downloader.go()
         if errors:
             my_broken_uris = [
                 (EntropyTransceiver.get_uri_name(x), y,) \
@@ -2915,7 +2967,7 @@ class Server(ServerNoticeBoardMixin):
             try:
                 upload_queue, download_queue, removal_queue, fine_queue, \
                     remote_packages_data = self.calculate_packages_to_sync(uri,
-                        self.SystemSettings['repositories']['branch'], repo)
+                        repo)
             except self.socket.error as err:
                 self.Entropy.output(
                     "[repo:%s|%s|branch:%s] %s: %s, %s %s" % (
@@ -2957,20 +3009,14 @@ class Server(ServerNoticeBoardMixin):
                 header = red(" ** ")
             )
 
-            upload, download, removal, copy, metainfo = self._expand_queues(
-                        upload_queue,
-                        download_queue,
-                        removal_queue,
-                        remote_packages_data,
-                        self.SystemSettings['repositories']['branch'],
-                        repo
-            )
+            upload, download, removal, copy_q, metainfo = self._expand_queues(
+                upload_queue, download_queue, removal_queue,
+                remote_packages_data, repo)
             del upload_queue, download_queue, removal_queue, \
                 remote_packages_data
-            self._show_sync_queues(upload, download, removal, copy, metainfo,
-                self.SystemSettings['repositories']['branch'])
+            self._show_sync_queues(upload, download, removal, copy_q, metainfo)
 
-            if not len(upload)+len(download)+len(removal)+len(copy):
+            if not len(upload)+len(download)+len(removal)+len(copy_q):
 
                 self.Entropy.output(
                     "[repo:%s|%s|branch:%s] %s %s" % (
@@ -3009,31 +3055,26 @@ class Server(ServerNoticeBoardMixin):
                     repo = repo)
 
                 if removal:
-                    self._sync_run_removal_queue(removal,
-                        self.SystemSettings['repositories']['branch'], repo)
+                    self._sync_run_removal_queue(removal, repo)
 
-                if copy:
-                    self._sync_run_copy_queue(copy,
-                        self.SystemSettings['repositories']['branch'], repo)
+                if copy_q:
+                    self._sync_run_copy_queue(copy_q, repo)
 
                 if upload or download:
                     mirrors_tainted = True
 
                 if upload:
                     d_errors, m_fine_uris, \
-                        m_broken_uris = self._sync_run_upload_queue(
-                            uri, upload,
-                            self.SystemSettings['repositories']['branch'], repo)
+                        m_broken_uris = self._sync_run_upload_queue(uri,
+                            upload, repo)
 
                     if d_errors:
                         mirror_errors = True
 
                 if download:
-                    my_downlist = [x[0] for x in download]
                     d_errors, m_fine_uris, \
-                        m_broken_uris = self._sync_run_download_queue(
-                            uri, my_downlist,
-                            self.SystemSettings['repositories']['branch'], repo)
+                        m_broken_uris = self._sync_run_download_queue(uri,
+                            download, repo)
 
                     if d_errors:
                         mirror_errors = True
@@ -3126,8 +3167,7 @@ class Server(ServerNoticeBoardMixin):
 
         # if at least one server has been synced successfully, move files
         if (len(successfull_mirrors) > 0) and not pretend:
-            self._remove_expiration_files(
-                self.SystemSettings['repositories']['branch'], repo)
+            self._move_files_over_from_upload(repo)
 
         if packages_check:
             check_data = self.Entropy.verify_local_packages([], ask = ask,
@@ -3136,39 +3176,38 @@ class Server(ServerNoticeBoardMixin):
         return mirrors_tainted, mirrors_errors, successfull_mirrors, \
             broken_mirrors, check_data
 
-    def _remove_expiration_files(self, branch, repo = None):
+    def _move_files_over_from_upload(self, repo = None):
 
         if repo is None:
             repo = self.Entropy.default_repository
 
-        branch_dir = os.path.join(
-            self.Entropy._get_local_upload_directory(repo), branch)
+        upload_dir = self.Entropy._get_local_upload_directory(repo = repo)
+        basedir_list = self.Entropy._get_basedir_pkg_listing(upload_dir,
+            repo = repo)
 
-        # check if it exists
-        if not os.path.isdir(branch_dir):
-            return None
+        for pkg_rel in basedir_list:
+
+            source_pkg = self.Entropy.complete_local_upload_package_path(
+                pkg_rel, repo = repo)
+            dest_pkg = self.Entropy.complete_local_package_path(pkg_rel,
+                repo = repo)
+            dest_pkg_dir = os.path.dirname(dest_pkg)
+            self.Entropy._ensure_dir_path(dest_pkg_dir)
+
+            try:
+                os.rename(source_pkg, dest_pkg)
+            except OSError: # on different hard drives?
+                shutil.move(source_pkg, dest_pkg)
 
-        branchcontent = os.listdir(branch_dir)
-        for xfile in branchcontent:
-            source = os.path.join(self.Entropy._get_local_upload_directory(repo),
-                branch, xfile)
-            destdir = os.path.join(
-                self.Entropy._get_local_packages_directory(repo), branch)
-            if not os.path.isdir(destdir):
-                os.makedirs(destdir)
-            dest = os.path.join(destdir, xfile)
-            shutil.move(source, dest)
             # clear expiration file
-            dest_expiration = dest + etpConst['packagesexpirationfileext']
+            dest_expiration = dest_pkg + etpConst['packagesexpirationfileext']
             if os.path.isfile(dest_expiration):
                 os.remove(dest_expiration)
 
+    def _is_package_expired(self, package_rel, repo = None):
 
-    def _is_package_expired(self, package_file, branch, repo = None):
-
-        pkg_path = os.path.join(
-            self.Entropy._get_local_packages_directory(repo), branch,
-            package_file)
+        pkg_path = self.Entropy.complete_local_package_path(package_rel,
+            repo = repo)
         pkg_path += etpConst['packagesexpirationfileext']
         if not os.path.isfile(pkg_path):
             return False
@@ -3184,12 +3223,10 @@ class Server(ServerNoticeBoardMixin):
             return True
         return False
 
-    def _create_expiration_file(self, package_file, branch, repo = None,
-        gentle = False):
+    def _create_expiration_file(self, package_rel, repo = None, gentle = False):
 
-        pkg_path = os.path.join(
-            self.Entropy._get_local_packages_directory(repo), branch,
-            package_file)
+        pkg_path = self.Entropy.complete_local_package_path(package_rel,
+            repo = repo)
         pkg_path += etpConst['packagesexpirationfileext']
         if gentle and os.path.isfile(pkg_path):
             return
@@ -3197,25 +3234,24 @@ class Server(ServerNoticeBoardMixin):
         f_exp.flush()
         f_exp.close()
 
-
     def _collect_expiring_packages(self, branch, repo = None):
 
         dbconn = self.Entropy.open_server_repository(just_reading = True,
             repo = repo)
-        database_bins = dbconn.listAllDownloads(do_sort = False,
-            full_path = True)
-        bins_dir = os.path.join(
-            self.Entropy._get_local_packages_directory(repo), branch)
-        repo_bins = set()
 
-        if os.path.isdir(bins_dir):
-            repo_bins = os.listdir(bins_dir)
-            repo_bins = set([
-                os.path.join('packages', etpSys['arch'], branch, x) for x \
-                    in repo_bins if x.endswith(etpConst['packagesext'])])
+        database_bins = set(dbconn.listAllDownloads(do_sort = False,
+            full_path = True))
+
+        repo_basedir = self.Entropy._get_local_repository_base_directory(
+            repo = repo)
+
+        repo_bins = self.Entropy._get_basedir_pkg_listing(repo_basedir,
+            repo = repo, branch = branch)
+
+        # convert to set, so that we can do fast thingszzsd
+        repo_bins = set(repo_bins)
         repo_bins -= database_bins
-
-        return set([os.path.basename(x) for x in repo_bins])
+        return repo_bins
 
 
     def tidy_mirrors(self, ask = True, pretend = False, repo = None):
@@ -3266,29 +3302,17 @@ class Server(ServerNoticeBoardMixin):
             for key, val in branch_pkglist_data.items():
                 branch_pkglist_data[key] = val.split("\n")
 
-
-            rel_path_basedir = etpConst['packagesrelativepath_basedir']
-            rel_path_basename = etpConst['packagesrelativepath_basename']
-            remote_relpath = rel_path_basedir + "/" + rel_path_basename + "/" \
-                + branch
-            my_expiring_pkgs = set([os.path.join(remote_relpath, x) for x in \
-                expiring_packages])
-
             for other_branch in branch_pkglist_data:
                 branch_pkglist = set(branch_pkglist_data[other_branch])
-                my_expiring_pkgs -= branch_pkglist
-
-            # fallback to normality, set new expiring packages var
-            expiring_packages = [os.path.basename(x) for x in my_expiring_pkgs]
+                expiring_packages -= branch_pkglist
 
         removal = []
-        for package in expiring_packages:
-            expired = self._is_package_expired(package, branch, repo)
+        for package_rel in expiring_packages:
+            expired = self._is_package_expired(package_rel, repo)
             if expired:
-                removal.append(package)
+                removal.append(package_rel)
             else:
-                self._create_expiration_file(package, branch, repo,
-                    gentle = True)
+                self._create_expiration_file(package_rel, repo, gentle = True)
 
         # fill returning data
         branch_data['removal'] = removal[:]
@@ -3335,72 +3359,87 @@ class Server(ServerNoticeBoardMixin):
             if rc_question == _("No"):
                 return errors, branch_data
 
-        myqueue = []
-        for package in removal:
-            myqueue.append(package+etpConst['packagesmd5fileext'])
-            myqueue.append(package)
+        # split queue by remote directories to work on
+        removal_map = {}
+        for package_rel in removal:
+            rel_path = self.Entropy.complete_remote_package_rel_path(
+                package_rel, repo = repo)
+            rel_dir = os.path.dirname(rel_path)
+            obj = removal_map.setdefault(rel_dir, [])
+            base_pkg = os.path.basename(package)
+            obj.append(base_pkg)
+            obj.append(base_pkg+etpConst['packagesmd5fileext'])
 
-        remote_dir = os.path.join(
-            self.Entropy._get_remote_packages_relative_path(repo), branch)
         for uri in self.Entropy.get_remote_mirrors(repo):
 
-            self.Entropy.output(
-                "[branch:%s] %s..." % (
-                    brown(branch),
-                    blue(_("removing packages remotely")),
-                ),
-                importance = 1,
-                type = "info",
-                header = blue(" @@ ")
-            )
+            ##
+            # remove remotely
+            ##
 
-            crippled_uri = EntropyTransceiver.get_uri_name(uri)
-            destroyer = self.TransceiverServerHandler(
-                self.Entropy,
-                [uri],
-                myqueue,
-                critical_files = [],
-                txc_basedir = remote_dir,
-                remove = True,
-                repo = repo
-            )
-            errors, m_fine_uris, m_broken_uris = destroyer.go()
-            if errors:
-                my_broken_uris = [
-                    (EntropyTransceiver.get_uri_name(x[0]), x[1]) \
-                        for x in m_broken_uris]
+            for remote_dir, myqueue in removal_map.items():
 
-                reason = my_broken_uris[0][1]
                 self.Entropy.output(
-                    "[branch:%s] %s: %s, %s: %s" % (
+                    "[branch:%s] %s..." % (
                         brown(branch),
-                        blue(_("remove errors")),
-                        red(crippled_uri),
-                        blue(_("reason")),
-                        reason,
+                        blue(_("removing packages remotely")),
                     ),
                     importance = 1,
-                    type = "warning",
-                    header = brown(" !!! ")
+                    type = "info",
+                    header = blue(" @@ ")
                 )
-                branch_data['errors'] = True
-                errors = True
 
-            self.Entropy.output(
-                "[branch:%s] %s..." % (
-                    brown(branch),
-                    blue(_("removing packages locally")),
-                ),
-                importance = 1,
-                type = "info",
-                header = blue(" @@ ")
-            )
+                crippled_uri = EntropyTransceiver.get_uri_name(uri)
+                destroyer = self.TransceiverServerHandler(
+                    self.Entropy,
+                    [uri],
+                    myqueue,
+                    critical_files = [],
+                    txc_basedir = remote_dir,
+                    remove = True,
+                    repo = repo
+                )
+                errors, m_fine_uris, m_broken_uris = destroyer.go()
+                if errors:
+                    my_broken_uris = [
+                        (EntropyTransceiver.get_uri_name(x[0]), x[1]) \
+                            for x in m_broken_uris]
+
+                    reason = my_broken_uris[0][1]
+                    self.Entropy.output(
+                        "[branch:%s] %s: %s, %s: %s" % (
+                            brown(branch),
+                            blue(_("remove errors")),
+                            red(crippled_uri),
+                            blue(_("reason")),
+                            reason,
+                        ),
+                        importance = 1,
+                        type = "warning",
+                        header = brown(" !!! ")
+                    )
+                    branch_data['errors'] = True
+                    errors = True
+
+                self.Entropy.output(
+                    "[branch:%s] %s..." % (
+                        brown(branch),
+                        blue(_("removing packages locally")),
+                    ),
+                    importance = 1,
+                    type = "info",
+                    header = blue(" @@ ")
+                )
+
+            ##
+            # remove locally
+            ##
 
             branch_data['removed'] = set()
-            for package in removal:
-                package_path = os.path.join(
-                    self.Entropy._get_local_packages_directory(repo),
-                    branch, package)
+            for package_rel in removal:
+
+                package_path = self.Entropy.complete_local_package_path(
+                    package_rel, repo = repo)
+
                 package_path_hash = package_path + \
                     etpConst['packagesmd5fileext']
                 package_path_expired = package_path + \
diff --git a/libraries/entropy/services/system/executors.py b/libraries/entropy/services/system/executors.py
index 2e727a9ed..6c99d6609 100644
--- a/libraries/entropy/services/system/executors.py
+++ b/libraries/entropy/services/system/executors.py
@@ -808,8 +808,7 @@ class Base:
                         try:
                             upload_queue, download_queue, removal_queue, \
                                 fine_queue, remote_packages_data = Entropy.Mirrors.calculate_packages_to_sync(
-                                    uri, Entropy.SystemSettings['repositories']['branch'],
-                                    repoid)
+                                    uri, repoid)
                         except socket.error:
                             entropy.tools.print_traceback(f = stdout_err)
                             stdout_err.write("\n"+_("Socket error, continuing...").encode('utf-8')+"\n")
diff --git a/server/reagent.py b/server/reagent.py
index 726ca2396..1ba83b27b 100644
--- a/server/reagent.py
+++ b/server/reagent.py
@@ -73,7 +73,6 @@ help_opts = [
     None,
     (1, 'database', 2, _('repository database functions')),
         (2, '--initialize', 3, _('(re)initialize the current repository database')),
-            (3, '--empty', 2, _('do not refill database using packages on mirrors')),
             (3, '--repo=<repo>', 2, _('(re)create the database for the specified repository')),
         (2, 'bump', 4, _('manually force a revision bump for the current repository database')),
             (3, '--sync', 3, _('synchronize the database')),
diff --git a/server/server_reagent.py b/server/server_reagent.py
index 812c1a1a7..e3afc71d3 100644
--- a/server/server_reagent.py
+++ b/server/server_reagent.py
@@ -594,7 +594,6 @@ def database(options):
 
     d_request_noask = False
     d_request_sync = False
-    d_request_empty = False
     repo = None
     _options = []
     for opt in options:
@@ -602,11 +601,8 @@ def database(options):
             d_request_noask = True
         elif opt.startswith("--sync"):
             d_request_sync = True
-        elif opt.startswith("--empty"):
-            d_request_empty = True
         elif opt.startswith("--repo=") and len(opt.split("=")) == 2:
             repo = opt.split("=")[1]
-            d_request_empty = True
         else:
             _options.append(opt)
     options = _options
@@ -617,8 +613,7 @@ def database(options):
 
     if (options[0] == "--initialize"):
 
-        rc = Entropy.initialize_server_repository(empty = d_request_empty,
-            repo = repo)
+        rc = Entropy.initialize_server_repository(repo = repo)
         if rc == 0:
             print_info(darkgreen(" * ")+red(_("Entropy database has been reinitialized using binary packages available")))