[services] add entropy package deltas generator tool

services/entropy-pkgdelta-generator

@@ -0,0 +1,160 @@
+#!/usr/bin/python2 -O
+# -*- coding: utf-8 -*-
+
+import os
+import errno
+import sys
+sys.path.insert(0, "../libraries")
+import tempfile
+import subprocess
+import bz2
+import gzip
+
+from entropy.const import etpConst
+import entropy.dep
+import entropy.tools
+
+
+BSPATCH_EXEC = "/usr/bin/bspatch"
+
+def generate_pkg_map(packages_directory):
+    """
+    
+    """
+    pkg_map = {}
+    for pkg_file in os.listdir(packages_directory):
+        if not pkg_file.endswith(etpConst['packagesext']):
+            continue
+        cat, name, ver, tag, rev = entropy.dep.exploit_package_filename(
+            pkg_file)
+        obj = pkg_map.setdefault((cat, name), set())
+        obj.add((ver, tag, rev, pkg_file))
+    return pkg_map
+
+def sort_packages(pkg_map_items):
+    """
+    Sort packages by version, tag, revision and return a sort map (dict) and
+    a sorted list of them (list)
+    """
+    def _generate_from_to(cat_name_map, sorted_pkg_list):
+        for pkg_idx in range(len(sorted_pkg_list)):
+            pkg_key = sorted_pkg_list[pkg_idx]
+            next_pkgs = set(sorted_pkg_list[pkg_idx:])
+            next_pkgs.discard(pkg_key)
+            sorted_next = sorted(next_pkgs, key = lambda x: cat_name_map[x])
+            for next_pkg_key in sorted_next:
+                yield (cat_name_map[pkg_key], cat_name_map[next_pkg_key])
+
+    cat_name_map = dict((((ver, tag, rev), pkg_path) \
+        for ver, tag, rev, pkg_path in pkg_map_items))
+    sorted_pkgs = entropy.dep.get_entropy_newer_version(list(cat_name_map))
+    sorted_pkgs.reverse()
+    return _generate_from_to(cat_name_map, sorted_pkgs)
+
+
+
+def generate_package_deltas(directory):
+    """
+    Generate Entropy package delta files.
+    """
+    for (cat, name), items in generate_pkg_map(directory).items():
+        # sort items, then generate deltas in one direction only
+        sorted_pkgs_couples = sort_packages(items)
+        for from_pkg_name, to_pkg_name in sorted_pkgs_couples:
+            pkg_path_a = os.path.join(directory, from_pkg_name)
+            next_pkg_path = os.path.join(directory, to_pkg_name)
+            delta_fn = entropy.tools.generate_entropy_delta_file_name(
+                pkg_path_a, next_pkg_path)
+            delta_path = os.path.join(directory,
+                etpConst['packagesdeltasubdir'], delta_fn)
+            if os.path.lexists(delta_path):
+                sys.stderr.write(delta_path + " already exists\n")
+                continue
+            delta_file = entropy.tools.generate_entropy_delta(pkg_path_a,
+                next_pkg_path)
+            if delta_file is not None:
+                sys.stdout.write(delta_file + "\n")
+
+def cleanup_package_deltas(directory):
+    """
+    Cleanup old Entropy package delta files.
+    """
+    def _list_delta_packages(delta_dir):
+        return set([os.path.join(delta_dir, x) for x in os.listdir(delta_dir) \
+            if x.endswith(etpConst['packagesdeltaext'])])
+
+    avail_deltas = _list_delta_packages(os.path.join(directory,
+        etpConst['packagesdeltasubdir']))
+    required_deltas = set()
+    for (cat, name), items in generate_pkg_map(directory).items():
+        # sort items, then generate deltas in one direction only
+        sorted_pkgs_couples = sort_packages(items)
+        for from_pkg_name, to_pkg_name in sorted_pkgs_couples:
+            pkg_path_a = os.path.join(directory, from_pkg_name)
+            next_pkg_path = os.path.join(directory, to_pkg_name)
+            delta_fn = entropy.tools.generate_entropy_delta_file_name(
+                pkg_path_a, next_pkg_path)
+            delta_path = os.path.join(directory,
+                etpConst['packagesdeltasubdir'], delta_fn)
+            if os.path.lexists(delta_path):
+                required_deltas.add(delta_path)
+
+    to_remove_deltas = avail_deltas - required_deltas
+    rc = 0
+    if not to_remove_deltas:
+        sys.stdout.write("nothing to remove for %s\n" % (directory,))
+    for old_pkg_delta in to_remove_deltas:
+        try:
+            os.remove(old_pkg_delta)
+            sys.stdout.write(old_pkg_delta + " removed\n")
+        except OSError as err:
+            sys.stderr.write("cannot remove %s: %s\n" % (old_pkg_delta,
+                err))
+            rc = 1
+    return rc
+
+def _generator_argv(argv):
+    for directory in argv:
+        if os.path.isdir(directory):
+            generate_package_deltas(directory)
+    return 0
+
+def _cleanup_argv(argv):
+    rc = 1
+    for directory in argv:
+        if os.path.isdir(directory):
+            rc = cleanup_package_deltas(directory)
+    return rc
+
+_cmds_map = {
+    'generate': _generator_argv,
+    'cleanup': _cleanup_argv,
+}
+
+def _opts_parser(args):
+    if not args:
+        return None, []
+    cmd, argv = args[0], args[1:]
+    if not argv:
+        return None, []
+    func = _cmds_map.get(cmd)
+    if func is None:
+        return None, []
+    return func, argv
+
+def _print_help():
+    sys.stdout.write(
+        "entropy-pkgdelta-generator <command> <pkgdir> [... <pkgdir> ...]\n\n")
+    sys.stdout.write("available commands:\n")
+    sys.stdout.write("\tgenerate\tgenerate pkgdelta files for given package directories\n")
+    sys.stdout.write("\tcleanup\t\tclean pkgdelta files for unavailable packages\n\n")
+
+
+if __name__ == "__main__":
+    func, argv = _opts_parser(sys.argv[1:])
+    if func is not None:
+        rc = func(argv)
+    else:
+        _print_help()
+        rc = 1
+    raise SystemExit(rc)