Fabio Erculiani
ab1e111502
Add --lock <lock_file_path> switch in order to make possible to control the execution of the tool and avoid to have multiple instances running.
249 lines
8.4 KiB
Python
Executable File
249 lines
8.4 KiB
Python
Executable File
#!/usr/bin/python2 -O
|
|
# -*- coding: utf-8 -*-
|
|
|
|
import os
|
|
import errno
|
|
import sys
|
|
import fcntl
|
|
sys.path.insert(0, "../libraries")
|
|
import tempfile
|
|
import subprocess
|
|
import bz2
|
|
import gzip
|
|
|
|
from entropy.const import etpConst
|
|
import entropy.dep
|
|
import entropy.tools
|
|
|
|
|
|
def generate_pkg_map(packages_directory):
|
|
"""
|
|
Generate handy hash table based on packages directory content. It will
|
|
be used by internal calling functions to determine the delta files that
|
|
have to be generated.
|
|
"""
|
|
pkg_map = {}
|
|
for pkg_file in os.listdir(packages_directory):
|
|
if not pkg_file.endswith(etpConst['packagesext']):
|
|
continue
|
|
try:
|
|
cat, name, ver, tag, rev = entropy.dep.exploit_package_filename(
|
|
pkg_file)
|
|
except AttributeError:
|
|
# skip invalid crap
|
|
continue
|
|
obj = pkg_map.setdefault((cat, name), set())
|
|
obj.add((ver, tag, rev, pkg_file))
|
|
return pkg_map
|
|
|
|
def sort_packages(pkg_map_items):
|
|
"""
|
|
Sort packages by version, tag, revision and return a sort map (dict) and
|
|
a sorted list of them (list)
|
|
"""
|
|
def _generate_from_to(cat_name_map, sorted_pkg_list):
|
|
for pkg_idx in range(len(sorted_pkg_list)):
|
|
pkg_key = sorted_pkg_list[pkg_idx]
|
|
next_pkgs = set(sorted_pkg_list[pkg_idx:])
|
|
next_pkgs.discard(pkg_key)
|
|
sorted_next = sorted(next_pkgs, key = lambda x: cat_name_map[x])
|
|
for next_pkg_key in sorted_next:
|
|
yield (cat_name_map[pkg_key], cat_name_map[next_pkg_key])
|
|
|
|
cat_name_map = dict((((ver, tag, rev), pkg_path) \
|
|
for ver, tag, rev, pkg_path in pkg_map_items))
|
|
sorted_pkgs = entropy.dep.get_entropy_newer_version(list(cat_name_map))
|
|
sorted_pkgs.reverse()
|
|
return _generate_from_to(cat_name_map, sorted_pkgs)
|
|
|
|
def generate_package_deltas(directory, quiet):
|
|
"""
|
|
Generate Entropy package delta files.
|
|
"""
|
|
for (cat, name), items in generate_pkg_map(directory).items():
|
|
# sort items, then generate deltas in one direction only
|
|
sorted_pkgs_couples = sort_packages(items)
|
|
for from_pkg_name, to_pkg_name in sorted_pkgs_couples:
|
|
pkg_path_a = os.path.join(directory, from_pkg_name)
|
|
next_pkg_path = os.path.join(directory, to_pkg_name)
|
|
hash_tag = entropy.tools.md5sum(pkg_path_a) + \
|
|
entropy.tools.md5sum(next_pkg_path)
|
|
delta_fn = entropy.tools.generate_entropy_delta_file_name(
|
|
from_pkg_name, to_pkg_name, hash_tag)
|
|
delta_path = os.path.join(directory,
|
|
etpConst['packagesdeltasubdir'], delta_fn)
|
|
delta_path_md5 = delta_path + etpConst['packagesmd5fileext']
|
|
if os.path.lexists(delta_path) and os.path.lexists(delta_path_md5):
|
|
if not quiet:
|
|
sys.stderr.write(delta_path + " already exists\n")
|
|
continue
|
|
delta_file = entropy.tools.generate_entropy_delta(pkg_path_a,
|
|
next_pkg_path, hash_tag)
|
|
entropy.tools.create_md5_file(delta_file)
|
|
if delta_file is not None:
|
|
sys.stdout.write(delta_file + "\n")
|
|
|
|
def cleanup_package_deltas(directory, quiet):
|
|
"""
|
|
Cleanup old Entropy package delta files.
|
|
"""
|
|
def _list_delta_packages(delta_dir):
|
|
return set([os.path.join(delta_dir, x) for x in os.listdir(delta_dir) \
|
|
if x.endswith(etpConst['packagesdeltaext'])])
|
|
|
|
avail_deltas = _list_delta_packages(os.path.join(directory,
|
|
etpConst['packagesdeltasubdir']))
|
|
required_deltas = set()
|
|
for (cat, name), items in generate_pkg_map(directory).items():
|
|
# sort items, then generate deltas in one direction only
|
|
sorted_pkgs_couples = sort_packages(items)
|
|
for from_pkg_name, to_pkg_name in sorted_pkgs_couples:
|
|
pkg_path_a = os.path.join(directory, from_pkg_name)
|
|
next_pkg_path = os.path.join(directory, to_pkg_name)
|
|
hash_tag = entropy.tools.md5sum(pkg_path_a) + \
|
|
entropy.tools.md5sum(next_pkg_path)
|
|
delta_fn = entropy.tools.generate_entropy_delta_file_name(
|
|
from_pkg_name, to_pkg_name, hash_tag)
|
|
delta_path = os.path.join(directory,
|
|
etpConst['packagesdeltasubdir'], delta_fn)
|
|
if os.path.lexists(delta_path):
|
|
required_deltas.add(delta_path)
|
|
|
|
to_remove_deltas = avail_deltas - required_deltas
|
|
rc = 0
|
|
if not to_remove_deltas:
|
|
sys.stdout.write("nothing to remove for %s\n" % (directory,))
|
|
for old_pkg_delta in to_remove_deltas:
|
|
try:
|
|
os.remove(old_pkg_delta + etpConst['packagesmd5fileext'])
|
|
except OSError:
|
|
pass
|
|
try:
|
|
os.remove(old_pkg_delta)
|
|
sys.stdout.write(old_pkg_delta + " removed\n")
|
|
except OSError as err:
|
|
if not quiet:
|
|
sys.stderr.write("cannot remove %s: %s\n" % (old_pkg_delta,
|
|
err))
|
|
rc = 1
|
|
return rc
|
|
|
|
def _generator_argv(argv, quiet):
|
|
for directory in argv:
|
|
if os.path.isdir(directory):
|
|
generate_package_deltas(directory, quiet)
|
|
return 0
|
|
|
|
def _cleanup_argv(argv, quiet):
|
|
rc = 1
|
|
for directory in argv:
|
|
if os.path.isdir(directory):
|
|
rc = cleanup_package_deltas(directory, quiet)
|
|
return rc
|
|
|
|
_cmds_map = {
|
|
'generate': _generator_argv,
|
|
'cleanup': _cleanup_argv,
|
|
}
|
|
|
|
def _opts_parser(args):
|
|
|
|
# --quiet handler
|
|
quiet = False
|
|
for q_opt in ("-q", "--quiet"):
|
|
if q_opt in args:
|
|
quiet = True
|
|
while True:
|
|
try:
|
|
args.remove("--quiet")
|
|
except ValueError:
|
|
break
|
|
|
|
lock_file = None
|
|
if "--lock" in args:
|
|
lock_idx = args.index("--lock")
|
|
try:
|
|
lock_file = args.pop(lock_idx + 1)
|
|
args.pop(lock_idx)
|
|
if not os.path.isdir(os.path.dirname(lock_file)):
|
|
raise ValueError("invalid lock file path provided")
|
|
if os.path.lexists(lock_file) and (not os.path.isfile(lock_file)):
|
|
raise ValueError("invalid lock file path provided, not a file")
|
|
except IndexError:
|
|
sys.stderr.write("--lock provided without path\n")
|
|
return None, [], False, lock_file
|
|
except ValueError as err:
|
|
sys.stderr.write(err + "\n")
|
|
return None, [], False, lock_file
|
|
|
|
if not args:
|
|
return None, [], False, lock_file
|
|
cmd, argv = args[0], args[1:]
|
|
if not argv:
|
|
return None, [], False, lock_file
|
|
func = _cmds_map.get(cmd)
|
|
if func is None:
|
|
return None, [], False, lock_file
|
|
return func, argv, quiet, lock_file
|
|
|
|
def _print_help():
|
|
sys.stdout.write(
|
|
"entropy-pkgdelta-generator [--quiet] [--lock <lock_path>] <command> <pkgdir> [... <pkgdir> ...]\n\n")
|
|
sys.stdout.write("available commands:\n")
|
|
sys.stdout.write("\tgenerate\tgenerate pkgdelta files for given package directories\n")
|
|
sys.stdout.write("\tcleanup\t\tclean pkgdelta files for unavailable packages\n\n")
|
|
|
|
_LOCK_HANDLES = {}
|
|
|
|
def _acquire_lock(lock_file):
|
|
lock_f = open(lock_file, "w")
|
|
try:
|
|
fcntl.flock(lock_f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except IOError as err:
|
|
if err.errno not in (errno.EACCES, errno.EAGAIN,):
|
|
# ouch, wtf?
|
|
raise
|
|
lock_f.close()
|
|
return False # lock already acquired
|
|
|
|
lock_f.write(str(os.getpid()))
|
|
lock_f.flush()
|
|
_LOCK_HANDLES[lock_file] = lock_f
|
|
return True
|
|
|
|
def _release_lock(lock_file):
|
|
try:
|
|
lock_f = _LOCK_HANDLES.pop(lock_file)
|
|
except KeyError:
|
|
lock_f = None
|
|
|
|
if lock_f is not None:
|
|
fcntl.flock(lock_f.fileno(), fcntl.LOCK_UN)
|
|
lock_f.close()
|
|
|
|
try:
|
|
os.remove(lock_file)
|
|
except OSError as err:
|
|
# cope with possible race conditions
|
|
if err.errno != errno.ENOENT:
|
|
raise
|
|
|
|
if __name__ == "__main__":
|
|
func, argv, quiet, lock_file = _opts_parser(sys.argv[1:])
|
|
if func is not None:
|
|
# acquire lock
|
|
if lock_file:
|
|
acquired = _acquire_lock(lock_file)
|
|
if not acquired:
|
|
sys.stderr.write("cannot acquire lock on " + lock_file + "\n")
|
|
raise SystemExit(5)
|
|
try:
|
|
rc = func(argv, quiet)
|
|
finally:
|
|
if lock_file:
|
|
_release_lock(lock_file)
|
|
else:
|
|
_print_help()
|
|
rc = 1
|
|
raise SystemExit(rc)
|