#!/usr/bin/python import sys import os import shlex import signal import argparse import tempfile import subprocess import errno import fcntl from Queue import Empty as EmptyQueue from multiprocessing import Queue, Lock # Entropy imports sys.path.insert(0,'/usr/lib/entropy/libraries') sys.path.insert(0,'/usr/lib/entropy/server') sys.path.insert(0,'/usr/lib/entropy/client') sys.path.insert(0,'../libraries') sys.path.insert(0,'../server') sys.path.insert(0,'../client') # Entropy imports from entropy.exceptions import PermissionDenied from entropy.const import etpConst, etpUi, const_convert_to_unicode, \ const_get_stringtype from entropy.output import print_info, print_error, print_warning, \ purple, darkgreen, is_stdout_a_tty, getcolor from entropy.exceptions import InvalidAtom, SPMError from entropy.server.interfaces import Server import entropy.tools import entropy.dep def get_entropy_server(community_mode): """ Return Entropy Server interface object. """ return Server(community_repo = community_mode) def exec_cmd(args, env = None): """ Execute a command with given environment. """ pid = os.fork() if pid == 0: try: if env is not None: os.execvpe(args[0], args, env) else: os.execvp(args[0], args) except Exception as exc: entropy.tools.print_traceback() os._exit(1) else: try: rcpid, rc = os.waitpid(pid, os.P_WAIT) except KeyboardInterrupt: rc = 1 os.kill(pid, signal.SIGTERM) return rc class EntropyResourceLock(object): """ This class exposes a Lock-like interface for acquiring Entropy Server resources. """ class NotAcquired(Exception): """ Raised when Entropy Resource Lock cannot be acquired """ def __init__(self, entropy_server, blocking): """ EntropyResourceLock constructor. @param entropy_server: Entropy Server instance @type entropy_server: entropy.server.interfaces.Server @param blocking: acquire lock in blocking mode? @type blocking: bool """ self._entropy = entropy_server self._blocking = blocking self.__inside_with_stmt = 0 def acquire(self): acquired = entropy.tools.acquire_entropy_resources_locks(self._entropy, blocking = self._blocking) if not acquired: raise EntropyResourceLock.NotAcquired("unable to acquire lock") def release(self): entropy.tools.release_entropy_locks(self._entropy) def __enter__(self): """ Acquire lock. Not thread-safe. """ if self.__inside_with_stmt < 1: self.acquire() self.__inside_with_stmt += 1 return self def __exit__(self, exc_type, exc_value, traceback): """ Release lock. Not thread-safe. """ self.__inside_with_stmt -= 1 if self.__inside_with_stmt < 1: self.release() class MatterResourceLock(object): """ This class exposes a Lock-like interface for acquiring Matter lock file. """ LOCK_FILE_PATH = "/var/tmp/.matter_resource.lock" class NotAcquired(Exception): """ Raised when Lock cannot be acquired """ def __init__(self, blocking): """ MatterResourceLock constructor. @param blocking: acquire lock in blocking mode? @type blocking: bool """ self._blocking = blocking self.__inside_with_stmt = 0 self.__lock_f = None self.__call_lock = Lock() def acquire(self): """ Acquire the lock file. """ file_path = MatterResourceLock.LOCK_FILE_PATH if self._blocking: flags = fcntl.LOCK_EX | fcntl.LOCK_NB else: flags = fcntl.LOCK_EX with self.__call_lock: if self.__lock_f is None: self.__lock_f = open(file_path, "wb") try: fcntl.flock(self.__lock_f.fileno(), flags) except IOError as err: if err.errno not in (errno.EACCES, errno.EAGAIN,): # ouch, wtf? raise raise MatterResourceLock.NotAcquired( "unable to acquire lock") def release(self): with self.__call_lock: if self.__lock_f is not None: fcntl.flock(self.__lock_f.fileno(), fcntl.LOCK_UN) self.__lock_f.close() self.__lock_f = None def __enter__(self): """ Acquire lock. Not thread-safe. """ if self.__inside_with_stmt < 1: self.acquire() self.__inside_with_stmt += 1 return self def __exit__(self, exc_type, exc_value, traceback): """ Release lock. Not thread-safe. """ self.__inside_with_stmt -= 1 if self.__inside_with_stmt < 1: self.release() class GenericSpecFunctions(object): def ne_string(self, x): return x, 'raw_unicode_escape' def ne_list(self, x): return x def not_none(self, x): return x is not None def valid_integer(self, x): try: int(x) except (TypeError, ValueError,): return False return True def always_valid(self, *args): return True def valid_path(self, x): return os.path.lexists(x) def valid_file(self, x): return os.path.isfile(x) def valid_dir(self, x): return os.path.isdir(x) def ve_string_open_file_read(self, x): try: return open(x, "r") except (IOError, OSError): return None def ve_string_stripper(self, x): return const_convert_to_unicode(x).strip() def ve_string_splitter(self, x): return const_convert_to_unicode(x).strip().split() def ve_integer_converter(self, x): return int(x) def valid_ascii(self, x): try: x = str(x) return x except (UnicodeDecodeError, UnicodeEncodeError,): return '' def valid_yes_no(self, x): return x in ("yes", "no") def valid_path_string(self, x): try: os.path.split(x) except OSError: return False return True def valid_path_string_first_list_item(self, x): if not x: return False myx = x[0] try: os.path.split(myx) except OSError: return False return True def valid_comma_sep_list(self, x): return [y.strip() for y in \ const_convert_to_unicode(x).split(",") if y.strip()] def valid_path_list(self, x): return [y.strip() for y in \ const_convert_to_unicode(x).split(",") if \ self.valid_path_string(y) and y.strip()] class MatterSpec(GenericSpecFunctions): def vital_parameters(self): """ Return a list of vital .spec file parameters @return: list of vital .spec file parameters @rtype: list """ return ["packages", "repository"] def parser_data_path(self): """ Return a dictionary containing parameter names as key and dict containing keys 've' and 'cb' which values are three callable functions that respectively do value extraction (ve), value verification (cb) and value modding (mod). @return: data path dictionary (see ChrootSpec code for more info) @rtype: dict """ return { 'dependencies': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'downgrade': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'keep-going': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'new-useflags': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'removed-useflags': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'rebuild': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'spm-repository-change': { 'cb': self.valid_yes_no, 've': self.ve_string_stripper, 'default': "no", }, 'pkgpre': { 'cb': self.not_none, 've': self.ve_string_open_file_read, 'default': None, }, 'pkgpost': { 'cb': self.not_none, 've': self.ve_string_open_file_read, 'default': None, }, 'packages': { 'cb': self.always_valid, 've': self.valid_comma_sep_list, }, 'repository': { 'cb': self.ne_string, 've': self.ve_string_stripper, }, } class SpecPreprocessor: PREFIX = "%" class PreprocessorError(Exception): """ Error while preprocessing file """ def __init__(self, spec_file_obj): self.__expanders = {} self.__builtin_expanders = {} self._spec_file_obj = spec_file_obj self._add_builtin_expanders() def add_expander(self, statement, expander_callback): """ Add Preprocessor expander. @param statement: statement to expand @type statement: string @param expand_callback: one argument callback that is used to expand given line (line is raw format). Line is already pre-parsed and contains a valid preprocessor statement that callback can handle. Preprocessor callback should raise SpecPreprocessor.PreprocessorError if line is malformed. @type expander_callback: callable @raise KeyError: if expander is already available @return: a raw string (containing \n and whatever) @rtype: string """ return self._add_expander(statement, expander_callback, builtin = False) def _add_expander(self, statement, expander_callback, builtin = False): obj = self.__expanders if builtin: obj = self.__builtin_expanders if statement in obj: raise KeyError("expander %s already provided" % (statement,)) obj[SpecPreprocessor.PREFIX + statement] = \ expander_callback def _add_builtin_expanders(self): # import statement self._add_expander("import", self._import_expander, builtin = True) def _import_expander(self, line): rest_line = line.split(" ", 1)[1].strip() if not rest_line: return line spec_f = self._spec_file_obj spec_f.seek(0) lines = '' try: for line in spec_f.readlines(): # call recursively split_line = line.split(" ", 1) if split_line: expander = self.__builtin_expanders.get(split_line[0]) if expander is not None: try: line = expander(line) except RuntimeError as err: raise SpecPreprocessor.PreprocessorError( "invalid preprocessor line: %s" % (err,)) lines += line finally: spec_f.seek(0) return lines def parse(self): content = [] spec_f = self._spec_file_obj spec_f.seek(0) try: for line in spec_f.readlines(): split_line = line.split(" ", 1) if split_line: expander = self.__builtin_expanders.get(split_line[0]) if expander is not None: line = expander(line) content.append(line) finally: spec_f.seek(0) final_content = [] for line in content: split_line = line.split(" ", 1) if split_line: expander = self.__expanders.get(split_line[0]) if expander is not None: line = expander(line) final_content.append(line) final_content = (''.join(final_content)).split("\n") return final_content class SpecParser: def __init__(self, file_object): self.file_object = file_object self._preprocessor = SpecPreprocessor(self.file_object) self.__plugin = MatterSpec() self.vital_parameters = self.__plugin.vital_parameters() self._parser_data_path = self.__plugin.parser_data_path() def _parse_line_statement(self, line_stmt): try: key, value = line_stmt.split(":", 1) except ValueError: return None, None key, value = key.strip(), value.strip() return key, value def parse(self): mydict = {} data = self._generic_parser() # compact lines properly old_key = None for line in data: key = None value = None v_key, v_value = self._parse_line_statement(line) check_dict = self._parser_data_path.get(v_key) if check_dict is not None: key, value = v_key, v_value old_key = key elif isinstance(old_key, const_get_stringtype()): key = old_key value = line.strip() if not value: continue # gather again... key is changed check_dict = self._parser_data_path.get(key) if not isinstance(check_dict, dict): continue value = check_dict['ve'](value) if not check_dict['cb'](value): continue if key in mydict: if isinstance(value, const_get_stringtype()): mydict[key] += " %s" % (value,) elif isinstance(value, list): mydict[key] += value else: continue else: mydict[key] = value self._validate_parse(mydict) self._extend_parse(mydict) return mydict.copy() def _extend_parse(self, mydata): """ Extend parsed data with default values for statements with default option available. """ for statement, opts in self._parser_data_path.items(): if "default" in opts and (statement not in mydata): mydata[statement] = opts['default'] def _validate_parse(self, mydata): for param in self.vital_parameters: if param not in mydata: raise ValueError( "'%s' missing or invalid" " '%s' parameter, it's vital. Your specification" " file is incomplete!" % (self.file_object.name, param,) ) def _generic_parser(self): data = [] content = self._preprocessor.parse() # filter comments and white lines content = [x.strip().rsplit("#", 1)[0].strip() for x in content if \ not x.startswith("#") and x.strip()] for line in content: if line in data: continue data.append(line) return data class PackageBuilder(object): """ Portage Package builder class """ DEFAULT_PORTAGE_SYNC_CMD = "emerge --sync" PORTAGE_SYNC_CMD = shlex.split(os.getenv("MATTER_PORTAGE_SYNC_CMD", DEFAULT_PORTAGE_SYNC_CMD)) DEFAULT_OVERLAYS_SYNC_CMD = "layman -S" OVERLAYS_SYNC_CMD = shlex.split(os.getenv("MATTER_OVERLAYS_SYNC_CMD", DEFAULT_OVERLAYS_SYNC_CMD)) DEFAULT_PORTAGE_BUILD_ARGS = "--verbose --nospinner" PORTAGE_BUILD_ARGS = os.getenv("MATTER_PORTAGE_BUILD_ARGS", DEFAULT_PORTAGE_BUILD_ARGS).split() def __init__(self, entropy_server, package, params, spec_number, tot_spec, pkg_number, tot_pkgs): self._entropy = entropy_server self._package = package self._params = params self._spec_number = spec_number self._tot_spec = tot_spec self._pkg_number = pkg_number self._tot_pkgs = tot_pkgs self._built_packages = [] self._not_found_packags = [] self._not_installed_packages = [] self._not_merged_packages = [] @staticmethod def _build_standard_environment(repository=None): env = os.environ.copy() if repository is not None: env["MATTER_REPOSITORY_ID"] = repository return env @staticmethod def setup(executable_hook_f, cwd): # ignore exit status subprocess.call(["env-update"]) hook_name = executable_hook_f.name if not hook_name.endswith("/"): # complete with current directory hook_name = os.path.join(cwd, hook_name) print_info("spawning pre hook: %s" % (hook_name,)) return exec_cmd([hook_name], env = PackageBuilder._build_standard_environment()) @staticmethod def teardown(executable_hook_f, cwd, exit_st): hook_name = executable_hook_f.name if not hook_name.endswith("/"): # complete with current directory hook_name = os.path.join(cwd, hook_name) print_info("spawning post hook: %s, passing exit status: %d" % ( hook_name, exit_st,)) env = PackageBuilder._build_standard_environment() env["MATTER_EXIT_STATUS"] = str(exit_st) return exec_cmd([hook_name], env = env) def _build_execution_header_output(self): """ Return a string used as stdout/stderr header text. """ my_str = "{%s of %s particles | %s of %s packages} " % ( darkgreen(str(self._spec_number)), purple(str(self._tot_spec)), darkgreen(str(self._pkg_number)), purple(str(self._tot_pkgs)),) return my_str def get_built_packages(self): """ Return the list of successfully built packages. """ return self._built_packages def get_not_found_packages(self): """ Return the list of packages that haven't been found in Portage. """ return self._not_found_packags def get_not_installed_packages(self): """ Return the list of packages that haven't been found on the System. """ return self._not_installed_packages def get_not_merged_packages(self): """ Return the list of packages that haven't been able to compile. """ return self._not_merged_packages def run(self): """ Execute Package building action. """ header = self._build_execution_header_output() print_info( header + "spawning package build: %s" % (self._package,)) std_env = PackageBuilder._build_standard_environment( repository=self._params["repository"]) std_env["MATTER_PACKAGE_NAME"] = self._package print_info("MATTER_PACKAGE_NAME = %s" % (self._package,)) # run pkgpre, if any pkgpre = self._params["pkgpre"] if pkgpre is not None: print_info( "spawning --pkgpre: %s, name: %s" % (pkgpre, pkgpre.name)) tmp_fd, tmp_path = tempfile.mkstemp(prefix="matter") with os.fdopen(tmp_fd, "wb") as tmp_f: tmp_f.write(pkgpre.read()) try: # now execute os.chmod(tmp_path, 0o700) exit_st = exec_cmd([tmp_path], env = std_env) if exit_st != 0: return exit_st finally: os.remove(tmp_path) pkg_queue = Queue() not_found_queue = Queue() not_installed_queue = Queue() not_merged_queue = Queue() # execute the update code pid = os.fork() if pid == 0: try: rc = self._run_builder(std_env, pkg_queue, not_found_queue, not_installed_queue, not_merged_queue) except KeyboardInterrupt: os._exit(1) except Exception as exc: entropy.tools.print_traceback() sys.stderr.write(repr(exc) + "\n") os._exit(1) finally: pkg_queue.close() pkg_queue.join_thread() not_found_queue.close() not_found_queue.join_thread() not_installed_queue.close() not_installed_queue.join_thread() not_merged_queue.close() not_merged_queue.join_thread() os._exit(rc) else: try: rcpid, exit_st = os.waitpid(pid, os.P_WAIT) except KeyboardInterrupt: try: os.kill(pid, signal.SIGTERM) except OSError as err: if err.errno != errno.ESRCH: raise exit_st = 1 except Exception as exc: try: os.kill(pid, signal.SIGTERM) except OSError as err: if err.errno != errno.ESRCH: raise exit_st = 1 print_info("builder terminated, exit status: %d" % (exit_st,)) queues = [(pkg_queue, self._built_packages, "queue"), (not_found_queue, self._not_found_packags, "not_found"), (not_installed_queue, self._not_installed_packages, "not_installed"), (not_merged_queue, self._not_merged_packages, "not_merged")] for queue, lst, queue_name in queues: while True: try: lst.append(queue.get(False)) except EmptyQueue: break queue.close() # run pkgpre, if any pkgpost = self._params["pkgpost"] if pkgpost is not None: print_info( "spawning --pkgpost: %s, name: %s" % (pkgpost, pkgpost.name)) tmp_fd, tmp_path = tempfile.mkstemp(prefix="matter") with os.fdopen(tmp_fd, "wb") as tmp_f: tmp_f.write(pkgpost.read()) try: # now execute os.chmod(tmp_path, 0o700) post_exit_st = exec_cmd([tmp_path, str(exit_st)], env = std_env) if post_exit_st != 0: return post_exit_st finally: os.remove(tmp_path) return exit_st def _run_builder(self, env, pkg_queue, not_found_queue, not_installed_queue, not_merged_queue): """ This method is called by _run and executes the whole package build logic, including constraints validation given by argv parameters. NOTE: negative errors indicate warnings that can be skipped. """ os.environ['ACCEPT_PROPERTIES'] = "* -interactive" from _emerge.depgraph import backtrack_depgraph from _emerge.actions import load_emerge_config, action_build from _emerge.create_depgraph_params import create_depgraph_params from _emerge.main import parse_opts, post_emerge, \ validate_ebuild_environment from _emerge.stdout_spinner import stdout_spinner import portage.versions import portage portdb = portage.portdb portdb.freeze() settings = portage.config(clone=portage.settings) vardb = portage.db[settings["ROOT"]]["vartree"].dbapi fakedb = portage.fakedbapi(settings=portage.settings) # Load the most current variables from /etc/profile.env, which # has been re-generated by the env-update call in _run() settings.unlock() settings.reload() settings.regenerate() settings.lock() best_visible = portdb.xmatch("bestmatch-visible", self._package) if not best_visible: # package not found, return error print_error("cannot match: %s, aborting" % (self._package,)) not_found_queue.put(self._package) pkg_queue.close() pkg_queue.join_thread() return 1 print_info("matched: %s for %s" % (best_visible, self._package,)) # now determine what's the installed version. best_installed = portage.best(vardb.match(self._package)) if not best_installed: # package not installed, behaviour not supported atm print_error("package not installed: %s, aborting" % ( self._package,)) not_installed_queue.put(self._package) return 1 print_info("found installed: %s for %s" % (best_installed, self._package,)) # now compare # -1 if best_installed is older than best_visible # 1 if best_installed is newer than best_visible # 0 if they are equal cmp_res = portage.versions.pkgcmp( portage.versions.pkgsplit(best_installed), portage.versions.pkgsplit(best_visible)) allow_rebuild = self._params['rebuild'] == "yes" allow_downgrade = self._params['downgrade'] == "yes" is_rebuild = cmp_res == 0 if (cmp_res == 1) and (not allow_downgrade): # downgrade in action and downgrade not allowed, aborting! print_warning( "package: %s, would be downgraded from %s to %s, aborting" % ( self._package, best_installed, best_visible,)) return 0 if (is_rebuild) and (not allow_rebuild): # rebuild in action and rebuild not allowed, aborting! print_warning( "package: %s, would be rebuilt to %s, aborting" % ( self._package, best_visible,)) return 0 # at this point we can go ahead building self._package print_info("starting to build: %s, to %s" % (self._package, best_visible,)) print_info("portage modules loaded with success") emerge_settings, emerge_trees, mtimedb = \ load_emerge_config(trees=portage.db) if not getcolor(): portage.output.nocolor() # non interactive properties, this is not really required # accept-properties just sets os.environ... builtin_args = ["--accept-properties=-interactive"] myaction, myopts, myfiles = parse_opts( PackageBuilder.PORTAGE_BUILD_ARGS + builtin_args + \ ["="+best_visible]) if "--pretend" in myopts: print_warning("cannot use --pretend emerge argument, you idiot") del myopts["--pretend"] if "--ask" in myopts: print_warning("cannot use --ask emerge argument, you idiot") del myopts["--ask"] spinner = stdout_spinner() if "--quiet" in myopts: spinner.update = spinner.update_basic elif "--nospinner" in myopts: spinner.update = spinner.update_basic if emerge_settings.get("TERM") == "dumb" or not is_stdout_a_tty(): spinner.update = spinner.update_basic print_info("emerge args: %s" % (" ".join(sorted(myopts.keys())),)) params = create_depgraph_params(myopts, myaction) success, graph, favorites = backtrack_depgraph(emerge_settings, emerge_trees, myopts, params, myaction, myfiles, spinner) if not success: # print issues to stdout and give up print_warning("dependencies calculation failed for %s, aborting" % ( best_visible,)) graph.display_problems() return 0 print_info("dependency graph generated successfully") # list of _emerge.Package.Package objects package_queue = graph.altlist() dep_list = [p.cpv+"::"+p.repo for p in package_queue] # calculate dependencies, if --dependencies is not enabled # because we have to validate it if (self._params['dependencies'] == "no") \ and (len(package_queue) > 1): # package is pulling in dependencies, but --dependencies is not # enabled. need to give up deps = ", ".join(dep_list) print_warning( "package %s is pulling in: %s, but --dependencies " "not specified, aborting" % (best_visible, deps,)) return 0 # inspect use flags changes allow_new_useflags = self._params['new-useflags'] == "yes" allow_removed_useflags = \ self._params['removed-useflags'] == "yes" use_flags_give_up = False if (not allow_new_useflags) or (not allow_removed_useflags): # checking for use flag changes for pkg in package_queue: # frozenset enabled_flags = pkg.use.enabled inst_atom = portage.best(vardb.match(pkg.slot_atom)) if not inst_atom: # new package, ignore check continue installed_flags = frozenset( vardb.aux_get(inst_atom, ["USE"])[0].split()) new_flags = enabled_flags - installed_flags removed_flags = installed_flags - enabled_flags if (not allow_new_useflags) and new_flags: print_warning("ouch: %s wants these new USE flags: %s" % ( p.cpv+"::"+p.repo, " ".join(sorted(new_flags)),)) use_flags_give_up = True if (not allow_removed_useflags) and removed_flags: print_warning("ouch: %s has these USE flags removed: %s" % ( p.cpv+"::"+p.repo, " ".join(sorted(removed_flags)),)) use_flags_give_up = True if use_flags_give_up: print_warning("cannot continue due to unmet USE flags constraint") return 0 changing_repo_pkgs = [] for pkg in package_queue: wanted_repo = pkg.repo inst_atom = portage.best(vardb.match(pkg.slot_atom)) current_repo = vardb.aux_get(inst_atom, ["repository"])[0] if current_repo: if current_repo != wanted_repo: changing_repo_pkgs.append( (pkg.cpv, current_repo, wanted_repo)) if changing_repo_pkgs: print_warning("") print_warning( "Attention, packages are moving across SPM repositories:") for pkg_atom, current_repo, wanted_repo in changing_repo_pkgs: print_warning(" %s [%s->%s]" % (pkg_atom, current_repo, wanted_repo,)) print_warning("") allow_spm_repo_change = self._params['spm-repository-change'] == "yes" if changing_repo_pkgs and (not allow_spm_repo_change): print_warning( "cannot continue due to unmet SPM repository change constraint") return 0 print_info("USE flags constraints are met for all the queued packages") print_info("about to build the following packages:") for dep in dep_list: print_info(" %s" % (dep,)) # re-calling action_build(), deps are re-calculated though validate_ebuild_environment(emerge_trees) retval = action_build(emerge_settings, emerge_trees, mtimedb, myopts, myaction, myfiles, spinner) not_merged = [] if retval != 0: merge_list = mtimedb.get("resume", {}).get("mergelist") for merge_type, merge_root, merge_atom, merge_act in merge_list: not_merged.append(merge_atom) not_merged_queue.put(merge_atom) for pkg in package_queue: cpv = pkg.cpv if cpv not in not_merged: # add to build queue print_info("package: %s, successfully built" % (cpv,)) pkg_queue.put(cpv) post_emerge(myaction, myopts, myfiles, emerge_settings["ROOT"], emerge_trees, mtimedb, retval) subprocess.call(["env-update"]) print_info("portage spawned, return value: %d" % (retval,)) return retval @staticmethod def sync(): """ Execute Portage and Overlays sync """ sync_cmd = PackageBuilder.PORTAGE_SYNC_CMD std_env = PackageBuilder._build_standard_environment() rc = exec_cmd(sync_cmd, env = std_env) if rc != 0: return rc # overlays update overlay_cmd = PackageBuilder.OVERLAYS_SYNC_CMD return exec_cmd(overlay_cmd, env = std_env) @staticmethod def check_preserved_libraries(): """ Ask portage whether there are preserved libraries on the system. This usually indicates that Entropy packages should not be really committed. @return: True, if preserved libraries are found @rtype: bool """ pid = os.fork() if pid == 0: try: import portage settings = portage.config(clone=portage.settings) vardb = portage.db[settings["ROOT"]]["vartree"].dbapi vardb._plib_registry.load() if vardb._plib_registry.hasEntries(): os._exit(1) os._exit(0) except KeyboardInterrupt: os._exit(1) except Exception as exc: sys.stderr.write(repr(exc) + "\n") os._exit(1) else: try: rcpid, rc = os.waitpid(pid, os.P_WAIT) except KeyboardInterrupt: os.kill(pid, signal.SIGTERM) rc = 1 except Exception as exc: os.kill(pid, signal.SIGTERM) rc = 1 if rc == 0: return False return True @staticmethod def commit(entropy_server, repository, packages): """ Commit packages to Entropy repository. """ spm = entropy_server.Spm() spm_atoms = set() exit_st = 0 print_info("committing packages: %s, to repository: %s" % ( ", ".join(sorted(packages)), repository,)) # if we get here, something has been compiled # successfully for package in packages: try: spm_atom = spm.match_installed_package(package) spm_atoms.add(spm_atom) except InvalidAtom: exit_st = 1 print_warning( "cannot find installed package: %s" % ( package,)) continue if not spm_atoms: return exit_st print_info("about to commit:") spm_packages = sorted(spm_atoms) for atom in spm_packages: item_txt = atom # this is a spm atom spm_key = entropy.dep.dep_getkey(atom) try: spm_slot = spm.get_installed_package_metadata( atom, "SLOT") spm_repo = spm.get_installed_package_metadata( atom, "repository") except KeyError: spm_slot = None spm_repo = None etp_repo = None if spm_repo is not None: pkg_id, repo_id = entropy_server.atom_match(spm_key, match_slot = spm_slot) if repo_id != 1: repo_db = entropy_server.open_repository(repo_id) etp_repo = repo_db.retrieveSpmRepository(pkg_id) if (etp_repo is not None) and (etp_repo != spm_repo): item_txt += ' [%s {%s=>%s}]' % ("warning", etp_repo, spm_repo,) print_info(item_txt) # always stuff new configuration files here # if --gentle was specified, the uncommitted stuff here belongs # to our packages. # if --gentle was NOT specified, we just don't give a shit uncommitted = entropy_server._check_config_file_updates() if uncommitted: subprocess.call("echo -5 | etc-update", shell = True) # test again uncommitted = entropy_server._check_config_file_updates() if uncommitted: # ouch, wtf? better aborting print_error("tried to commit configuration file changes and failed") return 1 print_info("about to compress:") store_dir = entropy_server._get_local_store_directory(repository) package_paths = [] for atom in spm_packages: print_info(atom) try: pkg_list = spm.generate_package(atom, store_dir) except OSError: entropy.tools.print_traceback() print_error("problem during package generation, aborting") return 1 except SPMError: entropy.tools.print_traceback() print_error("problem during package generation (2), aborting") return 1 package_paths.append(pkg_list) etp_pkg_files = [(pkg_list, False) for pkg_list in package_paths] # NOTE: any missing runtime dependency will be added # (beside those blacklisted), since this execution is not interactive package_ids = entropy_server.add_packages_to_repository( repository, etp_pkg_files, ask = False) if package_ids: # checking dependencies and print issues entropy_server.dependencies_test(repository) entropy_server.close_repositories() return exit_st @staticmethod def push(entropy_server, repository): """ Push staged packages in repository to online Entropy mirrors. """ rc = PackageBuilder._push_packages(entropy_server, repository) if rc != 0: return rc rc = PackageBuilder._push_repository(entropy_server, repository) return rc @staticmethod def _push_packages(entropy_server, repository): """ Upload newly built packages. """ mirrors_tainted, mirrors_errors, successfull_mirrors, \ broken_mirrors, check_data = \ entropy_server.Mirrors.sync_packages( repository, ask = False, pretend = False) if mirrors_errors and not successfull_mirrors: return 1 return 0 @staticmethod def _push_repository(entropy_server, repository): """ Update remote repository. """ sts = entropy_server.Mirrors.sync_repository(repository) return sts def matter_main(entropy_server, nsargs, cwd, specs): """ Main application code run after all the resources setup. """ exit_st = 0 preserved_libs = PackageBuilder.check_preserved_libraries() if preserved_libs: print_error( "preserved libraries are found on system, aborting.") raise SystemExit(7) if nsargs.gentle: # check if there is something to do to_be_added, to_be_removed, to_be_injected = \ entropy_server.scan_package_changes() if to_be_added: # only check this, others we can ignore to_be_added = [x[0] for x in to_be_added] to_be_added.sort() print_error("--gentle specified, and unstaged packages found:") for name in to_be_added: print_warning(" " + name) raise SystemExit(5) # also check for uncommitted configuration files changed problems = entropy_server._check_config_file_updates() if problems: print_error( "some configuration files have to be merged manually") raise SystemExit(6) print_info("matter loaded, starting to scan particles, pid: %s" % ( os.getpid(),)) # setup if nsargs.pre: rc = PackageBuilder.setup(nsargs.pre, cwd) if rc != 0: exit_st = rc if exit_st == 0: if nsargs.sync: rc = PackageBuilder.sync() if rc != 0: exit_st = rc if exit_st == 0: completed = [] not_found = [] not_installed = [] not_merged = [] tainted_repositories = set() preserved_libs_error = False spec_count = 0 tot_spec = len(specs) for spec in specs: spec_count += 1 keep_going = spec["keep-going"] == "yes" local_completed = [] pkg_count = 0 tot_pkgs = len(spec['packages']) for package in spec['packages']: pkg_count += 1 builder = PackageBuilder(entropy_server, package, spec, spec_count, tot_spec, pkg_count, tot_pkgs) rc = builder.run() not_found.extend(builder.get_not_found_packages()) not_installed.extend( builder.get_not_installed_packages()) not_merged.extend( builder.get_not_merged_packages()) preserved_libs = \ PackageBuilder.check_preserved_libraries() if preserved_libs: # abort, library breakages detected exit_st = 1 print_error( "preserved libraries detected, aborting") preserved_libs_error = True break if rc == 0: built_packages = builder.get_built_packages() print_info("built packages, in queue: %s" % ( " ".join(built_packages),)) # make some room print_info("") local_completed.extend([x for x in built_packages \ if x not in local_completed]) tainted_repositories.add(spec['repository']) elif rc < 0: # ignore warning and go ahead print_info("") continue else: print_info("") exit_st = rc if not keep_going: break if preserved_libs_error: # completely abort break completed.extend(local_completed) # portage calls setcwd() os.chdir(cwd) if local_completed and nsargs.commit: rc = PackageBuilder.commit(entropy_server, spec['repository'], local_completed) if exit_st == 0 and rc != 0: exit_st = rc if not keep_going: break if tainted_repositories and nsargs.push and nsargs.commit \ and not preserved_libs_error: for repository in tainted_repositories: rc = PackageBuilder.push(entropy_server, repository) if exit_st == 0 and rc != 0: exit_st = rc # print summary print_info("") print_info("Summary") print_info("Packages built: %s" % ( " ".join(completed),)) print_info("Packages not built: %s" % ( " ".join(not_merged),)) print_info("Packages not found: %s" % ( " ".join(not_found),)) print_info("Packages not installed: %s" % ( " ".join(not_installed),)) print_info("") if nsargs.post: rc = PackageBuilder.teardown(nsargs.post, cwd, exit_st) if exit_st == 0 and rc != 0: exit_st = rc raise SystemExit(exit_st) if __name__ == "__main__": ENV_VARS_HELP = """\ Environment variables for Package Builder module: %s = repository identifier %s = alternative command used to sync Portage default: %s %s = alternative command used to sync Portage overlays default: %s %s = custom emerge arguments default: %s Environment variables passed to --post executables: %s = exit status from previous execution phases, useful for detecting execution errors. Environment variables passed to --pkgpre/--pkgpost executables: %s = name of the package that would be built Matter Resources Lock file you can use to detect if matter is running: %s (--blocking switch makes it acquire in blocking mode) """ % ( purple("MATTER_REPOSITORY_ID"), purple("MATTER_PORTAGE_SYNC_CMD"), darkgreen(PackageBuilder.DEFAULT_PORTAGE_SYNC_CMD), purple("MATTER_OVERLAYS_SYNC_CMD"), darkgreen(PackageBuilder.DEFAULT_OVERLAYS_SYNC_CMD), purple("MATTER_PORTAGE_BUILD_ARGS"), darkgreen(PackageBuilder.DEFAULT_PORTAGE_BUILD_ARGS), purple("MATTER_EXIT_STATUS"), purple("MATTER_PACKAGE_NAME"), darkgreen(MatterResourceLock.LOCK_FILE_PATH),) parser = argparse.ArgumentParser( description='Automated Packages Builder', epilog=ENV_VARS_HELP, formatter_class=argparse.RawDescriptionHelpFormatter) # * instead of + in order to support --sync only tasks parser.add_argument("spec", nargs='+', metavar="", type=file, help="matter spec file") parser.add_argument("--blocking", help="when trying to acquire Entropy Server locks, block until success", action="store_true") parser.add_argument("--commit", help="commit built packages to repository", action="store_true") parser.add_argument("--community", help="enforce Community Repository mode on Entropy Server", action="store_true") parser.add_argument("--debug", help="print debug output", action="store_true") parser.add_argument("--gentle", help="do not run if staged packages are present in Entropy repository", action="store_true") parser.add_argument("--pre", metavar="", type=file, help="executable to be called once for setup purposes", default=None) parser.add_argument("--post", metavar="", type=file, help="executable to be called once for teardown purposes", default=None) parser.add_argument("--push", help="push entropy package updates to online repository (only if --commit)", action="store_true") parser.add_argument("--sync", help="sync Portage tree, and attached overlays, before starting", action="store_true") try: nsargs = parser.parse_args(sys.argv[1:]) except IOError as err: if err.errno == errno.ENOENT: print_error(err.strerror + ": " + err.filename) raise SystemExit(1) raise if os.getuid() != 0: # root access required print_error("superuser access required") raise SystemExit(1) if nsargs.community: etpConst['community']['mode'] = True if nsargs.debug: print_warning(repr(nsargs)) # if just one, drop from argv, so that it doesn't interfere with # entropy --debug support if sys.argv.count("--debug") < 2: etpUi['debug'] = False # parse spec files specs = [] for spec_f in nsargs.spec: spec = SpecParser(spec_f) data = spec.parse() if data: specs.append(data) if not specs: print_error("invalid spec files provided") raise SystemExit(1) entropy_server = None exit_st = 0 cwd = os.getcwd() # This application doesn't like implicit inteactivity etpUi['interactive'] = False try: try: entropy_server = get_entropy_server(etpConst['community']['mode']) except PermissionDenied: # repository not available or not configured print_error("no valid server-side repositories configured") raise SystemExit(3) # validate repository entries of spec metadata avail_repos = entropy_server.repositories() for spec in specs: if spec["repository"] not in avail_repos: print_error("invalid repository %s" % (spec["repository"],)) raise SystemExit(10) if nsargs.blocking: print_info("--blocking enabled, please wait for locks...") with EntropyResourceLock(entropy_server, nsargs.blocking): with MatterResourceLock(nsargs.blocking): matter_main(entropy_server, nsargs, cwd, specs) except EntropyResourceLock.NotAcquired: print_error("unable to acquire Entropy Resources lock") raise SystemExit(42) except MatterResourceLock.NotAcquired: print_error("unable to acquire Matter Resources lock") raise SystemExit(42) except KeyboardInterrupt: print_error("Keyboard Interrupt, pid: %s" % (os.getpid(),)) raise SystemExit(100) finally: if entropy_server is not None: entropy_server.shutdown() print_warning("") print_warning("") print_warning("Tasks complete, exit status: %d" % (exit_st,)) raise SystemExit(exit_st)