From 3659c5db7155773d7940cb03ef5a4b6d91dedb6b Mon Sep 17 00:00:00 2001 From: lxnay Date: Tue, 3 Feb 2009 07:34:12 +0000 Subject: [PATCH] Molecule: - initial code in place git-svn-id: http://svn.sabayonlinux.org/projects/molecule/trunk@2982 cd1c1023-2f26-0410-ae45-c471fc1f0318 --- examples/Sabayon4-x86.spec | 7 +- molecule/handlers.py | 213 ++++++++++++++++++++++++++++++++++--- molecule/settings.py | 11 +- molecule/utils.py | 34 ++++++ 4 files changed, 246 insertions(+), 19 deletions(-) diff --git a/examples/Sabayon4-x86.spec b/examples/Sabayon4-x86.spec index bb98e0c..97eaa75 100644 --- a/examples/Sabayon4-x86.spec +++ b/examples/Sabayon4-x86.spec @@ -25,6 +25,9 @@ # Merge directory with destination chroot # merge_destination_chroot: /path/to/your/chroot/overlay +# Extra mirror (r)sync parameters +# extra_rsync_parameters: + # Outer chroot script command, to be executed outside destination chroot before packing it # outer_chroot_script: /path/to/script/to/be/executed/outside @@ -50,8 +53,8 @@ # destination_iso_image_name: # Directories to remove completely (comma separated) -# directories_to_remove: /path/to/abc, /path/to/xyz, +# paths_to_remove: /path/to/abc, /path/to/xyz, # Directories to empty (comma separated) -# directories_to_empty: /path/to/abc, /path/to/xyz, +# paths_to_empty: /path/to/abc, /path/to/xyz, diff --git a/molecule/handlers.py b/molecule/handlers.py index e5fe93f..c7df3fe 100644 --- a/molecule/handlers.py +++ b/molecule/handlers.py @@ -19,9 +19,11 @@ from __future__ import with_statement import os +import shutil from molecule.i18n import _ from molecule.output import red, brown, blue, green, purple, darkgreen, darkred, bold, darkblue, readtext from molecule.exception import EnvironmentError, NotImplementedError +import molecule.utils # This is an interface that has to be reimplemented in order to make it doing something useful class GenericHandlerInterface: @@ -52,25 +54,38 @@ class MirrorHandler(GenericHandlerInterface): GenericHandlerInterface.__init__(self, *args, **kwargs) def pre_run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing pre_run"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing pre_run"),)) return 0 def post_run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing post_run"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing post_run"),)) return 0 def kill(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing kill"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing kill"),)) return 0 def run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing run"),)) - # running the syncer + + self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring running"),)) + # creating destination chroot dir source_dir = self.metadata['source_chroot'] dest_dir = os.path.join(self.metadata['destination_chroot'],os.path.basename(source_dir)) if not os.path.isdir(dest_dir): os.makedirs(dest_dir,0755) - # FIXME: complete this + + # running sync + args = [self.Config['mirror_syncer']] + args.extend(self.Config['mirror_syncer_builtin_args']) + args.extend(self.metadata.get('extra_rsync_parameters',[])) + args.extend([source_dir,dest_dir]) + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("spawning"),[exec_script],)) + rc = molecule.utils.exec_cmd([exec_script]) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring failed"),rc,)) + return rc + + self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring completed successfully"),)) return 0 class ChrootHandler(GenericHandlerInterface): @@ -79,19 +94,190 @@ class ChrootHandler(GenericHandlerInterface): GenericHandlerInterface.__init__(self, *args, **kwargs) def pre_run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing pre_run"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing pre_run"),)) return 0 def post_run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing post_run"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing post_run"),)) return 0 def kill(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing kill"),)) + self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing kill"),)) return 0 def run(self): - self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing run"),)) + + self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("hooks running"),)) + + source_dir = self.metadata['source_chroot'] + dest_dir = os.path.join(self.metadata['destination_chroot'],os.path.basename(source_dir)) + # run inner chroot script + exec_script = self.metadata.get('inner_chroot_script') + if exec_script: + while 1: + tmp_dir = os.path.join(dest_dir,molecule.utils.get_random_number()) + if not os.path.lexists(tmp_dir): break + os.makedirs(tmp_dir) + tmp_exec = os.path.join(tmp_dir,"inner_exec") + if os.path.isfile(exec_script) and os.access(exec_script,os.R_OK): + shutil.copy2(exec_script,tmp_exec) + os.chmod(tmp_exec,0755) + dest_exec = tmp_exec[len(dest_dir):] + if not dest_exec.startswith("/"): + dest_exec = "/%s" % (dest_exec,) + rc = molecule.utils.exec_chroot_cmd([dest_exec], dest_dir) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("inner chroot hook failed"),rc,)) + return rc + + # run outer chroot script + exec_script = self.metadata.get('outer_chroot_script') + if exec_script: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("spawning"),[exec_script],)) + rc = molecule.utils.exec_cmd([exec_script]) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("outer chroot hook failed"),rc,)) + return rc + + # now remove paths to empty + empty_paths = self.metadata.get('paths_to_empty',[]) + for mypath in empty_paths: + mypath = os.path.join(dest_dir,mypath) + if os.path.isdir(mypath): + molecule.utils.empty_dir(mypath) + + # now remove paths to remove (...) + remove_paths = self.metadata.get('paths_to_remove',[]) + for mypath in remove_paths: + mypath = os.path.join(dest_dir,mypath) + if not os.path.lexists(mypath): continue + rc = molecule.utils.remove_path(mypath) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("removal failed for"),mypath,rc,)) + return rc + + # write release file + release_file = self.metadata.get('release_file') + if release_file: + release_file = os.path.join(dest_dir,release_file) + if os.path.lexists(release_file) and not os.path.isfile(release_file): + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("release file creation failed, not a file"),release_file,)) + return 1 + release_string = self.metadata.get('release_string','') + release_version = self.metadata.get('release_version','') + release_desc = self.metadata.get('release_desc','') + file_string = "%s %s %s\n" % (release_string,release_version,release_desc,) + try: + f = open(release_file,"w") + f.write(file_string) + f.flush() + f.close() + except (IOError,OSError,), e: + self.Output.updateProgress("[%s|%s] %s: %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("release file creation failed, system error"),release_file,e,)) + return 1 + + self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("hooks completed succesfully"),)) + return 0 + +class CdrootHandler(GenericHandlerInterface): + + def __init__(self, *args, **kwargs): + GenericHandlerInterface.__init__(self, *args, **kwargs) + + def pre_run(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing pre_run"),)) + return 0 + + def post_run(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing post_run"),)) + return 0 + + def kill(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing kill"),)) + return 0 + + def run(self): + + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("preparing environment"),)) + source_chroot = os.path.join(self.metadata['destination_chroot'],os.path.basename(self.metadata['source_chroot'])) + dest_root = self.Config['destination_livecd_root'] + if os.path.isdir(dest_root): + molecule.utils.empty_dir(dest_root) + if not os.path.isdir(dest_root): + os.makedirs(dest_root,0755) + + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("compressing chroot"),)) + args = [self.Config['chroot_compressor']] + args.extend(self.Config['chroot_compressor_builtin_args']) + args.extend(self.metadata.get('extra_mksquashfs_parameters',[])) + comp_output = self.Config['chroot_compressor_output_file'] + if "chroot_compressor_output_file" in self.metadata: + comp_output = self.metadata.get('chroot_compressor_output_file') + args.extend([source_chroot,comp_output]) + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("spawning"),args,)) + rc = molecule.utils.exec_cmd(args) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("chroot compression failed"),rc,)) + return rc + self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("chroot compressed successfully"),)) + + # merge dir + merge_dir = self.Config['merge_livecd_root'] + self.Output.updateProgress("[%s|%s] %s %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("merging livecd root"),merge_dir,)) + try: + shutil.copytree(merge_dir,dest_root) + except shutil.error, e: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("error during livecd root copy"),e,)) + return 1 + + return 0 + +class IsoHandler(GenericHandlerInterface): + + def __init__(self, *args, **kwargs): + GenericHandlerInterface.__init__(self, *args, **kwargs) + + def pre_run(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing pre_run"),)) + return 0 + + def post_run(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing post_run"),)) + return 0 + + def kill(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing kill"),)) + return 0 + + def run(self): + self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("building ISO image"),)) + + # setup paths + source_path = self.Config['destination_livecd_root'] + dest_iso_dir = self.metadata['destination_iso_directory'] + if not os.path.isdir(dest_iso_dir): + os.makedirs(dest_iso_dir,0755) + dest_iso_filename = self.metadata.get('destination_iso_image_name') + if not dest_iso_filename: + dest_iso_filename = "%s_%s_%s.iso" ( + self.metadata.get('release_string','').replace(' ','_'), + self.metadata.get('release_version','').replace(' ','_'), + self.metadata.get('release_desc','').replace(' ','_'), + ) + dest_iso = os.path.join(dest_iso_dir,dest_iso_filename) + + # build ! + args = [self.Config['iso_builder']] + args.extend(self.Config['iso_builder_builtin_args']) + args.extend(self.metadata.get('extra_mkisofs_parameters',[])) + args.extend(['-o',dest_iso,source_path]) + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("spawning"),args,)) + rc = molecule.utils.exec_cmd(args) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("ISO image build failed"),rc,)) + return rc + + self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("built ISO image"),dest_iso,)) return 0 @@ -99,7 +285,7 @@ class Runner(GenericHandlerInterface): def __init__(self, spec_path, metadata): GenericHandlerInterface.__init__(self, spec_path, metadata) - self.execution_order = [MirrorHandler,ChrootHandler] + self.execution_order = [MirrorHandler,ChrootHandler,CdrootHandler,IsoHandler] def pre_run(self): return 0 @@ -139,8 +325,5 @@ class Runner(GenericHandlerInterface): if rc: return rc self.post_run() - self.Output.updateProgress( "[%s|%s] %s!" % (darkgreen("Runner"),brown(self.spec_name),_("All done"),)) + self.Output.updateProgress( "[%s|%s] %s" % (darkgreen("Runner"),brown(self.spec_name),_("All done"),)) return 0 - - - diff --git a/molecule/settings.py b/molecule/settings.py index 33998be..6292608 100644 --- a/molecule/settings.py +++ b/molecule/settings.py @@ -97,7 +97,10 @@ class Configuration(Dictionary): 'chroot_compressor': "mksquashfs", 'iso_builder': "mkisofs", 'mirror_syncer': "rsync", + 'chroot_compressor_builtin_args': ["-noappend"], 'iso_builder_builtin_args': ["-J","-R","-l","-no-emul-boot","-boot-load-size","4","-udf","-boot-info-table"], + 'mirror_syncer_builtin_args': ["-a","--delete","--delete-excluded","--numeric-ids"], + 'chroot_compressor_output_file': "livecd.squashfs", } self._settings.clear() self._settings.update(settings) @@ -187,6 +190,10 @@ class SpecParser: 'cb': valid_path_string, 've': ve_string_stripper, }, + 'extra_rsync_parameters': { + 'cb': valid_path_string, + 've': ve_string_splitter, + }, 'merge_destination_chroot': { 'cb': valid_dir, 've': ve_string_stripper, @@ -223,11 +230,11 @@ class SpecParser: 'cb': valid_ascii, 've': ve_string_stripper, }, - 'directories_to_remove': { + 'paths_to_remove': { 'cb': valid_path_list, 've': ve_string_stripper, }, - 'directories_to_empty': { + 'paths_to_empty': { 'cb': valid_path_list, 've': ve_string_stripper, }, diff --git a/molecule/utils.py b/molecule/utils.py index cc3d583..6e177de 100644 --- a/molecule/utils.py +++ b/molecule/utils.py @@ -42,3 +42,37 @@ def is_exec_available(exec_name): if os.path.isfile(path) and os.access(path,os.X_OK): return True return False + +def exec_cmd(args): + p = subprocess.Popen(args) + rc = p.wait() + return rc + +def exec_chroot_cmd(args, chroot): + pid = os.fork() + if pid == 0: + os.chroot(chroot) + os.chdir("/") + p = subprocess.Popen(args) + rc = p.wait() + os._exit(rc) + else: + pid, status = os.waitpid(pid,0) + return status + +def empty_dir(dest_dir): + for el in os.listdir(dest_dir): + el = os.path.join(dest_dir,el) + if os.path.isfile(el) or os.path.islink(el): + os.remove(el) + elif os.path.isdir(el): + shutil.rmtree(el,True) + if os.path.isdir(el): + os.rmdir(el) + +# using subprocess.call to not care about wildcards +def remove_path(path): + return subprocess.call(["rm","-rf",path]) + +def get_random_number(): + return abs(hash(os.urandom(2)))%99999