Molecule:

- initial code in place


git-svn-id: http://svn.sabayonlinux.org/projects/molecule/trunk@2982 cd1c1023-2f26-0410-ae45-c471fc1f0318

examples/Sabayon4-x86.spec

@@ -25,6 +25,9 @@
 # Merge directory with destination chroot
 # merge_destination_chroot: /path/to/your/chroot/overlay
 
+# Extra mirror (r)sync parameters
+# extra_rsync_parameters:
+
 # Outer chroot script command, to be executed outside destination chroot before packing it
 # outer_chroot_script: /path/to/script/to/be/executed/outside
 
@@ -50,8 +53,8 @@
 # destination_iso_image_name:
 
 # Directories to remove completely (comma separated)
-# directories_to_remove:  /path/to/abc, /path/to/xyz,
+# paths_to_remove:  /path/to/abc, /path/to/xyz,
 
 # Directories to empty (comma separated)
-# directories_to_empty:  /path/to/abc, /path/to/xyz,
+# paths_to_empty:  /path/to/abc, /path/to/xyz,
 

molecule/handlers.py

@@ -19,9 +19,11 @@
 
 from __future__ import with_statement
 import os
+import shutil
 from molecule.i18n import _
 from molecule.output import red, brown, blue, green, purple, darkgreen, darkred, bold, darkblue, readtext
 from molecule.exception import EnvironmentError, NotImplementedError
+import molecule.utils
 
 # This is an interface that has to be reimplemented in order to make it doing something useful
 class GenericHandlerInterface:
@@ -52,25 +54,38 @@ class MirrorHandler(GenericHandlerInterface):
         GenericHandlerInterface.__init__(self, *args, **kwargs)
 
     def pre_run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing pre_run"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing pre_run"),))
         return 0
 
     def post_run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing post_run"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing post_run"),))
         return 0
 
     def kill(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing kill"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing kill"),))
         return 0
 
     def run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("executing run"),))
-        # running the syncer
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring running"),))
+        # creating destination chroot dir
         source_dir = self.metadata['source_chroot']
         dest_dir = os.path.join(self.metadata['destination_chroot'],os.path.basename(source_dir))
         if not os.path.isdir(dest_dir):
             os.makedirs(dest_dir,0755)
-        # FIXME: complete this
+
+        # running sync
+        args = [self.Config['mirror_syncer']]
+        args.extend(self.Config['mirror_syncer_builtin_args'])
+        args.extend(self.metadata.get('extra_rsync_parameters',[]))
+        args.extend([source_dir,dest_dir])
+        self.Output.updateProgress("[%s|%s] %s: %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("spawning"),[exec_script],))
+        rc = molecule.utils.exec_cmd([exec_script])
+        if rc != 0:
+            self.Output.updateProgress("[%s|%s] %s: %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring failed"),rc,))
+            return rc
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("MirrorHandler"),darkred(self.spec_name),_("mirroring completed successfully"),))
         return 0
 
 class ChrootHandler(GenericHandlerInterface):
@@ -79,19 +94,190 @@ class ChrootHandler(GenericHandlerInterface):
         GenericHandlerInterface.__init__(self, *args, **kwargs)
 
     def pre_run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing pre_run"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing pre_run"),))
         return 0
 
     def post_run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing post_run"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing post_run"),))
         return 0
 
     def kill(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing kill"),))
+        self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing kill"),))
         return 0
 
     def run(self):
-        self.Output.updateProgress( "[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("executing run"),))
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("hooks running"),))
+
+        source_dir = self.metadata['source_chroot']
+        dest_dir = os.path.join(self.metadata['destination_chroot'],os.path.basename(source_dir))
+        # run inner chroot script
+        exec_script = self.metadata.get('inner_chroot_script')
+        if exec_script:
+            while 1:
+                tmp_dir = os.path.join(dest_dir,molecule.utils.get_random_number())
+                if not os.path.lexists(tmp_dir): break
+            os.makedirs(tmp_dir)
+            tmp_exec = os.path.join(tmp_dir,"inner_exec")
+            if os.path.isfile(exec_script) and os.access(exec_script,os.R_OK):
+                shutil.copy2(exec_script,tmp_exec)
+                os.chmod(tmp_exec,0755)
+                dest_exec = tmp_exec[len(dest_dir):]
+                if not dest_exec.startswith("/"):
+                    dest_exec = "/%s" % (dest_exec,)
+                rc = molecule.utils.exec_chroot_cmd([dest_exec], dest_dir)
+                if rc != 0:
+                    self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("inner chroot hook failed"),rc,))
+                    return rc
+
+        # run outer chroot script
+        exec_script = self.metadata.get('outer_chroot_script')
+        if exec_script:
+            self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("spawning"),[exec_script],))
+            rc = molecule.utils.exec_cmd([exec_script])
+            if rc != 0:
+                self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("outer chroot hook failed"),rc,))
+                return rc
+
+        # now remove paths to empty
+        empty_paths = self.metadata.get('paths_to_empty',[])
+        for mypath in empty_paths:
+            mypath = os.path.join(dest_dir,mypath)
+            if os.path.isdir(mypath):
+                molecule.utils.empty_dir(mypath)
+
+        # now remove paths to remove (...)
+        remove_paths = self.metadata.get('paths_to_remove',[])
+        for mypath in remove_paths:
+            mypath = os.path.join(dest_dir,mypath)
+            if not os.path.lexists(mypath): continue
+            rc = molecule.utils.remove_path(mypath)
+            if rc != 0:
+                self.Output.updateProgress("[%s|%s] %s: %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("removal failed for"),mypath,rc,))
+                return rc
+
+        # write release file
+        release_file = self.metadata.get('release_file')
+        if release_file:
+            release_file = os.path.join(dest_dir,release_file)
+            if os.path.lexists(release_file) and not os.path.isfile(release_file):
+                self.Output.updateProgress("[%s|%s] %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("release file creation failed, not a file"),release_file,))
+                return 1
+            release_string = self.metadata.get('release_string','')
+            release_version = self.metadata.get('release_version','')
+            release_desc = self.metadata.get('release_desc','')
+            file_string = "%s %s %s\n" % (release_string,release_version,release_desc,)
+            try:
+                f = open(release_file,"w")
+                f.write(file_string)
+                f.flush()
+                f.close()
+            except (IOError,OSError,), e:
+                self.Output.updateProgress("[%s|%s] %s: %s: %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("release file creation failed, system error"),release_file,e,))
+                return 1
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("ChrootHandler"),darkred(self.spec_name),_("hooks completed succesfully"),))
+        return 0
+
+class CdrootHandler(GenericHandlerInterface):
+
+    def __init__(self, *args, **kwargs):
+        GenericHandlerInterface.__init__(self, *args, **kwargs)
+
+    def pre_run(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing pre_run"),))
+        return 0
+
+    def post_run(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing post_run"),))
+        return 0
+
+    def kill(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("executing kill"),))
+        return 0
+
+    def run(self):
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("preparing environment"),))
+        source_chroot = os.path.join(self.metadata['destination_chroot'],os.path.basename(self.metadata['source_chroot']))
+        dest_root = self.Config['destination_livecd_root']
+        if os.path.isdir(dest_root):
+            molecule.utils.empty_dir(dest_root)
+        if not os.path.isdir(dest_root):
+            os.makedirs(dest_root,0755)
+
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("compressing chroot"),))
+        args = [self.Config['chroot_compressor']]
+        args.extend(self.Config['chroot_compressor_builtin_args'])
+        args.extend(self.metadata.get('extra_mksquashfs_parameters',[]))
+        comp_output = self.Config['chroot_compressor_output_file']
+        if "chroot_compressor_output_file" in self.metadata:
+            comp_output = self.metadata.get('chroot_compressor_output_file')
+        args.extend([source_chroot,comp_output])
+        self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("spawning"),args,))
+        rc = molecule.utils.exec_cmd(args)
+        if rc != 0:
+            self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("chroot compression failed"),rc,))
+            return rc
+        self.Output.updateProgress("[%s|%s] %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("chroot compressed successfully"),))
+
+        # merge dir
+        merge_dir = self.Config['merge_livecd_root']
+        self.Output.updateProgress("[%s|%s] %s %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("merging livecd root"),merge_dir,))
+        try:
+            shutil.copytree(merge_dir,dest_root)
+        except shutil.error, e:
+            self.Output.updateProgress("[%s|%s] %s: %s" % (blue("CdrootHandler"),darkred(self.spec_name),_("error during livecd root copy"),e,))
+            return 1
+
+        return 0
+
+class IsoHandler(GenericHandlerInterface):
+
+    def __init__(self, *args, **kwargs):
+        GenericHandlerInterface.__init__(self, *args, **kwargs)
+
+    def pre_run(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing pre_run"),))
+        return 0
+
+    def post_run(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing post_run"),))
+        return 0
+
+    def kill(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("executing kill"),))
+        return 0
+
+    def run(self):
+        self.Output.updateProgress("[%s|%s] %s" % (blue("IsoHandler"),darkred(self.spec_name),_("building ISO image"),))
+
+        # setup paths
+        source_path = self.Config['destination_livecd_root']
+        dest_iso_dir = self.metadata['destination_iso_directory']
+        if not os.path.isdir(dest_iso_dir):
+            os.makedirs(dest_iso_dir,0755)
+        dest_iso_filename = self.metadata.get('destination_iso_image_name')
+        if not dest_iso_filename:
+            dest_iso_filename = "%s_%s_%s.iso" (
+                self.metadata.get('release_string','').replace(' ','_'),
+                self.metadata.get('release_version','').replace(' ','_'),
+                self.metadata.get('release_desc','').replace(' ','_'),
+            )
+        dest_iso = os.path.join(dest_iso_dir,dest_iso_filename)
+
+        # build !
+        args = [self.Config['iso_builder']]
+        args.extend(self.Config['iso_builder_builtin_args'])
+        args.extend(self.metadata.get('extra_mkisofs_parameters',[]))
+        args.extend(['-o',dest_iso,source_path])
+        self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("spawning"),args,))
+        rc = molecule.utils.exec_cmd(args)
+        if rc != 0:
+            self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("ISO image build failed"),rc,))
+            return rc
+
+        self.Output.updateProgress("[%s|%s] %s: %s" % (blue("IsoHandler"),darkred(self.spec_name),_("built ISO image"),dest_iso,))
         return 0
 
 
@@ -99,7 +285,7 @@ class Runner(GenericHandlerInterface):
 
     def __init__(self, spec_path, metadata):
         GenericHandlerInterface.__init__(self, spec_path, metadata)
-        self.execution_order = [MirrorHandler,ChrootHandler]
+        self.execution_order = [MirrorHandler,ChrootHandler,CdrootHandler,IsoHandler]
 
     def pre_run(self):
         return 0
@@ -139,8 +325,5 @@ class Runner(GenericHandlerInterface):
             if rc: return rc
 
         self.post_run()
-        self.Output.updateProgress( "[%s|%s] %s!" % (darkgreen("Runner"),brown(self.spec_name),_("All done"),))
+        self.Output.updateProgress( "[%s|%s] %s" % (darkgreen("Runner"),brown(self.spec_name),_("All done"),))
         return 0
-
-
-

molecule/settings.py

@@ -97,7 +97,10 @@ class Configuration(Dictionary):
                 'chroot_compressor': "mksquashfs",
                 'iso_builder': "mkisofs",
                 'mirror_syncer': "rsync",
+                'chroot_compressor_builtin_args': ["-noappend"],
                 'iso_builder_builtin_args': ["-J","-R","-l","-no-emul-boot","-boot-load-size","4","-udf","-boot-info-table"],
+                'mirror_syncer_builtin_args': ["-a","--delete","--delete-excluded","--numeric-ids"],
+                'chroot_compressor_output_file': "livecd.squashfs",
             }
             self._settings.clear()
             self._settings.update(settings)
@@ -187,6 +190,10 @@ class SpecParser:
                 'cb': valid_path_string,
                 've': ve_string_stripper,
             },
+            'extra_rsync_parameters': {
+                'cb': valid_path_string,
+                've': ve_string_splitter,
+            },
             'merge_destination_chroot': {
                 'cb': valid_dir,
                 've': ve_string_stripper,
@@ -223,11 +230,11 @@ class SpecParser:
                 'cb': valid_ascii,
                 've': ve_string_stripper,
             },
-            'directories_to_remove': {
+            'paths_to_remove': {
                 'cb': valid_path_list,
                 've': ve_string_stripper,
             },
-            'directories_to_empty': {
+            'paths_to_empty': {
                 'cb': valid_path_list,
                 've': ve_string_stripper,
             },

molecule/utils.py

@@ -42,3 +42,37 @@ def is_exec_available(exec_name):
         if os.path.isfile(path) and os.access(path,os.X_OK):
             return True
     return False
+
+def exec_cmd(args):
+    p = subprocess.Popen(args)
+    rc = p.wait()
+    return rc
+
+def exec_chroot_cmd(args, chroot):
+    pid = os.fork()
+    if pid == 0:
+        os.chroot(chroot)
+        os.chdir("/")
+        p = subprocess.Popen(args)
+        rc = p.wait()
+        os._exit(rc)
+    else:
+        pid, status = os.waitpid(pid,0)
+        return status
+
+def empty_dir(dest_dir):
+    for el in os.listdir(dest_dir):
+        el = os.path.join(dest_dir,el)
+        if os.path.isfile(el) or os.path.islink(el):
+            os.remove(el)
+        elif os.path.isdir(el):
+            shutil.rmtree(el,True)
+            if os.path.isdir(el):
+                os.rmdir(el)
+
+# using subprocess.call to not care about wildcards
+def remove_path(path):
+    return subprocess.call(["rm","-rf",path])
+
+def get_random_number():
+    return abs(hash(os.urandom(2)))%99999