From eb700e51331e06ad234d6f36708b9ae5fd6abb52 Mon Sep 17 00:00:00 2001 From: Fabio Erculiani Date: Sun, 25 Oct 2009 13:57:13 +0100 Subject: [PATCH] [molecule] complete support for easily remastering Sabayon ISO images --- .../specs/5-x86-g-remaster-add-games.spec | 39 ++- molecule/handlers.py | 36 ++- molecule/settings.py | 23 +- molecule/specs/plugins/builtin.py | 51 ++-- molecule/specs/plugins/remaster.py | 280 +++++++++++++++--- molecule/utils.py | 17 +- 6 files changed, 359 insertions(+), 87 deletions(-) diff --git a/examples/specs/5-x86-g-remaster-add-games.spec b/examples/specs/5-x86-g-remaster-add-games.spec index 1aad4c7..5c8a83c 100644 --- a/examples/specs/5-x86-g-remaster-add-games.spec +++ b/examples/specs/5-x86-g-remaster-add-games.spec @@ -12,7 +12,7 @@ execution_strategy: iso_remaster # have to append "linux32" this is useful for inner_chroot_script prechroot: linux32 -# Path to source ISO file +# Path to source ISO file (MANDATORY) source_iso: /sabayon/iso_images/Sabayon_5.0_G.iso # Outer chroot script command, to be executed outside destination chroot before packing it @@ -29,10 +29,45 @@ extra_mkisofs_parameters: -b isolinux/isolinux.bin -c isolinux/boot.cat # Pre-ISO building script. Hook to be able to copy kernel images in place, for example # pre_iso_script: /sabayon/scripts/cdroot.py -# Destination directory for the ISO image path +# Destination directory for the ISO image path (MANDATORY) destination_iso_directory: /home/fabio # Destination ISO image name, call whatever you want.iso, not mandatory # destination_iso_image_name: +# Output iso image title +# iso_title: +# Alternative ISO file mount command (default is: mount -o loop -t iso9660) +# iso_mounter: + +# Alternative ISO umounter command (default is: umount) +# iso_umounter: + +# Alternative squashfs file mount command (default is: mount -o loop -t squashfs) +# squash_mounter: + +# Alternative ISO squashfs command (default is: umount) +# squash_umounter: + +# Merge directory with destination LiveCD root +# merge_livecd_root: /put/more/files/onto/CD/root + +# List of packages that would be removed from chrooted system +# packages_to_remove: + +# Custom shell call to packages removal (default is: equo install) +# custom_packages_remove_cmd: + +# List of packages that would be added from chrooted system +# packages_to_add: + +# Custom shell call to packages add (default is: equo install) +# custom_packages_add_cmd: + +# Custom command for updating repositories (default is: equo update) +# repositories_update_cmd: + +# Determine whether repositories update should be run (if packages_to_add is set) +# (default is: no), values are: yes, no. +# execute_repositories_update: no diff --git a/molecule/handlers.py b/molecule/handlers.py index cb6f078..556a0ad 100644 --- a/molecule/handlers.py +++ b/molecule/handlers.py @@ -33,7 +33,7 @@ class Runner(GenericExecutionStep): def post_run(self): return 0 - def kill(self): + def kill(self, success = True): return 0 def run(self): @@ -56,19 +56,29 @@ class Runner(GenericExecutionStep): rc = 0 while 1: - # pre-run - rc = my.pre_run() - if rc: break - # run - rc = my.run() - if rc: break - # post-run - rc = my.post_run() - if rc: break - break - my.kill() - if rc: return rc + try: + # pre-run + rc = my.pre_run() + if rc: + break + # run + rc = my.run() + if rc: + break + # post-run + rc = my.post_run() + if rc: + break + + break + except: + my.kill(success = False) + raise + + my.kill(success = rc == 0) + if rc: + return rc self.post_run() self.Output.updateProgress( "[%s|%s] %s" % ( diff --git a/molecule/settings.py b/molecule/settings.py index 0e6d49c..a4b9677 100644 --- a/molecule/settings.py +++ b/molecule/settings.py @@ -16,9 +16,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -from __future__ import with_statement import os -import threading from molecule.exception import SpecFileError from molecule.specs.plugins import SPEC_PLUGS from molecule.specs.plugins.builtin import LivecdSpec @@ -49,24 +47,33 @@ class Configuration(dict): self.Constants = Constants() self.load() - def load(self, mysettings = {}): + def load(self, mysettings = None): + if mysettings is None: + mysettings = {} settings = { 'version': "0.3", 'chroot_compressor': "mksquashfs", 'iso_builder': "mkisofs", 'mirror_syncer': "rsync", 'chroot_compressor_builtin_args': ["-noappend"], - 'iso_builder_builtin_args': ["-J","-R","-l","-no-emul-boot", - "-boot-load-size","4","-udf","-boot-info-table"], - 'mirror_syncer_builtin_args': ["-a","--delete","--delete-excluded", - "--delete-before","--numeric-ids","--recursive","-d","-A","-H"], + 'iso_builder_builtin_args': ["-J", "-R", "-l", "-no-emul-boot", + "-boot-load-size", "4", "-udf", "-boot-info-table"], + 'mirror_syncer_builtin_args': ["-a", "--delete", "--delete-excluded", + "--delete-before", "--numeric-ids", "--recursive", "-d", "-A", "-H"], 'chroot_compressor_output_file': "livecd.squashfs", + 'iso_mounter': ["mount", "-o", "loop", "-t", "iso9660"], + 'iso_umounter': ["umount"], + 'squash_mounter': ["mount", "-o", "loop", "-t", "squashfs"], + 'squash_umounter': ["umount"], + 'pkgs_adder': ["equo", "install"], + 'pkgs_remover': ["equo", "remove"], + 'pkgs_updater': ["equo", "update"], } self.clear() self.update(settings) self.update(mysettings) - paths_to_check = ["chroot_compressor","iso_builder","mirror_syncer"] + paths_to_check = ["chroot_compressor", "iso_builder", "mirror_syncer"] for key in paths_to_check: molecule.utils.valid_exec_check(self.get(key)) diff --git a/molecule/specs/plugins/builtin.py b/molecule/specs/plugins/builtin.py index 3b1d668..0dfd714 100644 --- a/molecule/specs/plugins/builtin.py +++ b/molecule/specs/plugins/builtin.py @@ -55,7 +55,7 @@ class MirrorHandler(GenericExecutionStep): ) return 0 - def kill(self): + def kill(self, success = True): self.Output.updateProgress("[%s|%s] %s" % ( blue("MirrorHandler"),darkred(self.spec_name), _("executing kill"), @@ -154,7 +154,7 @@ class ChrootHandler(GenericExecutionStep): return 0 - def kill(self): + def kill(self, success = True): self.Output.updateProgress("[%s|%s] %s" % ( blue("ChrootHandler"), darkred(self.spec_name),_("executing kill"), @@ -294,7 +294,7 @@ class CdrootHandler(GenericExecutionStep): if os.path.isdir(self.dest_root): molecule.utils.empty_dir(self.dest_root) if not os.path.isdir(self.dest_root): - os.makedirs(self.dest_root,0755) + os.makedirs(self.dest_root, 0755) return 0 @@ -306,7 +306,7 @@ class CdrootHandler(GenericExecutionStep): ) return 0 - def kill(self): + def kill(self, success = True): self.Output.updateProgress("[%s|%s] %s" % ( blue("CdrootHandler"),darkred(self.spec_name), _("executing kill"), @@ -419,6 +419,26 @@ class IsoHandler(GenericExecutionStep): os.path.basename(self.source_chroot) ) + return 0 + + def post_run(self): + self.Output.updateProgress("[%s|%s] %s" % ( + blue("IsoHandler"),darkred(self.spec_name), + _("executing post_run"), + ) + ) + return 0 + + def kill(self, success = True): + self.Output.updateProgress("[%s|%s] %s" % ( + blue("IsoHandler"),darkred(self.spec_name), + _("executing kill"), + ) + ) + return 0 + + def run(self): + # run outer chroot script exec_script = self.metadata.get('pre_iso_script') if exec_script: @@ -439,25 +459,6 @@ class IsoHandler(GenericExecutionStep): ) return rc - return 0 - - def post_run(self): - self.Output.updateProgress("[%s|%s] %s" % ( - blue("IsoHandler"),darkred(self.spec_name), - _("executing post_run"), - ) - ) - return 0 - - def kill(self): - self.Output.updateProgress("[%s|%s] %s" % ( - blue("IsoHandler"),darkred(self.spec_name), - _("executing kill"), - ) - ) - return 0 - - def run(self): self.Output.updateProgress("[%s|%s] %s" % ( blue("IsoHandler"),darkred(self.spec_name), _("building ISO image"), @@ -471,8 +472,8 @@ class IsoHandler(GenericExecutionStep): args.extend(["-V",'"',self.iso_title[:32],'"']) args.extend(['-o',self.dest_iso,self.source_path]) self.Output.updateProgress("[%s|%s] %s: %s" % ( - blue("IsoHandler"),darkred(self.spec_name), - _("spawning"),args, + blue("IsoHandler"), darkred(self.spec_name), + _("spawning"), args, ) ) rc = molecule.utils.exec_cmd(args) diff --git a/molecule/specs/plugins/remaster.py b/molecule/specs/plugins/remaster.py index 097d30a..59b3ab1 100644 --- a/molecule/specs/plugins/remaster.py +++ b/molecule/specs/plugins/remaster.py @@ -16,15 +16,40 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +import os +import tempfile +import shutil + from molecule.i18n import _ from molecule.output import red, brown, blue, green, purple, darkgreen, \ darkred, bold, darkblue, readtext from molecule.specs.skel import GenericExecutionStep, GenericSpec +from molecule.specs.plugins.builtin import ChrootHandler as BuiltinChrootHandler +from molecule.specs.plugins.builtin import CdrootHandler as BuiltinCdrootHandler +from molecule.specs.plugins.builtin import IsoHandler as BuiltinIsoHandler +import molecule.utils class IsoUnpackHandler(GenericExecutionStep): def __init__(self, *args, **kwargs): GenericExecutionStep.__init__(self, *args, **kwargs) + self.tmp_mount = tempfile.mkdtemp(prefix = "molecule", dir = "/var/tmp") + self.tmp_squash_mount = tempfile.mkdtemp(prefix = "molecule", + dir = "/var/tmp") + self.iso_mounted = False + self.squash_mounted = False + + # setup chroot unpack dir + # can't use /tmp because it could be mounted with "special" options + unpack_prefix = tempfile.mkdtemp(prefix = "molecule", dir = "/var/tmp", + suffix = "chroot") + self.metadata['chroot_tmp_dir'] = unpack_prefix + self.metadata['chroot_unpack_path'] = os.path.join(unpack_prefix, + "root") + + # setup upcoming new chroot path + self.dest_root = os.path.join(unpack_prefix, "cdroot") + self.metadata['cdroot_path'] = self.dest_root def pre_run(self): self.Output.updateProgress("[%s|%s] %s" % ( @@ -32,9 +57,84 @@ class IsoUnpackHandler(GenericExecutionStep): _("executing pre_run"), ) ) - # FIXME: make it working + + # setup paths + self.iso_image = self.metadata['source_iso'] + + # mount + mounter = self.metadata.get('iso_mounter', self.Config['iso_mounter']) + mount_args = mounter + [self.iso_image, self.tmp_mount] + self.Output.updateProgress("[%s|%s] %s: %s" % ( + blue("IsoUnpackHandler"), darkred(self.spec_name), + _("spawning"), mount_args, + ) + ) + rc = molecule.utils.exec_cmd(mount_args) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % ( + blue("IsoUnpackHandler"), darkred(self.spec_name), + _("iso mount failed"), rc, + ) + ) + return rc + + # copy iso content over, including squashfs yeah, it will be + # replaced later on + # this is mandatory and used to make iso recreation easier + rc = molecule.utils.copy_dir(self.tmp_mount, self.dest_root) + if rc != 0: + return rc + + self.iso_mounted = True + + # mount squash + mounter = self.metadata.get('squash_mounter', + self.Config['squash_mounter']) + squash_file = os.path.join(self.tmp_mount, + self.Config['chroot_compressor_output_file']) + mount_args = mounter + [squash_file, self.tmp_squash_mount] + self.Output.updateProgress("[%s|%s] %s: %s" % ( + blue("IsoUnpackHandler"), darkred(self.spec_name), + _("spawning"), mount_args, + ) + ) + rc = molecule.utils.exec_cmd(mount_args) + if rc != 0: + self.Output.updateProgress("[%s|%s] %s: %s" % ( + blue("IsoUnpackHandler"), darkred(self.spec_name), + _("squash mount failed"), rc, + ) + ) + return rc + self.squash_mounted = True + return 0 + def run(self): + + self.Output.updateProgress("[%s|%s] %s: %s => %s" % ( + blue("IsoUnpackHandler"), darkred(self.spec_name), + _("iso unpacker running"), self.tmp_squash_mount, + self.metadata['chroot_unpack_path'], + ) + ) + + def dorm(): + shutil.rmtree(self.metadata['chroot_unpack_path'], True) + + # create chroot path + try: + rc = molecule.utils.copy_dir(self.tmp_squash_mount, + self.metadata['chroot_unpack_path']) + except: + dorm() + raise + + if rc != 0: + dorm() + + return rc + def post_run(self): self.Output.updateProgress("[%s|%s] %s" % ( blue("IsoUnpackHandler"),darkred(self.spec_name), @@ -43,34 +143,60 @@ class IsoUnpackHandler(GenericExecutionStep): ) return 0 - def kill(self): + def kill(self, success = True): self.Output.updateProgress("[%s|%s] %s" % ( blue("IsoUnpackHandler"),darkred(self.spec_name), _("executing kill"), ) ) + + rc = 0 + if self.squash_mounted: + umounter = self.metadata.get('squash_umounter', + self.Config['squash_umounter']) + args = umounter + [self.tmp_squash_mount] + rc = molecule.utils.exec_cmd(args) + + if rc == 0: + try: + os.rmdir(self.tmp_squash_mount) + except OSError: + pass + + rc = 0 + if self.iso_mounted: + umounter = self.metadata.get('iso_umounter', + self.Config['iso_umounter']) + args = umounter + [self.tmp_mount] + rc = molecule.utils.exec_cmd(args) + + if rc == 0: + try: + os.rmdir(self.tmp_mount) + except OSError: + # if not empty, skip + pass + + if not success: + try: + shutil.rmtree(self.metadata['chroot_unpack_path'], True) + except (shutil.Error, OSError,): + pass + return 0 - def run(self): - self.Output.updateProgress("[%s|%s] %s" % ( - blue("IsoUnpackHandler"),darkred(self.spec_name), - _("iso unpacker running"), - ) - ) - return 0 - -from molecule.specs.plugins.builtin import ChrootHandler as BuiltinChrootHandler class ChrootHandler(BuiltinChrootHandler): - # INFO: Make external chroot, internal chroot hooks execution working - def pre_run(self): self.Output.updateProgress("[%s|%s] %s" % ( blue("ChrootHandler"),darkred(self.spec_name), _("executing pre_run"), ) ) + # to make superclass working + self.source_dir = self.metadata['chroot_unpack_path'] + self.dest_dir = self.source_dir return 0 def post_run(self): @@ -88,34 +214,60 @@ class ChrootHandler(BuiltinChrootHandler): _("hooks running"), ) ) - # FIXME: make parent method working + + packages_to_add = self.metadata.get('packages_to_add', []) + if packages_to_add: + + # update repos first? + do_update = self.metadata.get('execute_repositories_update', 'no') + if do_update == 'yes': + update_cmd = self.metadata.get('repositories_update_cmd', + self.Config['pkgs_updater']) + rc = molecule.utils.exec_chroot_cmd(update_cmd, + self.source_dir, + self.metadata.get('prechroot',[])) + if rc != 0: + return rc + + add_cmd = self.metadata.get('custom_packages_add_cmd', + self.Config['pkgs_adder']) + args = add_cmd + packages_to_add + rc = molecule.utils.exec_chroot_cmd(args, + self.source_dir, + self.metadata.get('prechroot',[])) + if rc != 0: + return rc + + packages_to_remove = self.metadata.get('packages_to_remove', []) + if packages_to_remove: + rm_cmd = self.metadata.get('custom_packages_remove_cmd', + self.Config['pkgs_remover']) + args = rm_cmd + packages_to_remove + rc = molecule.utils.exec_chroot_cmd(args, + self.source_dir, + self.metadata.get('prechroot',[])) + if rc != 0: + return rc + + rc = BuiltinChrootHandler.run(self) + if rc != 0: + return rc + return 0 -from molecule.specs.plugins.builtin import CdrootHandler as BuiltinCdrootHandler class CdrootHandler(BuiltinCdrootHandler): - # INFO: create cdroot dir back, compress chroot - def pre_run(self): self.Output.updateProgress("[%s|%s] %s" % ( blue("CdrootHandler"),darkred(self.spec_name), _("executing pre_run"), ) ) - # FIXME: make parent method working + self.dest_root = self.metadata['cdroot_path'] + self.source_chroot = self.metadata['chroot_unpack_path'] return 0 - def run(self): - self.Output.updateProgress("[%s|%s] %s" % ( - blue("CdrootHandler"),darkred(self.spec_name), - _("compressing chroot"), - ) - ) - # FIXME: make parent method working - return 0 - -from molecule.specs.plugins.builtin import IsoHandler as BuiltinIsoHandler class IsoHandler(BuiltinIsoHandler): def pre_run(self): @@ -124,16 +276,13 @@ class IsoHandler(BuiltinIsoHandler): _("executing pre_run"), ) ) - # FIXME: make parent method working - return 0 - - def run(self): - self.Output.updateProgress("[%s|%s] %s" % ( - blue("IsoHandler"),darkred(self.spec_name), - _("building ISO image"), - ) - ) - # FIXME: make parent method working + # cdroot dir + self.source_path = self.metadata['cdroot_path'] + self.dest_iso = os.path.join(self.metadata['destination_iso_directory'], + "remaster_" + os.path.basename(self.metadata['source_iso'])) + self.iso_title = self.metadata.get('iso_title', 'Molecule remaster') + self.source_chroot = self.metadata['chroot_unpack_path'] + self.chroot_dir = self.source_chroot return 0 class RemasterSpec(GenericSpec): @@ -158,6 +307,10 @@ class RemasterSpec(GenericSpec): 'cb': self.valid_path_string, 've': self.ve_string_stripper, }, + 'iso_title': { + 'cb': self.ne_string, + 've': self.ve_string_stripper, + }, 'outer_chroot_script': { 'cb': self.valid_exec, 've': self.ve_string_stripper, @@ -183,6 +336,57 @@ class RemasterSpec(GenericSpec): 'cb': self.valid_ascii, 've': self.ve_string_stripper, }, + 'iso_mounter': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'iso_umounter': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'squash_mounter': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'squash_umounter': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'merge_livecd_root': { + 'cb': self.valid_dir, + 've': self.ve_string_stripper, + }, + 'custom_packages_remove_cmd': { + 'cb': self.valid_ascii, + 've': self.ve_string_stripper, + }, + 'custom_packages_add_cmd': { + 'cb': self.valid_ascii, + 've': self.ve_string_stripper, + }, + 'packages_to_remove': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'packages_to_add': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'repositories_update_cmd': { + 'cb': self.ne_list, + 've': self.ve_string_splitter, + 'mod': self.ve_string_splitter, + }, + 'execute_repositories_update': { + 'cb': self.valid_ascii, + 've': self.ve_string_stripper, + }, } def get_execution_steps(self): diff --git a/molecule/utils.py b/molecule/utils.py index 5488015..dc14031 100644 --- a/molecule/utils.py +++ b/molecule/utils.py @@ -1,5 +1,5 @@ #!/usr/bin/python -O -# -*- coding: iso-8859-1 -*- +# -*- coding: utf-8 -*- # Molecule Disc Image builder for Sabayon Linux # Copyright (C) 2009 Fabio Erculiani # @@ -95,3 +95,18 @@ def md5sum(filepath): block = readfile.read(1024) readfile.close() return m.hexdigest() + +def copy_dir(src_dir, dest_dir): + args = ["cp", "-Rap", src_dir, dest_dir] + return exec_cmd(args) + +def print_traceback(f = None): + """ + Function called when an exception occurs with the aim to give + user a clue of what went wrong. + + @keyword f: write to f (file) object instead of stdout + @type f: valid file handle + """ + import traceback + traceback.print_exc(file = f) \ No newline at end of file