From 9fb1cd3e0223fbc8d4c150085c453455f420cf4c Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Fri, 27 Jan 2012 01:04:47 +0100 Subject: [PATCH] - moved builder frome itwatch to devtools - restructured, added variable handling, callbacks, .... --- bin/builder.sh | 70 +++++++++++ conf/build-default.cfg | 41 +++++++ doc/README-builder.txt | 29 +++++ lib/builder-targets.sh | 231 +++++++++++++++++++++++++++++++++++ lib/builder-utils.sh | 56 +++++++++ sample/builder-product.cfg | 41 +++++++ sample/builder-targets-cb.sh | 52 ++++++++ 7 files changed, 520 insertions(+) create mode 100755 bin/builder.sh create mode 100644 conf/build-default.cfg create mode 100644 doc/README-builder.txt create mode 100644 lib/builder-targets.sh create mode 100644 lib/builder-utils.sh create mode 100644 sample/builder-product.cfg create mode 100644 sample/builder-targets-cb.sh diff --git a/bin/builder.sh b/bin/builder.sh new file mode 100755 index 0000000..3bc5b7a --- /dev/null +++ b/bin/builder.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# + +# enabled debug +# set -x + +##################### +# Controller +#################### +builder_controller() { + local result + + # read config + call_entry_point result config; (( $result == "0" )) || builder_config + + # prepare + call_entry_point result prepare ; (( $result == "0" )) || builder_prepare + + # download and process dist files + call_entry_point result retrieve; (( $result == "0" )) || builder_retrieve + + # Create some special files + call_entry_point result create; (( $result == "0" )) || builder_create + + # Start packaging + call_entry_point result package; (( $result == "0" )) || builder_package + + # Upload to repos + call_entry_point result publish; (( $result == "0" )) || builder_publish + + # git commit + call_entry_point result commit; (( $result == "0" )) || builder_commit + + # cleanup + call_entry_point result cleanup; (( $result == "0" )) || builder_cleanup +} + +#################### +# Main +#################### + +# get the 'real' directory this program stored in (resolve symbolic links) +PRG=$(readlink -f $0) +BASEDIR=`dirname "$PRG"` +BASEDIR=`cd "$BASEDIR" && pwd`/.. + +# Parameters +PRODUCT_DIR=$1 +TARGET=$2 + +# read libraries +. $BASEDIR/lib/builder-targets.sh + +# check product directory +if [ -z ${PRODUCT_DIR} ]; then PRODUCT_DIR="`pwd`" ; fi +test -d $PRODUCT_DIR +builder_check_error "no opsi product directory specified: $PRODUCT_DIR" + +# source additional, product dependent callback (cb) targets +if [ -f "$PRODUCT_DIR/builder-targets-cb.sh" ] ; then + . "$PRODUCT_DIR/builder-targets-cb.sh" +fi + +# call main +builder_controller + +# exit +exit 0 + diff --git a/conf/build-default.cfg b/conf/build-default.cfg new file mode 100644 index 0000000..1ae753d --- /dev/null +++ b/conf/build-default.cfg @@ -0,0 +1,41 @@ +################################################### +# build-default.cfg +# Default values for OPSI builder system +#################################################### + +################################## +# Global variables +################################# +TMP_DIR=/tmp + +################################## +# OPSI repository settings +################################# +# Basedirectory storing OPSI-packages after building +OPSI_REPOS_BASE_DIR=$HOME/.opsi-repository + +# Directory- and filename pattern +OPSI_REPOS_PRODUCT_DIR=${OPSI_REPOS_BASE_DIR}/${STATUS}/${VENDOR}/${PN}/${VERSION}-${RELEASE} + +# OPSI-package filepattern +OPSI_REPOS_FILE_PATTERN=${PN}_${VERSION}-${RELEASE}.opsi + +# Force alway upload to opsi repos +OPSI_REPOS_FORCE_UPLOAD=1 + + +############################### +# Distribution / Vendor settings +############################### + +# Directory downloading/cacheing the artifacts like MSI package or icons from the vendor webside +DIST_CACHE_DIR=$HOME/.opsi-dist-cache/${PN}-${VERSION} + +# private dist repository. You can use all variables from release.cfg insde the URL +# this optinal URL will be the FIRST url to check for downloading vendor stuff. +# To copy a local file, the repos should start with file:// +DIST_PRIVATE_REPOS=file://$HOME/opsi-dist.private/$VENDOR/$PN/$VERSION + +# Force always downloading dist files +DIST_FORCE_DOWNLOAD=1 + diff --git a/doc/README-builder.txt b/doc/README-builder.txt new file mode 100644 index 0000000..2b2046d --- /dev/null +++ b/doc/README-builder.txt @@ -0,0 +1,29 @@ +############################## +# Prerequirements: +# - Setup your environment variable inside the shell, e.g. ~/.bashrc +# export BUILD_LOCAL_CFG=/home/dschwager/work/build_local.cfg +# check the sample build_local.cfg located inside the itwatch project +############################## + +# requirements + yum install plowshare + yum install ImageMagick + yum install git + +# Define your local, private, individual, not-project dependent build setup +# in the file ~/.builder.cfg OR by using the environment variable BUILD_LOCAL_CFG +# pointing the the configuration. +# export BUILD_LOCAL_CFG=/home/dschwager/work/itwatch/build_local.cfg +# If no files are availble, the default values will be use. + +# Start build + builder.sh + builder.sh /home/dschwager/work/itwatch + + # Force downloading vendor files + DIST_FORCE_DOWNLOAD=1 builder.sh /home/dschwager/work/itwatch + + # Force upload independent of existing OPSI-Package in repository + OPSI_REPOS_FORCE_UPLOAD=1 builder.sh /home/dschwager/work/itwatch + + diff --git a/lib/builder-targets.sh b/lib/builder-targets.sh new file mode 100644 index 0000000..cb74945 --- /dev/null +++ b/lib/builder-targets.sh @@ -0,0 +1,231 @@ +##################### +# Call user entry point +#################### + +# source generic utility functions +. $BASEDIR/lib/builder-utils.sh + +##################### +# Read config +#################### +builder_config() { + # Check temp dir + test -d ${TMP_DIR} + builder_check_error "temp directory not available: $TMP_DIR" + + # Source product release configuration + test -f ${PRODUCT_DIR}/builder-product.cfg + builder_check_error "can't read release configuration: ${PRODUCT_DIR}/product.cfg" + . $PRODUCT_DIR/builder-product.cfg + + # set default build configuration and source the user dependent file + . $BASEDIR/conf/build-default.cfg + + # Source local build configuration (must be done AFTER sourcing the release.cfg) + test -f $HOME/.builder.cfg && . $HOME/.builder.cfg && echo "Loaded builder configuration: $HOME/.builder.cfg" + test -f "$BUILD_LOCAL_CFG" && . $BUILD_LOCAL_CFG && echo "Loaded builder configuration: $BUILD_LOCAL_CFG" + + # Check variables + if [ -z ${OPSI_REPOS_BASE_DIR} ] || [ ! -d ${OPSI_REPOS_BASE_DIR} ] ; then + echo "configuration error: OPSI_REPOS_BASE_DIR directory does not exist: $OPSI_REPOS_BASE_DIR" + exit 2 + fi + +} + +##################### +# Prepare build +#################### +builder_prepare() { + + # Check if the package is still build + if [ -z "$OPSI_REPOS_FORCE_UPLOAD" ] && [ -f ${OPSI_REPOS_PRODUCT_DIR}/${OPSI_REPOS_FILE_PATTERN} ] ; then + echo "ERROR: package ${OPSI_REPOS_PRODUCT_DIR}/${OPSI_REPOS_FILE_PATTERN} already generated" + exit 1 + fi + + mkdir -p $DIST_CACHE_DIR + echo "Distribution directory: $DIST_CACHE_DIR" + + # setup work directory + output_dir=$(mktemp -d $TMP_DIR/opsi-builder.XXXXXXXXXX) || { echo "Failed to create temp dir"; exit 1; } + +} + + +##################### +# Download all dist files from one of the defined URLs. +# and validate the checksum +#################### +builder_retrieve() { + + for (( i = 0 ; i < ${#SOURCE[@]} ; i++ )) ; do + basename=${FILE[$i]} + urls=${SOURCE[$i]} + arch=${ARCH[$i]} + downloaded=0 + + # Add private repos to the urls + if [ ! -z ${DIST_PRIVATE_REPOS} ]; then + urls="${DIST_PRIVATE_REPOS}/$basename;$urls" + fi + + # check existence of CRC file + if [ ! -e ${PRODUCT_DIR}/${basename}.sha1sum ] ; then + echo "You need to create the checksums with: sha1sum ${DIST_CACHE_DIR}/${basename} > ${PRODUCT_DIR}/${basename}.sha1sum" + exit 1 + fi + + echo "Downloading $basename" + # check downloading from the defined URLs + for src in `echo $urls | sed -e 's/;/\n/g'` ; do + if [ $downloaded == 1 ]; then continue; fi + + echo " Info: Downloding from $src" + mkdir -p ${DIST_CACHE_DIR}/$arch + DIST_FILE[$i]=${DIST_CACHE_DIR}/$arch/$basename + retrieve_file $src ${DIST_FILE[$i]} + + if [ $? == 0 ] ; then + # testing the checksum of the downloaded files + SHA1SUM=`cat ${PRODUCT_DIR}/${basename}.sha1sum | cut -d " " -f1` + CHECKSUM=`sha1sum ${DIST_FILE[$i]} | cut -d " " -f1` + if [ "$CHECKSUM" == "$SHA1SUM" ] ; then + downloaded=1 + echo " Info: Downloaded successfully" + else + echo " Error: The checksums do not match - try next URL" + fi + else + echo " Warning: Failed to download file - try next URL" + fi + done + echo + + # Ups - no URL works + if [ $downloaded != 1 ] ; then + echo " Error: can download the file or checksum wrong (sha1sum ${DIST_CACHE_DIR}/${basename} > ${basename}.sha1sum)" + exit 1; + fi + + done +} + +##################### +# Create files +#################### +builder_create() { + + # converting icon file + iconfile=${DIST_FILE[$ICON_FILE_INDEX]} + echo iconfile=$iconfile + convert -colorspace rgb $iconfile -transparent white -background transparent -resize 160x160 \ + -size 160x160 xc:transparent +swap -gravity center -composite $output_dir/$PN.png + builder_check_error "converting image" + +} + +##################### +# build opsi package +##################### +builder_package() { + + # prepare + inst_dir=$output_dir/$PN + mkdir $inst_dir + + # Copy files and convert text files to dos format + cp -Rv ${PRODUCT_DIR}/OPSI $inst_dir + cp -Rv ${PRODUCT_DIR}/CLIENT_DATA $inst_dir + find $inst_dir/CLIENT_DATA -type f | xargs -n1 -iREP sh -c 'file -i $0 | grep "text/plain" && dos2unix $0' REP + + # copy image + cp -a $output_dir/$PN.png $inst_dir/CLIENT_DATA + + # copy binaries + for (( i = 0 ; i < ${#SOURCE[@]} ; i++ )) ; do + distfile=${DIST_FILE[$i]} + mkdir -p $inst_dir/CLIENT_DATA/${ARCH[$i]} + cp ${DIST_FILE[$i]} $inst_dir/CLIENT_DATA/${ARCH[$i]} + done + + # create variables + var_file=$output_dir/variable.ins + echo -n >$var_file + for (( i = 0 ; i < ${#SOURCE[@]} ; i++ )) ; do + if [ -z ${WINST[$i]} ] ; then continue ; fi + if [ ! -z "${ARCH[$i]}" ] ; then arch_str="${ARCH[$i]}\\" ; fi + echo "DefVar \$${WINST[$i]}\$" >>$var_file + echo "Set \$${WINST[$i]}\$ = \"%ScriptPath%\\$arch_str${FILE[$i]}\"" >>$var_file + done + echo >>$var_file + + # add the new vaiables to all *.ins winst files + for ins_file in `find ${PRODUCT_DIR}/CLIENT_DATA -type f -name "*.ins"` ; do + cat $var_file $ins_file >${inst_dir}/CLIENT_DATA/`basename $ins_file` + done + + # replace variables from OPSI control + sed -e "s!VERSION!$VERSION!g" -e "s!RELEASE!$RELEASE!g" -e "s!PRIORITY!$PRIORITY!g" -e "s!ADVICE!$ADVICE!g" ${PRODUCT_DIR}/OPSI/control >$inst_dir/OPSI/control + + # Create changelog based on git - if available + if [ -d "${PRODUCT_DIR}/.git" ] ; then + git log --date-order --date=short | \ + sed -e '/^commit.*$/d' | \ + awk '/^Author/ {sub(/\\$/,""); getline t; print $0 t; next}; 1' | \ + sed -e 's/^Author: //g' | \ + sed -e 's/>Date: \([0-9]*-[0-9]*-[0-9]*\)/>\t\1/g' | \ + sed -e 's/^\(.*\) \(\)\t\(.*\)/\3 \1 \2/g' > $inst_dir/OPSI/changelog.txt + else + echo "No git repository present." + fi + + # Callback + call_entry_point result cb_package_makeproductfile + + # building package + pushd ${output_dir} + rm -f ${PN}_${VERSION}-${RELEASE}.opsi $OPSI_REPOS_FILE_PATTERN + opsi-makeproductfile -v $inst_dir + builder_check_error "Building OPSI-package" + popd + + # rename opsi package file + if [ "${PN}_${VERSION}-${RELEASE}.opsi" != "$OPSI_REPOS_FILE_PATTERN" ]; then + mv ${PN}_${VERSION}-${RELEASE}.opsi $OPSI_REPOS_FILE_PATTERN + fi + +} + +##################### +# build opsi package +##################### +builder_publish() { + + # Upload file to repository + mkdir -p ${OPSI_REPOS_PRODUCT_DIR} + cp $output_dir/$OPSI_REPOS_FILE_PATTERN ${OPSI_REPOS_PRODUCT_DIR}/${OPSI_REPOS_FILE_PATTERN} + builder_check_error "Can't upload file $output_dir/$OPSI_REPOS_FILE_PATTERN to ${OPSI_REPOS_PRODUCT_DIR}/${OPSI_REPOS_FILE_PATTERN}" + +} + +################### +# Commiting changes to repos +################### +builder_commit() { + if [ -d "${PRODUCT_DIR}/.git" ]; then + echo -n + # echo "builder_commit() not implemented yet." + fi +} + + +##################### +# build opsi package +##################### +builder_cleanup() { + # Paranoia + if [ -d "$output_dir" ] && [[ $output_dir == $TMP_DIR/opsi-builder.* ]] ; then + rm -rf $output_dir + fi +} diff --git a/lib/builder-utils.sh b/lib/builder-utils.sh new file mode 100644 index 0000000..4cf2c71 --- /dev/null +++ b/lib/builder-utils.sh @@ -0,0 +1,56 @@ +############################################# +# void retrieve_file (src, dst) +# +# Description: retrieve file from an URL +# +# Parameter +# src: source url to get file from +# dst: path to store file to +# +############################################# +function retrieve_file() { + local src=$1 + local dst=$2 + + # Check, if the URL is a file URL starting with file:// + if [ -f $dst ] && [ -z ${DIST_FORCE_DOWNLOAD} ]; then + echo " Info: File still cached/downloaded. To force a download, set DIST_FORCE_DOWNLOAD=1" + elif [[ $src == file://* ]]; then + fileurl=`echo $src | sed "s/^file:\/\///"` + cp $fileurl $dst 2>/dev/null + else + rm -f $dst + wget --tries=1 -O $dst --timeout=5 -q --no-verbose $src + fi +} + + +############################################# +# check if method is available and call it +############################################# +function call_entry_point() { + local _resultvar=$1 + local func=$2 + + # Entry point + type $func &>/dev/null + if [ $? == 0 ] ; then + $func + eval $_resultvar="0" + else + eval $_resultvar="1" + fi + +} + + +################### +# Check error +################### +builder_check_error() { + if [ $? == 1 ] ; then + echo "FATAL: $1" + exit 0 + fi +} + diff --git a/sample/builder-product.cfg b/sample/builder-product.cfg new file mode 100644 index 0000000..3ff61fc --- /dev/null +++ b/sample/builder-product.cfg @@ -0,0 +1,41 @@ +############################ +# Setup product information +############################ +VENDOR="itwatch.de" +PN="itwatch" +VERSION="4.10.19" +RELEASE="3" +PRIORITY="0" +ADVICE="" + +# Status integration, testing, release +STATUS="review" + +################# +# Setup url for downloading the binaries +################ + +# all downloads should not have any traling parameters like ?downlaodid=1234 .....(should rewrite in python) +# You can setup many URL's separated by ";" + +# Format: basename ; URL1 ; URL2 ; ... +ICON_FILE_INDEX=0 + +# Format: basename ; URL1 ; URL2 ; ... +#FILE1="itWESS-Client_German.msi;http://ivyrepos.dtnet.de/v-$PN/itwatch/itwatch/4.10.19/itWESS-Client_German.msi;http://internal.graz.disconnected-by-peer.at/Orig/itWatch/itWESS/HB/4.10.19/itWESS-Client_German.msi" +#FILE2="itWESS-64bit_German.msi;http://ivyrepos.dtnet.de/itwatch/itwatch/4.10.19/itWESS-64bit_German.msi;http://internal.graz.disconnected-by-peer.at/Orig/itWatch/itWESS/HB/4.10.19/itWESS-64bit_German.msi" + +FILE[0]="itWatchLogo.jpg" +SOURCE[0]="http://www.itwatch.de/logos/itWatchLogo.jpg" +WINST[0]=IconFile + +FILE[1]="itWESS-Client_German.msi" +SOURCE[1]="http://ivyrepos.dtnet.de/v-$PN/itwatch/itwatch/4.10.19/itWESS-Client_German.msi;http://internal.graz.disconnected-by-peer.at/Orig/itWatch/itWESS/HB/4.10.19/itWESS-Client_German.msi" +ARCH[1]="X86" +WINST[1]=itWESS + +FILE[2]="itWESS-64bit_German.msi" +SOURCE[2]="http://ivyrepos.dtnet.de/itwatch/itwatch/4.10.19/itWESS-64bit_German.msi;http://internal.graz.disconnected-by-peer.at/Orig/itWatch/itWESS/HB/4.10.19/itWESS-64bit_German.msi" +ARCH[2]="X86_64" +WINST[2]=itWESS64 + diff --git a/sample/builder-targets-cb.sh b/sample/builder-targets-cb.sh new file mode 100644 index 0000000..6525a3e --- /dev/null +++ b/sample/builder-targets-cb.sh @@ -0,0 +1,52 @@ +############################################################################## +# This optional file "builder-targets-cb.sh" will be called by builder.sh +# +# You can overwrite target functions like +# config, prepare, retrieve, create, package, publish, commit, cleanup +# and define callback functions +# cb_package_makeproductfile +# +############################################################################## + +#function config() { +# echo "Config" +# builder_config +#} + +#function prepare() { +# echo "Prepare" +# builder_prepare +#} + +function retrieve() { + echo "Retrieve" + builder_retrieve +} + +function create() { + echo "Create" + builder_create +} + +function package() { + echo "Package" + builder_package +} + +function cb_package_makeproductfile() { + echo "May add/replace files to the files to $inst_dir" +} + +function publish() { + echo "Publish" + builder_publish +} + +function commit() { + echo "Commit" + # builder_commit +} +function cleanup() { + echo "Cleanup: output_dir: $output_dir" + # builder_cleanup +}