From 02fac0f3d14c3f2a123f559faffdd3d5fd9cdf55 Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Mon, 6 Feb 2012 14:20:56 +0100 Subject: [PATCH 1/6] Fix: fix autopurging bug (probs mit sorting ..) New: Added new RELEASE-incrementing algorithm dependent on the VERSION. Refer to opsi-builder.cfg:STATUS_INTEGRATION_RELEASE="func:inc1" --- conf/opsi-builder.cfg | 11 ++++++----- lib/builder-targets.sh | 40 +++++++++++++++++++++++++++++----------- lib/builder-utils.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/conf/opsi-builder.cfg b/conf/opsi-builder.cfg index be054a1..bda97bd 100644 --- a/conf/opsi-builder.cfg +++ b/conf/opsi-builder.cfg @@ -24,9 +24,10 @@ STATUS="integration" # STATUS_INTEGRATION_RELEASE - automatically created release number, # if STATUS is set to "integration". Valid values: anything, default value: "`date +%Y%m%d%H%M`" # Samples: -# "`date +%Y%m%d%H%M`" -# "$BUILD_NUMBER" - jenkins https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-JenkinsSetEnvironmentVariables -STATUS_INTEGRATION_RELEASE="`date +%Y%m%d%H%M`" +# "func:inc1" - build-in function "inc1": search the opsi-repository for the latested release and increment 1 +# "`date +%Y%m%d%H%M`" - command output, e.g. the date format YYMMDD-HHMM +# "$BUILD_NUMBER" - Environment variables, jenkins https://wiki.jenkins-ci.org/display/JENKINS/Building+a+software+project#Buildingasoftwareproject-JenkinsSetEnvironmentVariables +STATUS_INTEGRATION_RELEASE="func:inc1" # CHECKSUM_AUTOCREATE - Every downloaded file is checked using SHA1 algorithm # against a checksum file, if this file ist not available, it will be created @@ -51,7 +52,7 @@ CREATOR_EMAIL="your-email@domain.de" # OPSI repository settings ################################# # Basedirectory storing OPSI-packages after building -OPSI_REPOS_BASE_DIR=$HOME/.opsi-repository +OPSI_REPOS_BASE_DIR=$HOME/opsi-repository # Directory- and filename pattern OPSI_REPOS_PRODUCT_DIR=${OPSI_REPOS_BASE_DIR}/${STATUS}/${VENDOR}/${PN}/${VERSION}-${CREATOR_TAG}${RELEASE} @@ -74,7 +75,7 @@ OPSI_REPOS_FILE_PATTERN=${PN}_${VERSION}-${CREATOR_TAG}${RELEASE} # You must be member of the unix group pcpatch&opsiadmin OPSI_REPOS_UPLOAD_OPSI=true OPSI_REPOS_UPLOAD_OPSI_ZSYNC=false -OPSI_REPOS_UPLOAD_SOURCE=true +OPSI_REPOS_UPLOAD_SOURCE=false OPSI_REPOS_UPLOAD_BIN=false OPSI_REPOS_OPSIMANAGER_INSTALL=false diff --git a/lib/builder-targets.sh b/lib/builder-targets.sh index 1f7ec31..4b0e04e 100644 --- a/lib/builder-targets.sh +++ b/lib/builder-targets.sh @@ -45,8 +45,13 @@ builder_config() { # change some variable from the builder-product.cfg dynamically: # autogenerate release number, if we are in status "integration" if [ "$STATUS" = "integration" ] ; then - # OPSI/control:RELEASE is limited to max 16 chars - take care in regards to the CREATOR_TAG - RELEASE="${STATUS_INTEGRATION_RELEASE}" + if [ "${STATUS_INTEGRATION_RELEASE}" = "func:inc1" ] ; then + . ${config} + calc_release + else + # OPSI/control:RELEASE is limited to max 16 chars - take care in regards to the CREATOR_TAG + RELEASE="${STATUS_INTEGRATION_RELEASE}" + fi fi # Read configurationfile @@ -310,6 +315,8 @@ EOF # Purge old product versions - defined by limit OPSI_REPOS_PURGE_LIMIT if [ "${OPSI_REPOS_PURGE}" = "true" ] && [ ! -z "${OPSI_REPOS_PURGE_LIMIT}" ] && [ "${OPSI_REPOS_PURGE_LIMIT}" > 0 ] ; then echo "Autopurging enabled" + + # determinte max version to delete local limit eval "`echo limit=\\$\\{OPSI_REPOS_PURGE_LIMIT_${PN}\\}`" if [ -z "$limit" ] || [ ! `expr $limit + 1 2> /dev/null` ] ; then @@ -317,10 +324,21 @@ EOF fi echo " Purging, max. number of versions: $limit" - # Find all revision files + # Find all revision files and sort them local file_list=${OUTPUT_DIR}/product-file-list.txt - find ${OPSI_REPOS_BASE_DIR} -name "${PN}-${VERSION}-${CREATOR_TAG}*.cfg" -exec echo {} \; | sort > ${file_list} - for cfg_file in `tail -${limit} ${file_list} | ${CMD_comm} -13 - ${file_list}` ; do + local file_sort_list=${OUTPUT_DIR}/product-file-sort-list.txt + rm -f ${file_list} + for cfg_file in `find ${OPSI_REPOS_BASE_DIR} -name "${PN}-${VERSION}-${CREATOR_TAG}*.cfg" -print ` ; do + . ${cfg_file} + printf "%08d;$cfg_file\n" $REV_RELEASE >> ${file_list} + done + sort -n ${file_list} > ${file_sort_list} + + # Delete the oldest files + log_debug "base list for calculate purge:" + for cfg_sort_file in `tail -${limit} ${file_sort_list} | ${CMD_comm} -13 - ${file_sort_list}` ; do + + local cfg_file=`echo $cfg_sort_file | cut -f 2 -d ";"` dir_base=`dirname ${cfg_file}` . ${cfg_file} @@ -330,12 +348,12 @@ EOF # Paranoid ... check the files to delete first if [ ! -z "${dir_base}" ] && [ -d "${OPSI_REPOS_BASE_DIR}" ] ; then rm -f ${product_file}.* ${cfg_file} - fi - - # remove directory - if it's empty - if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then - rmdir ${dir_base} - fi + + # remove directory - if it's empty + if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then + rmdir ${dir_base} + fi + fi done fi } diff --git a/lib/builder-utils.sh b/lib/builder-utils.sh index 9c8f3b9..1fb28f3 100644 --- a/lib/builder-utils.sh +++ b/lib/builder-utils.sh @@ -224,3 +224,36 @@ create_winst_varfile() { echo >>$var_file } + +##################### +# void calc_release() +# +# Description: +# Calculate new release based on +# the latest one found in the repository. +# +# $RELEASE is set to the calculated release. +# +#################### +function calc_release() { + + # Find all revision files and sort them + local file_list=`mktemp /tmp/opsi-builder.calc_release.XXXXXXXXXXX` + for cfg_file in `find ${OPSI_REPOS_BASE_DIR} -name "${PN}-${VERSION}-${CREATOR_TAG}*.cfg" -print ` ; do + . ${cfg_file} + printf "%08d;$cfg_file\n" $REV_RELEASE >> ${file_list} + done + local oldest_cfg=`sort -n ${file_list} | cut -f 2 -d ";" | tail -1` + rm -f ${file_list} + + if [ ! -f "${oldest_cfg}" ] ; then + echo "Warning: no cfg-file found for this product. RELEASE will be set to 1" + RELEASE=1 + else + log_debug "calc_release() oldest_cfg: ${oldest_cfg}" + . ${oldest_cfg} + log_debug " latest release: $REV_RELEASE" + RELEASE=`expr ${REV_RELEASE} + 1 2> /dev/null` + builder_check_error "Cannot incrememnt REV_RELEASE from file ${oldest_cfg}" + fi +} From f0108632cddad9e975d6e82c462600bd2d454e2c Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Mon, 6 Feb 2012 19:13:57 +0100 Subject: [PATCH 2/6] - Improvement: Purging product version only related to a defined OPSI_REPOS_PURGE_STATUS - New: Add template for testing products --- conf/opsi-builder.cfg | 5 +++++ lib/builder-targets.sh | 10 +++++++--- sample/Testing.txt | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 sample/Testing.txt diff --git a/conf/opsi-builder.cfg b/conf/opsi-builder.cfg index bda97bd..4e64374 100644 --- a/conf/opsi-builder.cfg +++ b/conf/opsi-builder.cfg @@ -84,6 +84,11 @@ OPSI_REPOS_OPSIMANAGER_INSTALL=false # Valid values: true|false OPSI_REPOS_PURGE=false +# Only purge projects using the defined STATUS +# ValidValues: integration|testing|release +# Defaut: integration +OPSI_REPOS_PURGE_STATUS=integration + # defines the max. instances of published versions per product # Valid values: 0...n, 0: disable autocleanup OPSI_REPOS_PURGE_LIMIT=3 diff --git a/lib/builder-targets.sh b/lib/builder-targets.sh index 4b0e04e..295d10e 100644 --- a/lib/builder-targets.sh +++ b/lib/builder-targets.sh @@ -303,17 +303,20 @@ builder_publish() { # Create revision file for this local rev_file=${OPSI_REPOS_PRODUCT_DIR}/${PN}-${VERSION}-${CREATOR_TAG}${RELEASE}.cfg cat > $rev_file < 0 ] ; then + if [ "${OPSI_REPOS_PURGE}" = "true" ] && [ ! -z "${OPSI_REPOS_PURGE_LIMIT}" ] && [ "${OPSI_REPOS_PURGE_LIMIT}" > 0 ] && [ "${STATUS}" = "${OPSI_REPOS_PURGE_STATUS}" ] ; then echo "Autopurging enabled" # determinte max version to delete @@ -339,9 +342,10 @@ EOF for cfg_sort_file in `tail -${limit} ${file_sort_list} | ${CMD_comm} -13 - ${file_sort_list}` ; do local cfg_file=`echo $cfg_sort_file | cut -f 2 -d ";"` - dir_base=`dirname ${cfg_file}` . ${cfg_file} + if [ "${REV_STATUS}" != "${OPSI_REPOS_PURGE_STATUS}" ] ; then continue; fi + dir_base=`dirname ${cfg_file}` product_file="${dir_base}/${REV_OPSI_REPOS_FILE_PATTERN}" echo " Purging product version: $product_file.*" diff --git a/sample/Testing.txt b/sample/Testing.txt new file mode 100644 index 0000000..d6b5e6c --- /dev/null +++ b/sample/Testing.txt @@ -0,0 +1,28 @@ +*** Qualitiymanagement/Testing procedure + +- Product: + - Name: productname + - Version: xxx + - Release: yyyy + +- Environment + - OS: Native windows XP-32Bit installation + - Preinstalled packages: None (also no aio) + - Files used while testing: + File1: http://domain.de/file.ext + +- Testing process: + OPSI-action: + - Installation (without dependencies like AIO/Firefox): ?? + Result: + - File extensions related to program: ?? + - Start menu entry: ?? + - Starting/using program: viewer works, File1 is shown correctly: ?? + + OPSI-action: + - Uninstall + Result: + - File extensions related to program: unrelated: ?? + - Start menu entry: none: ?? + + From 5b562a8911120372560edfff2450a92bb63fa989 Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Mon, 6 Feb 2012 19:51:07 +0100 Subject: [PATCH 3/6] Cleanup file --- sample/builder-targets-cb.sh | 39 ++---------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/sample/builder-targets-cb.sh b/sample/builder-targets-cb.sh index 639eff1..9a3d4a8 100644 --- a/sample/builder-targets-cb.sh +++ b/sample/builder-targets-cb.sh @@ -19,42 +19,7 @@ # ############################################################################## -#function config() { -# echo "Config - doing some commands before calling the builder_config" -# builder_config -# echo "Config - doing some commands after calling the builder_config" -#} - -#function prepare() { -# echo "Prepare" -# builder_prepare -#} - -function retrieve() { - echo "Retrieve" - builder_retrieve -} - -function create() { - echo "Create" - builder_create -} - -function package() { - echo "Package" - builder_package -} - -function publish() { - echo "Publish" - builder_publish -} - -function commit() { - echo "Commit" - # builder_commit -} function cleanup() { - echo "Cleanup: output_dir: $output_dir" - # builder_cleanup + echo "Cleanup" + builder_cleanup } From 4bb35048296d3a182c863552b534ce1417862755 Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Mon, 6 Feb 2012 20:11:30 +0100 Subject: [PATCH 4/6] changed structure --- sample/{ => product-template/CLIENT_DATA}/setup32.ins | 0 sample/{ => product-template}/Testing.txt | 0 sample/{ => product-template}/builder-targets-cb.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename sample/{ => product-template/CLIENT_DATA}/setup32.ins (100%) rename sample/{ => product-template}/Testing.txt (100%) rename sample/{ => product-template}/builder-targets-cb.sh (100%) diff --git a/sample/setup32.ins b/sample/product-template/CLIENT_DATA/setup32.ins similarity index 100% rename from sample/setup32.ins rename to sample/product-template/CLIENT_DATA/setup32.ins diff --git a/sample/Testing.txt b/sample/product-template/Testing.txt similarity index 100% rename from sample/Testing.txt rename to sample/product-template/Testing.txt diff --git a/sample/builder-targets-cb.sh b/sample/product-template/builder-targets-cb.sh similarity index 100% rename from sample/builder-targets-cb.sh rename to sample/product-template/builder-targets-cb.sh From 21a6d88a8c6b822f5c3b25e2695af5a82df6a192 Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Mon, 6 Feb 2012 21:09:13 +0100 Subject: [PATCH 5/6] - Improvement: using jenkins, a temp-direcory based on the date is not enough - added pid --- bin/opsi-builder.sh | 1 - lib/builder-targets.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/opsi-builder.sh b/bin/opsi-builder.sh index 3bc5b7a..1f2752d 100755 --- a/bin/opsi-builder.sh +++ b/bin/opsi-builder.sh @@ -39,7 +39,6 @@ builder_controller() { #################### # Main #################### - # get the 'real' directory this program stored in (resolve symbolic links) PRG=$(readlink -f $0) BASEDIR=`dirname "$PRG"` diff --git a/lib/builder-targets.sh b/lib/builder-targets.sh index 295d10e..de77ba8 100644 --- a/lib/builder-targets.sh +++ b/lib/builder-targets.sh @@ -90,7 +90,7 @@ builder_prepare() { log_debug "Distribution directory: $DIST_CACHE_DIR" # setup work directory - OUTPUT_DIR=$TMP_DIR/opsi-builder.`date +%Y%m%d-%H%M%S` + OUTPUT_DIR="$TMP_DIR/opsi-builder.`date +%Y%m%d-%H%M%S`.$$" mkdir -p ${OUTPUT_DIR} builder_check_error "Cannot create temp directory ${OUTPUT_DIR}" From 7b88ef224be45d4ca0aa863ab7a0ccfa60d8d9ba Mon Sep 17 00:00:00 2001 From: DT Netsolution GmbH Date: Tue, 7 Feb 2012 09:02:43 +0100 Subject: [PATCH 6/6] - improvement: using command variable CMD_sha1sum - improvement: ignore .git directory while zipping - fix: GOPSI-12 Max Limit of packages per version delete release --- lib/builder-targets.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/lib/builder-targets.sh b/lib/builder-targets.sh index de77ba8..500e1c9 100644 --- a/lib/builder-targets.sh +++ b/lib/builder-targets.sh @@ -18,6 +18,7 @@ builder_config() { CMD_identify="`which identify`" ; builder_check_error "Command 'identify' (ImageMagick) not installed" CMD_zsyncmake="`which zsyncmake`" ; builder_check_error "Command 'zsyncmake' not installed" CMD_comm="`which comm`" ; builder_check_error "Command 'comm' not installed" + CMD_sha1sum="`which sha1sum`" ; builder_check_error "Command 'sha1sum' not installed" # Check temp dir test -d ${TMP_DIR} @@ -145,14 +146,16 @@ builder_retrieve() { # Check sha1 if [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" == "true" ] ; then - sha1sum ${DL_DIST_FILE[$i]} > ${PRODUCT_DIR}/${basename}.sha1sum + $CMD_sha1sum ${DL_DIST_FILE[$i]} > ${PRODUCT_DIR}/${basename}.sha1sum downloaded=1 echo " WARNING: SHA1 checksum (${DL_DIST_FILE[$i]}.sha1sum) was created dynamically because auf CHECKSUM_AUTOCREATE=$CHECKSUM_AUTOCREATE" else # testing the checksum of the downloaded files local sha1sum_val=`cat ${PRODUCT_DIR}/${basename}.sha1sum | cut -d " " -f1` local checksum_val=`sha1sum ${DL_DIST_FILE[$i]} | cut -d " " -f1` - if [ "$checksum_val" == "$sha1sum_val" ] ; then +echo sha1sum_val: $sha1sum_val +echo checksum_val: $checksum_val + if [ "$checksum_val" = "$sha1sum_val" ] ; then downloaded=1 fi fi @@ -253,9 +256,10 @@ builder_package() { builder_check_error "can't move file ${OUTPUT_DIR}/${opsi_file} ${OUTPUT_DIR}/${OPSI_REPOS_FILE_PATTERN}.opsi" fi +# --exclude \*/.git\* # create source- and binary package package - test "${OPSI_REPOS_UPLOAD_BIN}" = "true" && $CMD_zip -r ${OUTPUT_DIR}/${OPSI_REPOS_FILE_PATTERN}.zip $INST_DIR - test "${OPSI_REPOS_UPLOAD_SOURCE}" = "true" && $CMD_zip -r ${OUTPUT_DIR}/${OPSI_REPOS_FILE_PATTERN}-src.zip ${PRODUCT_DIR} + test "${OPSI_REPOS_UPLOAD_BIN}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${OPSI_REPOS_FILE_PATTERN}.zip $INST_DIR + test "${OPSI_REPOS_UPLOAD_SOURCE}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${OPSI_REPOS_FILE_PATTERN}-src.zip ${PRODUCT_DIR} } @@ -322,7 +326,7 @@ EOF # determinte max version to delete local limit eval "`echo limit=\\$\\{OPSI_REPOS_PURGE_LIMIT_${PN}\\}`" - if [ -z "$limit" ] || [ ! `expr $limit + 1 2> /dev/null` ] ; then + if [ -z "$limit" ] || [ ! `expr $limit + 1 2>/dev/null` ] ; then limit=${OPSI_REPOS_PURGE_LIMIT} fi echo " Purging, max. number of versions: $limit" @@ -347,11 +351,11 @@ EOF dir_base=`dirname ${cfg_file}` product_file="${dir_base}/${REV_OPSI_REPOS_FILE_PATTERN}" - echo " Purging product version: $product_file.*" + echo " Purging product version: $product_file*" # Paranoid ... check the files to delete first - if [ ! -z "${dir_base}" ] && [ -d "${OPSI_REPOS_BASE_DIR}" ] ; then - rm -f ${product_file}.* ${cfg_file} + if [ ! -z "${dir_base}" ] && [ -d "${OPSI_REPOS_BASE_DIR}" ] && [ ! -z "$product_file" ] ; then + rm -f ${product_file}* ${cfg_file} # remove directory - if it's empty if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then