##################### # Call user entry point #################### # source generic utility functions . $BASEDIR/lib/builder-utils.sh ##################### # Read config #################### builder_config() { # Define commands CMD_7z="`which 7z`" ; builder_check_error "Command '7z' not installed" CMD_unzip="`which unzip.exe`" ; builder_check_error "Command 'unzip' not installed" CMD_unrar="`which UnRAR.exe`" ; builder_check_error "Command 'UnRAR' not installed" CMD_zip="`which zip.exe`" ; builder_check_error "Command 'zip' not installed" CMD_tar="`which tar.exe`" ; builder_check_error "Command 'tar' not installed" CMD_cabextract="`which cabextract.exe`" ; builder_check_error "Command 'cabextract' not installed" CMD_unix2dos="`which unix2dos.exe`" ; builder_check_error "Command 'unix2dos' not installed" CMD_sha1sum="`which sha1sum.exe`" ; builder_check_error "Command 'sha1sum' not installed" CMD_inifile="`which inifile.exe`" ; builder_check_error "Command 'inifile' not installed (http://www.horstmuc.de/wbat32d.htm#inifile)" CMD_tidy="`which tidy.exe`" ; builder_check_error "Command 'tidy' not installed" CMD_msiexec="`which msiexec.exe`" ; builder_check_error "Command 'msiexec' not installed" CMD_innounp="`which innounp.exe`" ; builder_check_error "Command 'innounp' not installed" CMD_xmlstarlet="`which xmlstarlet.exe`" ; builder_check_error "Command 'xmlstarlat' not installed" CMD_msix="`which MsiX.exe`" ; builder_check_error "Command 'msix' not installed" # Check temp dir test -d ${TMP_DIR} builder_check_error "temp directory not available: $TMP_DIR" # project dependent configuration local config=${PRODUCT_DIR}/builder-product.cfg test -f ${config} || builder_check_error "cannot read product config: ${config}" . ${config} # set default build configuration and source the user dependent file local config=$BASEDIR/conf/aio-builder.cfg . ${config} # Source local build configuration (must be done AFTER sourcing the builder-product.cfg.cfg) if [ -f "$AIO_BUILDER" ] ; then config=$AIO_BUILDER else test -f $HOME/.aio-builder.cfg && config=$HOME/.aio-builder.cfg fi # Read ONLY the STATUS variable from the build configuration file eval "`grep -E "^(STATUS|STATUS_INTEGRATION_RELEASE)=" $config`" # change some variable from the builder-product.cfg dynamically: # autogenerate release number, if we are in status "integration" if [ "$STATUS" = "integration" ] ; then if [ "${STATUS_INTEGRATION_RELEASE}" = "func:inc1" ] ; then . ${config} calc_release else # AIO/control:RELEASE is limited to max 16 chars - take care in regards to the CREATOR_TAG RELEASE="${STATUS_INTEGRATION_RELEASE}" fi fi # Read configurationfile . ${config} echo "Loaded builder configuration: $config" # Check variables if [ -z ${AIO_REPOS_BASE_DIR} ] || [ ! -d ${AIO_REPOS_BASE_DIR} ] ; then echo "configuration error: AIO_REPOS_BASE_DIR directory does not exist: $AIO_REPOS_BASE_DIR" exit 2 fi if [ "$TYPE" != "public" ] && [ "$TYPE" != "restricted" ] ; then fatal_error "unknown TYPE: $TYPE" fi # Check, if variable is numeric if [ ! `expr ${AIO_REPOS_PURGE_LIMIT} + 1 2> /dev/null` ] ; then fatal_error "AIO_REPOS_PURGE_LIMIT must be numeric" fi } ##################### # Prepare build #################### builder_prepare() { echo "builder_prepare: " # Check if the package is still build if [ "$AIO_REPOS_FORCE_UPLOAD" != "true" ] && [ -f "${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z" ] ; then echo "File ${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z already exists." exit 1 fi mkdir -p $DIST_CACHE_DIR log_debug "Distribution directory: $DIST_CACHE_DIR" # setup work directory OUTPUT_DIR="$TMP_DIR/aio-builder.`date +%Y%m%d-%H%M%S`.$$" mkdir -p ${OUTPUT_DIR} builder_check_error "Cannot create temp directory ${OUTPUT_DIR}" # prepare INST_DIR=$OUTPUT_DIR/$PN mkdir $INST_DIR log_info " OUTPUT_DIR: $OUTPUT_DIR" } ##################### # Download all dist files from one of the defined URLs. # and validate the checksum #################### builder_retrieve() { for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do local basename=${DL_FILE[$i]} local urls=${DL_SOURCE[$i]} local arch=${DL_ARCH[$i]} local downloaded=0 # Add private repos to the urls if [ ! -z ${DIST_PRIVATE_REPOS} ]; then urls="${DIST_PRIVATE_REPOS}/$basename;$urls" fi # check existence of CRC file only in non devel mode if [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" != "true" ] ; then fatal_error "You need to create the checksums with: sha1sum ${DIST_CACHE_DIR}/${basename} > ${PRODUCT_DIR}/${basename}.sha1sum" fi echo "Downloading $basename" # check downloading from the defined URLs for src in `echo $urls | sed -e 's/[;,]/\n/g'` ; do if [ $downloaded == 1 ]; then continue; fi # Download file echo " Info: Downloding from $src" local downloader=${DL_DOWNLOADER[$i]} if [ -z $downloader ]; then downloader="wget" ; fi mkdir -p ${DIST_CACHE_DIR}/$arch DL_DIST_FILE[$i]=${DIST_CACHE_DIR}/$arch/$basename retrieve_file $downloader $src ${DL_DIST_FILE[$i]} if [ $? != 0 ] ; then echo " Warning: Failed to download file - try next URL" continue; fi # Check sha1 if [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" == "true" ] ; then $CMD_sha1sum ${DL_DIST_FILE[$i]} > ${PRODUCT_DIR}/${basename}.sha1sum downloaded=1 echo " WARNING: SHA1 checksum (${DL_DIST_FILE[$i]}.sha1sum) was created dynamically because auf CHECKSUM_AUTOCREATE=$CHECKSUM_AUTOCREATE" else # testing the checksum of the downloaded files local sha1sum_val=`cat ${PRODUCT_DIR}/${basename}.sha1sum | cut -d " " -f1` local checksum_val=`sha1sum ${DL_DIST_FILE[$i]} | cut -d " " -f1` if [ "$checksum_val" = "$sha1sum_val" ] ; then downloaded=1 fi fi # Print result if [ "$downloaded" == "1" ] ; then echo " Info: Downloaded successfully" else echo " Error: The checksums do not match - try next URL" fi done echo # Ups - no URL works if [ $downloaded != 1 ] ; then echo " Error: can download the file or checksum wrong (sha1sum ${DIST_CACHE_DIR}/${basename} > ${basename}.sha1sum)" exit 1; fi done } ##################### # Create files #################### builder_create() { # Copy files and convert text files to dos format # cp -Rv ${PRODUCT_DIR}/AIO $INST_DIR # cp -Rv ${PRODUCT_DIR}/Temp $INST_DIR # if [ -d "${PRODUCT_DIR}/SERVER_DATA" ] ; then # cp -Rv ${PRODUCT_DIR}/SERVER_DATA $INST_DIR # fi # copy binaries for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do DL_EXTRACT_PATH[$i]=${INST_DIR}/Temp/${DL_ARCH[$i]}/${DL_EXTRACT_TO[$i]} local format=${DL_EXTRACT_FORMAT[$i]} if [ -z "$format" ] ; then format="cp"; fi local option=${DL_EXTRACT_OPTION[$i]} if [ -z "$option" ] ; then option=""; fi mkdir -p ${DL_EXTRACT_PATH[$i]} process_file $format ${DL_DIST_FILE[$i]} ${DL_EXTRACT_PATH[$i]} $option done # write ini file local ini_file=${INST_DIR}/aio-$PN.ini for var in VENDOR PN VERSION RELEASE TYPE CREATOR_TAG CREATOR_NAME CREATOR_EMAIL ; do write_ini_file $ini_file "COMMON" ${var}="${!var}" done # convert to dos file linefeed # find $INST_DIR -type f | xargs -n1 -iREP sh -c 'file -i $0 | grep -v "utf-16" | grep "text/plain" && '$CMD_unix2dos' $0 ' REP >/dev/null # set exec bit on executeables # find $INST_DIR -type f -iname "*.exe" -o -iname "*.bat" -o -iname "*.cmd" -o -iname "*.msi" -o -iname "*.msp" | xargs chmod +x -v # Create changelog based on git - if available if [ -d "${PRODUCT_DIR}/.git" ] ; then # new changelog format echo "" >> $INST_DIR/changelog.txt echo "[Changelog]" >> $INST_DIR/changelog.txt #$CMD_ruby $BASEDIR/libexec/gitlog-to-deblog.rb >> $INST_DIR/AIO/changelog.txt echo "" >> $INST_DIR/changelog.txt else echo "No git repository present." fi } ##################### # build aio package ##################### builder_package() { # creating package local release_new=${CREATOR_TAG}${RELEASE} local aio_file=${PN}_${VERSION}-${release_new}.7z rm -rf $INST_DIR/Temp pushd ${OUTPUT_DIR} rm -f ${aio_file} ${AIO_REPOS_FILE_PATTERN}.7z $CMD_7z a ${AIO_REPOS_FILE_PATTERN}.7z $INST_DIR/* builder_check_error "Building AIO-package" popd # rename aio package file if [ "${aio_file}" != "${AIO_REPOS_FILE_PATTERN}.7z" ]; then mv ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z builder_check_error "can't move file ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z" fi # --exclude \*/.git\* # create source- and binary package package test "${AIO_REPOS_UPLOAD_BIN}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.zip $INST_DIR test "${AIO_REPOS_UPLOAD_SOURCE}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}-src.zip ${PRODUCT_DIR} } ##################### # publish ##################### builder_publish() { # Upload file to repository mkdir -p ${AIO_REPOS_PRODUCT_DIR} echo "Publishing aio-package to ${AIO_REPOS_PRODUCT_DIR}" local src=${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN} local dst=${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN} # Link dir if [ "${AIO_REPOS_LINK_NEWBUILDS}" = "true" ] ; then mkdir -p ${AIO_REPOS_BASE_DIR}/.new_builds ln -sf ${AIO_REPOS_PRODUCT_DIR} ${AIO_REPOS_BASE_DIR}/.new_builds/${AIO_REPOS_FILE_PATTERN} builder_check_error "Can't Link file $dst.7z --> $dst.7z" fi # copy files if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then cp ${src}.7z ${dst}.7z builder_check_error "Can't upload file $dst.7z --> $dst.7z" fi if [ "${AIO_REPOS_UPLOAD_BIN}" = "true" ] ; then cp ${src}.zip ${dst}.zip builder_check_error "Can't upload file $dst.zip --> $dst.zip" fi if [ "${AIO_REPOS_UPLOAD_SOURCE}" = "true" ] ; then cp ${src}-src.zip ${dst}-src.zip builder_check_error "Can't upload file ${dst}-src.zip --> ${dst}-src.zip" fi if [ "${AIO_REPOS_AIOMANAGER_INSTALL}" = "true" ] ; then aio-package-manager -i -v ${src}.7z builder_check_error "Can't install ${src}.7z" fi if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then md5sum "${src}.7z" | sed 's/ .*//' > ${dst}.7z.md5 builder_check_error "Can't create md5 file" fi if [ "${AIO_REPOS_UPLOAD_AIO_GPG}" = "true" ] ; then ${CMD_gpg} --batch --passphrase ${GPG_PASSPHRASE} --output "${dst}.7z.gpg" --detach-sig "${src}.7z" builder_check_error "Can't create gpg file" fi # Create revision file for this local rev_file=${AIO_REPOS_PRODUCT_DIR}/${PN}-${VERSION}-${CREATOR_TAG}${RELEASE}.cfg cat > $rev_file < 0 ] && [ "${STATUS}" = "${AIO_REPOS_PURGE_STATUS}" ] ; then echo "Autopurging enabled" # determinte max version to delete local limit local pn_limit=`echo ${PN} | sed "s/[\.\-]/_/g"` eval "`echo limit=\\$\\{AIO_REPOS_PURGE_LIMIT_${pn_limit}\\}`" if [ -z "$limit" ] || [ ! `expr $limit + 1 2>/dev/null` ] ; then limit=${AIO_REPOS_PURGE_LIMIT} fi echo " Purging, max. number of versions: $limit" # Find all revision files and sort them local file_list=${OUTPUT_DIR}/product-file-list.txt local file_sort_list=${OUTPUT_DIR}/product-file-sort-list.txt local file_sort_list_version=${OUTPUT_DIR}/product-file-sort-list-version.txt local file_sort_list_release=${OUTPUT_DIR}/product-file-sort-list-release.txt local file_sort_list_final=${OUTPUT_DIR}/product-file-sort-list-final.txt rm -f ${file_list} # first uniq sort all cfg based on version for cfg_file in `find ${AIO_REPOS_PRODUCT_DIR} -name "${PN}-*.cfg" -print ` ; do . ${cfg_file} echo $REV_VERSION >> ${file_list} done sort -V ${file_list} | uniq > ${file_sort_list_version} # second uniq sort all versions based in release for pkg_version in `cat ${file_sort_list_version}` ; do for cfg_file_ver in ${AIO_REPOS_PRODUCT_DIR}/${PN}-${pkg_version}-*.cfg ; do . ${cfg_file_ver} echo ${pkg_version}-$REV_CREATOR_TAG$REV_RELEASE >> ${file_sort_list_release} done done sort -V ${file_sort_list_release} | uniq > ${file_sort_list_final} # third create versionrelease for release_file_list in `cat ${file_sort_list_final}` ; do . ${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg echo "${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg" >> ${file_sort_list} done # Delete the oldest files log_debug "base list for calculate purge:" for cfg_sort_file in `head -n-${limit} ${file_sort_list}` ; do . ${cfg_sort_file} if [ "${REV_STATUS}" != "${AIO_REPOS_PURGE_STATUS}" ] ; then continue; fi dir_base=`dirname ${cfg_file}` product_file="${dir_base}/${REV_AIO_REPOS_FILE_PATTERN}" echo " Purging product version: $product_file*" # Paranoid ... check the files to delete first if [ ! -z "${dir_base}" ] && [ -d "${AIO_REPOS_BASE_DIR}" ] && [ ! -z "$product_file" ] ; then rm -f ${product_file}* ${cfg_sort_file} # remove directory - if it's empty if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then rmdir ${dir_base} fi fi done fi } ################### # Commiting changes to repos ################### builder_commit() { if [ -d "${PRODUCT_DIR}/.git" ]; then echo -n log_debug "builder_commit() not implemented yet." fi } ##################### # build aio package ##################### builder_cleanup() { # Paranoia if [ -d "$OUTPUT_DIR" ] && [[ $OUTPUT_DIR == $TMP_DIR/aio-builder.* ]] ; then rm -rf $OUTPUT_DIR fi }