aio-builder/lib/builder-targets.sh

#####################
# Call user entry point
####################

# source generic utility functions
. $BASEDIR/lib/builder-utils.sh

#####################
# Read config
####################
builder_config() {
  
  # Define commands
  CMD_7z="`which 7z`"                      ; builder_check_error "Command '7z' not installed"
  CMD_unzip="`which unzip.exe`"            ; builder_check_error "Command 'unzip' not installed"
  CMD_unrar="`which UnRAR.exe`"            ; builder_check_error "Command 'UnRAR' not installed"
  CMD_zip="`which zip.exe`"                ; builder_check_error "Command 'zip' not installed"
  CMD_tar="`which tar.exe`"                ; builder_check_error "Command 'tar' not installed"
  CMD_cabextract="`which cabextract.exe`"  ; builder_check_error "Command 'cabextract' not installed"
  CMD_unix2dos="`which unix2dos.exe`"      ; builder_check_error "Command 'unix2dos' not installed"
  CMD_sha1sum="`which sha1sum.exe`"        ; builder_check_error "Command 'sha1sum' not installed"
  CMD_inifile="`which inifile.exe`"        ; builder_check_error "Command 'inifile' not installed (http://www.horstmuc.de/wbat32d.htm#inifile)"
  CMD_tidy="`which tidy.exe`"              ; builder_check_error "Command 'tidy' not installed"
  CMD_msiexec="`which msiexec.exe`"        ; builder_check_error "Command 'msiexec' not installed"
  CMD_innounp="`which innounp.exe`"        ; builder_check_error "Command 'innounp' not installed"
  CMD_xmlstarlet="`which xmlstarlet.exe`"  ; builder_check_error "Command 'xmlstarlat' not installed"
  CMD_msix="`which MsiX.exe`"              ; builder_check_error "Command 'msix' not installed"
  
  # Check temp dir
  test -d ${TMP_DIR}
  builder_check_error "temp directory not available: $TMP_DIR"
  
  # project dependent configuration
  local config=${PRODUCT_DIR}/builder-product.cfg
  test -f ${config} || builder_check_error "cannot read product config: ${config}"
  . ${config}
  
  # set default build configuration and source the user dependent file
  local config=$BASEDIR/conf/aio-builder.cfg
  . ${config}
  
  #  Source local build configuration (must be done AFTER sourcing the builder-product.cfg.cfg)
  if [ -f "$AIO_BUILDER" ] ; then
    config=$AIO_BUILDER
  else
    test -f $HOME/.aio-builder.cfg && config=$HOME/.aio-builder.cfg
  fi
  
  # Read ONLY the STATUS variable from the build configuration file
  eval "`grep -E "^(STATUS|STATUS_INTEGRATION_RELEASE)=" $config`"
  
  # change some variable from the builder-product.cfg dynamically:
  # autogenerate release number, if we are in status "integration"
  if [ "$STATUS" = "integration" ] ; then
    if [ "${STATUS_INTEGRATION_RELEASE}" = "func:inc1" ] ; then
      . ${config}
      calc_release
    else
      # AIO/control:RELEASE is limited to max 16 chars - take care in regards to the CREATOR_TAG
      RELEASE="${STATUS_INTEGRATION_RELEASE}"
    fi
  fi
  
  # Read configurationfile
  . ${config}
  echo "Loaded builder configuration: $config"
  
  # Check variables
  if [ -z ${AIO_REPOS_BASE_DIR} ] || [ ! -d ${AIO_REPOS_BASE_DIR} ] ; then
    echo "configuration error: AIO_REPOS_BASE_DIR directory does not exist: $AIO_REPOS_BASE_DIR"
    exit 2
  fi
  
  if [  "$TYPE" != "public" ] && [  "$TYPE" != "restricted" ]  ; then
    fatal_error "unknown TYPE: $TYPE"
  fi
  
  # Check, if variable is numeric
  if [ ! `expr ${AIO_REPOS_PURGE_LIMIT} + 1 2> /dev/null` ]   ; then
    fatal_error "AIO_REPOS_PURGE_LIMIT must be numeric"
  fi
  
}

#####################
# Prepare build
####################
builder_prepare() {
  echo "builder_prepare: "
  # Check if the package is still build
  if  [  "$AIO_REPOS_FORCE_UPLOAD" != "true" ] && [ -f "${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z" ]  ; then
    echo "File ${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z already exists."
    exit 1
  fi
  
  mkdir -p $DIST_CACHE_DIR
  log_debug "Distribution directory: $DIST_CACHE_DIR"
  
  # setup work directory
  OUTPUT_DIR="$TMP_DIR/aio-builder.`date +%Y%m%d-%H%M%S`.$$"
  mkdir -p ${OUTPUT_DIR}
  builder_check_error "Cannot create temp directory ${OUTPUT_DIR}"
  
  # prepare
  INST_DIR=$OUTPUT_DIR/$PN
  mkdir $INST_DIR
  
  log_info "  OUTPUT_DIR: $OUTPUT_DIR"
}


#####################
# Download all dist files from one of the defined URLs.
# and validate the checksum
####################
builder_retrieve() {
  
  for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do
    local basename=${DL_FILE[$i]}
    local urls=${DL_SOURCE[$i]}
    local arch=${DL_ARCH[$i]}
    local downloaded=0
    
    # Add private repos to the urls
    if [ ! -z ${DIST_PRIVATE_REPOS} ]; then
      urls="${DIST_PRIVATE_REPOS}/$basename;$urls"
    fi
    
    # check existence of CRC file only in non devel mode
    if [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" != "true" ] ; then
      fatal_error "You need to create the checksums with: sha1sum ${DIST_CACHE_DIR}/${basename}  > ${PRODUCT_DIR}/${basename}.sha1sum"
    fi
    
    echo "Downloading $basename"
    # check downloading from the defined URLs
    for src in `echo  $urls | sed -e 's/[;,]/\n/g'`  ; do
      if [ $downloaded == 1 ]; then continue; fi
      
      # Download file
      echo "  Info: Downloding from $src"
      local downloader=${DL_DOWNLOADER[$i]}
      if [ -z $downloader ]; then downloader="wget" ; fi
      
      mkdir -p ${DIST_CACHE_DIR}/$arch
      DL_DIST_FILE[$i]=${DIST_CACHE_DIR}/$arch/$basename
      retrieve_file $downloader $src  ${DL_DIST_FILE[$i]}
      
      if [ $? != 0 ] ; then
        echo "  Warning: Failed to download file - try next URL"
        continue;
      fi
      
      # Check sha1
      if  [ ! -e "${PRODUCT_DIR}/${basename}.sha1sum" ] && [ "$CHECKSUM_AUTOCREATE" == "true" ] ; then
        $CMD_sha1sum ${DL_DIST_FILE[$i]}  > ${PRODUCT_DIR}/${basename}.sha1sum
        downloaded=1
        echo "  WARNING: SHA1 checksum (${DL_DIST_FILE[$i]}.sha1sum) was created dynamically because auf CHECKSUM_AUTOCREATE=$CHECKSUM_AUTOCREATE"
      else
        # testing the checksum of the downloaded files
        local sha1sum_val=`cat ${PRODUCT_DIR}/${basename}.sha1sum | cut -d " " -f1`
        local checksum_val=`sha1sum ${DL_DIST_FILE[$i]} | cut -d " " -f1`
        if [ "$checksum_val" = "$sha1sum_val" ] ; then
          downloaded=1
        fi
      fi
      
      # Print result
      if [ "$downloaded" == "1" ] ; then
        echo "  Info: Downloaded successfully"
      else
        echo "  Error: The checksums do not match - try next URL"
      fi
      
    done
    echo
    
    # Ups - no URL works
    if [ $downloaded != 1 ] ; then
      echo "  Error: can download the file or checksum wrong (sha1sum ${DIST_CACHE_DIR}/${basename}  > ${basename}.sha1sum)"
      exit 1;
    fi
    
  done
}

#####################
# Create files
####################
builder_create() {
  
  # Copy files and convert text files to dos format
#  cp -Rv ${PRODUCT_DIR}/AIO         $INST_DIR
 # cp -Rv ${PRODUCT_DIR}/Temp  $INST_DIR
 # if [ -d "${PRODUCT_DIR}/SERVER_DATA" ] ; then
#    cp -Rv ${PRODUCT_DIR}/SERVER_DATA  $INST_DIR
 # fi
  
  # copy binaries
  for (( i = 0 ; i < ${#DL_SOURCE[@]} ; i++ )) ; do
    DL_EXTRACT_PATH[$i]=${INST_DIR}/Temp/${DL_ARCH[$i]}/${DL_EXTRACT_TO[$i]}
    local format=${DL_EXTRACT_FORMAT[$i]}
    if [ -z "$format" ] ; then format="cp"; fi
    local option=${DL_EXTRACT_OPTION[$i]}
    if [ -z "$option" ] ; then option=""; fi
    
    mkdir -p ${DL_EXTRACT_PATH[$i]}
    process_file $format ${DL_DIST_FILE[$i]} ${DL_EXTRACT_PATH[$i]} $option
  done
  
  # write ini file
  local ini_file=${INST_DIR}/aio-$PN.ini

  for var in VENDOR PN VERSION RELEASE TYPE CREATOR_TAG CREATOR_NAME CREATOR_EMAIL ; do
    write_ini_file $ini_file "COMMON" ${var}="${!var}"
  done

  # convert to dos file linefeed
#  find $INST_DIR -type f | xargs -n1 -iREP sh -c 'file -i $0 | grep -v "utf-16" | grep "text/plain" && '$CMD_unix2dos' $0 ' REP >/dev/null

  # set exec bit on executeables
#  find $INST_DIR -type f -iname "*.exe" -o -iname "*.bat" -o -iname "*.cmd" -o -iname "*.msi" -o -iname "*.msp" | xargs chmod +x -v

  # Create changelog based on git - if available
  if [ -d "${PRODUCT_DIR}/.git" ] ; then
    # new changelog format
    echo "" >> $INST_DIR/changelog.txt
    echo "[Changelog]" >> $INST_DIR/changelog.txt
    #$CMD_ruby $BASEDIR/libexec/gitlog-to-deblog.rb >> $INST_DIR/AIO/changelog.txt
    echo "" >> $INST_DIR/changelog.txt
  else
    echo "No git repository present."
  fi
  
}

#####################
# build aio package
#####################
builder_package() {
  
  # creating package
  local release_new=${CREATOR_TAG}${RELEASE}
  local aio_file=${PN}_${VERSION}-${release_new}.7z
  
  rm -rf $INST_DIR/Temp
  pushd ${OUTPUT_DIR}
  rm -f ${aio_file} ${AIO_REPOS_FILE_PATTERN}.7z
  $CMD_7z a ${AIO_REPOS_FILE_PATTERN}.7z  $INST_DIR/*
  builder_check_error "Building AIO-package"
  popd
  
  # rename aio package file
  if [ "${aio_file}" != "${AIO_REPOS_FILE_PATTERN}.7z" ]; then
    mv ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z
    builder_check_error "can't move file  ${OUTPUT_DIR}/${aio_file} ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.7z"
  fi
  
  # --exclude \*/.git\*
  # create source- and binary package package
  test "${AIO_REPOS_UPLOAD_BIN}" = "true"    && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}.zip $INST_DIR
  test "${AIO_REPOS_UPLOAD_SOURCE}" = "true" && $CMD_zip --exclude \*/.git\* @ -r ${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}-src.zip ${PRODUCT_DIR}
}


#####################
# publish
#####################
builder_publish() {
  
  # Upload file to repository
  mkdir -p ${AIO_REPOS_PRODUCT_DIR}

  echo "Publishing aio-package to ${AIO_REPOS_PRODUCT_DIR}"
  local src=${OUTPUT_DIR}/${AIO_REPOS_FILE_PATTERN}
  local dst=${AIO_REPOS_PRODUCT_DIR}/${AIO_REPOS_FILE_PATTERN}
  
  # Link dir
  if [ "${AIO_REPOS_LINK_NEWBUILDS}" = "true" ] ; then
    mkdir -p ${AIO_REPOS_BASE_DIR}/.new_builds
    ln -sf ${AIO_REPOS_PRODUCT_DIR}  ${AIO_REPOS_BASE_DIR}/.new_builds/${AIO_REPOS_FILE_PATTERN}
    builder_check_error "Can't Link file $dst.7z --> $dst.7z"
  fi
  
  # copy files
  if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then
    cp  ${src}.7z  ${dst}.7z
    builder_check_error "Can't upload file $dst.7z --> $dst.7z"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_BIN}" = "true" ] ; then
    cp  ${src}.zip   ${dst}.zip
    builder_check_error "Can't upload file $dst.zip --> $dst.zip"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_SOURCE}" = "true" ] ; then
    cp  ${src}-src.zip   ${dst}-src.zip
    builder_check_error "Can't upload file ${dst}-src.zip --> ${dst}-src.zip"
  fi
  
  if [ "${AIO_REPOS_AIOMANAGER_INSTALL}" = "true" ] ; then
    aio-package-manager -i -v ${src}.7z
    builder_check_error "Can't install ${src}.7z"
  fi
  
  if [ "${AIO_REPOS_UPLOAD_AIO}" = "true" ] ; then
    md5sum "${src}.7z" | sed 's/ .*//' > ${dst}.7z.md5
    builder_check_error "Can't create md5 file"
  fi

  if [ "${AIO_REPOS_UPLOAD_AIO_GPG}" = "true" ] ; then
    ${CMD_gpg} --batch --passphrase ${GPG_PASSPHRASE} --output "${dst}.7z.gpg" --detach-sig "${src}.7z"
    builder_check_error "Can't create gpg file"
  fi

  # Create revision file for this
  local rev_file=${AIO_REPOS_PRODUCT_DIR}/${PN}-${VERSION}-${CREATOR_TAG}${RELEASE}.cfg
    cat > $rev_file <<EOF
REV_VENDOR=${VENDOR}
REV_PN=${PN}
REV_NAME="${NAME}"
REV_VERSION=${VERSION}
REV_RELEASE=${RELEASE}
REV_TYPE=${TYPE}
REV_STATUS=${STATUS}
REV_TIMESTAMP=`date +"%s"`
REV_CREATOR_TAG=${CREATOR_TAG}
REV_AIO_REPOS_FILE_PATTERN=${AIO_REPOS_FILE_PATTERN}
EOF
  
  
  # Purge old product versions - defined by limit AIO_REPOS_PURGE_LIMIT
  if [ "${AIO_REPOS_PURGE}" = "true" ]  && [ ! -z "${AIO_REPOS_PURGE_LIMIT}" ]  && [ "${AIO_REPOS_PURGE_LIMIT}" > 0 ] && [ "${STATUS}" = "${AIO_REPOS_PURGE_STATUS}" ] ; then
    echo "Autopurging enabled"
    
    # determinte max version to delete
    local limit
    local pn_limit=`echo ${PN} | sed "s/[\.\-]/_/g"`
    eval "`echo limit=\\$\\{AIO_REPOS_PURGE_LIMIT_${pn_limit}\\}`"
    if [ -z "$limit" ] || [ ! `expr $limit + 1 2>/dev/null` ]  ; then
      limit=${AIO_REPOS_PURGE_LIMIT}
    fi
    echo "  Purging, max. number of versions: $limit"
    
    # Find all revision files and sort them
    local file_list=${OUTPUT_DIR}/product-file-list.txt
    local file_sort_list=${OUTPUT_DIR}/product-file-sort-list.txt
    local file_sort_list_version=${OUTPUT_DIR}/product-file-sort-list-version.txt
    local file_sort_list_release=${OUTPUT_DIR}/product-file-sort-list-release.txt
    local file_sort_list_final=${OUTPUT_DIR}/product-file-sort-list-final.txt
    rm -f ${file_list}
    
    # first uniq sort all cfg based on version
    for cfg_file in `find ${AIO_REPOS_PRODUCT_DIR} -name "${PN}-*.cfg" -print ` ; do
      . ${cfg_file}
      echo $REV_VERSION >> ${file_list}
    done
    sort -V ${file_list} | uniq > ${file_sort_list_version}
    
    # second uniq sort all versions based in release
    for pkg_version in `cat ${file_sort_list_version}` ; do
      for cfg_file_ver in ${AIO_REPOS_PRODUCT_DIR}/${PN}-${pkg_version}-*.cfg ; do
        . ${cfg_file_ver}
        echo ${pkg_version}-$REV_CREATOR_TAG$REV_RELEASE >> ${file_sort_list_release}
      done
    done
    sort -V ${file_sort_list_release} | uniq > ${file_sort_list_final}
    
    # third create versionrelease
    for release_file_list in `cat ${file_sort_list_final}` ; do
      . ${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg
      echo "${AIO_REPOS_PRODUCT_DIR}/${PN}-${release_file_list}.cfg" >> ${file_sort_list}
    done
    
    # Delete the oldest files
    log_debug "base list for calculate purge:"
    for cfg_sort_file in `head -n-${limit} ${file_sort_list}` ; do
      
      . ${cfg_sort_file}
      if [ "${REV_STATUS}" != "${AIO_REPOS_PURGE_STATUS}" ] ; then continue; fi
      
      dir_base=`dirname ${cfg_file}`
      product_file="${dir_base}/${REV_AIO_REPOS_FILE_PATTERN}"
      echo "  Purging product version: $product_file*"
      
      # Paranoid ... check the files to delete first
      if [ ! -z "${dir_base}" ] && [ -d "${AIO_REPOS_BASE_DIR}" ] && [ ! -z "$product_file" ] ; then
        rm -f ${product_file}* ${cfg_sort_file}
        
        # remove directory - if it's empty
        if [ $(ls -1A ${dir_base} | wc -l) -eq 0 ]; then
          rmdir ${dir_base}
        fi
      fi
    done
  fi
}

###################
# Commiting changes to repos
###################
builder_commit() {
  if [ -d "${PRODUCT_DIR}/.git" ]; then
    echo -n
    log_debug "builder_commit() not implemented yet."
  fi
}


#####################
# build aio package
#####################
builder_cleanup() {
  # Paranoia
  if [ -d "$OUTPUT_DIR" ] && [[ $OUTPUT_DIR == $TMP_DIR/aio-builder.* ]] ; then
    rm -rf $OUTPUT_DIR
  fi
}