Merge pull request #24 from Sabayon/develop

New features

builder

@@ -20,6 +20,7 @@ use Getopt::Long;
 use v5.10;
 no feature "say";
 use Storable 'dclone';
+use Sort::Versions;
 
 my $profile              = $ENV{BUILDER_PROFILE} // 16;
 my $jobs                 = $ENV{BUILDER_JOBS} // 1;
@@ -38,6 +39,7 @@ my $entropy_repository   = $ENV{ENTROPY_REPOSITORY}
     // "main";    # Can be weekly, main, testing
 my $artifacts_folder          = $ENV{ARTIFACTS_DIR};
 my $dep_scan_depth            = $ENV{DEPENDENCY_SCAN_DEPTH} // 2;
+my $dep_ignore_versions       = $ENV{DEPENDENCY_IGNORE_VERSIONS} // 0;
 my $skip_portage_sync         = $ENV{SKIP_PORTAGE_SYNC} // 0;
 my $emerge_split_install      = $ENV{EMERGE_SPLIT_INSTALL} // 0;
 my $webrsync                  = $ENV{WEBRSYNC} // 0;
@@ -176,6 +178,146 @@ auto-sync = yes' > /etc/portage/repos.conf/$reponame.conf
 # Input: package, depth, and atom. Package: sys-fs/foobarfs, Depth: 1 (depth of the package tree) , Atom: 1/0 (enable disable atom output)
 my %package_dep_cache;
 
+sub parse_package_str {
+  my ($pkg, $p, $cat, $v) = @_;
+  my @parts = split '/', $pkg;
+  $p = $parts[1];
+  $cat = $parts[0];
+
+  $regex = '[-]r[\d]$';
+  $regex .= '|_p[\d]+[-]r[\d]$';
+  $regex .= '|_p[\d]+$';
+  $regex .= '|_alpha[\d]*$';
+  $regex .= '|_beta[\d]*$';
+  $regex .= '|_pre[\d]*$';
+  $regex .= '|_pre[\d]*[-]r[\d]$';
+  $regex .= '|_rc[\d]*$';
+
+  # Drop extra version suffix for now.
+  $p =~ s/($regex)//g;
+
+  $v = $p;  # Version regex
+  # 1.1
+  $version_regex = "[0-9]+[.][0-9]+[a-z]*";
+  # 1
+  $version_regex .= "|[0-9]+[a-z]*";
+  # 1.1.1
+  $version_regex .= "|[0-9]+[.][0-9]+[.][0-9]+[a-z]*";
+  # 1.1.1.1
+  $version_regex .= "|[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*";
+  # 1.1.1.1.1
+  $version_regex .= "|[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*";
+  # 1.1.1.1.1.1
+  $version_regex .= "|[0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[.][0-9]+[a-z]*";
+  $p =~ m/[-]($version_regex)*$/;
+  $v = $&; $v = substr $v, 1 if length($v) > 1;
+
+  $p =~ s/-[\d.]*$//g;
+
+  return $cat, $p, $v;
+}
+
+sub parse_package_version {
+  my ($cat, $p, $v) = parse_package_str(@_);
+  return $v;
+}
+
+sub simplify_package {
+  my ($cat, $p, $v) = parse_package_str(@_);
+  $ans = $cat . "/" . $p ;
+  return $ans;
+}
+
+sub sanitize_deps {
+  my ($pkg, $atom, $dref, $verion) = @_;
+  my @dependencies = @{ $dref };
+  my $prev_key = undef;
+  my %revs = ();
+
+  if ($dep_ignore_versions) {
+    for (@dependencies) {
+      chomp($_);
+      if (length($_) > 0) {
+        if ($_ =~ m/\[\s+\d\]/) {
+          my $pkgs = $revs{$prev_key};
+          push(@$pkgs, $_);
+        } else {
+          $prev_key = substr $_, 0, length($_)-1
+            unless substr($_, -1) cmp ":";
+          my ($pkg, $p, $v) = parse_package_str($prev_key);
+          $revs{$v} = [];
+          $prev_key = $v;
+        }
+      }
+    }
+
+    # revs dictionary contains as keys versions of the package
+    # and as values an array ref with list of dependencies + itself
+    # Ex:
+    # %revs = (
+    #   '1.8.5' => [
+    #      ' [  0]  dev-lang/spidermonkey-1.8.5-r9   ',
+    #      ' [  1]  dev-libs/nspr-4.22   ',
+    #      ' [  1]  sys-libs/readline-7.0_p5   ',
+    #      ' [  1]  dev-libs/jemalloc-3.6.0   ',
+    #      ' [  1]  dev-lang/python-2.7.16   ',
+    #      ' [  1]  app-arch/zip-3.0-r3   ',
+    #      ' [  1]  virtual/pkgconfig-1   ',
+    #      ' [  1]  app-portage/elt-patches-20170815   ',
+    #      ' [  1]  sys-devel/automake-1.16.1-r1   ',
+    #      ' [  1]  sys-devel/automake-1.15.1-r2   ',
+    #      ' [  1]  sys-devel/autoconf-2.13-r1   ',
+    #      ' [  1]  sys-devel/libtool-2.4.6-r3   '
+    #   ],
+    # )
+
+
+    $v = parse_package_version($pkg);
+    if (length($v) > 0 && defined %revs{$v}) {
+      $version = $v;
+    } else {
+      @versions = sort { &Sort::Versions::versioncmp($a,$b) } keys %revs;
+      $version = $versions[scalar(@versions)-1];
+    }
+    $ref = $revs{$version};
+
+    loud("For package $pkg use version $version with deps: ", @$ref)
+    if $verbose;
+
+    @dependencies = uniq(
+        grep {$_}
+        map {
+          $_ =~ s/\[.*\]|\s//g;
+          $_ =~ s/[:]$//g; # drop : at the end
+          &abs_atom($_) if $atom;
+          $_
+        }
+        @$ref
+    );
+  } else {
+    # Merge all versions dependencies as previous implementation.
+    #
+# If an unversioned atom is given, equery returns results for all versions in the portage tree
+# leading to duplicates. The sanest thing to do is dedup the list. This gives the superset of all
+# possible dependencies, which isn't perfectly accurate but should be good enough. For completely
+# accurate results, pass in a versioned atom.
+    @dependencies = uniq(
+        sort
+        grep {$_}
+        map {
+          $_ =~ s/\[.*\]|\s//g;
+          $_ =~ s/[:]$//g; # drop : at the end
+          &abs_atom($_) if $atom;
+          $_
+        }
+        @dependencies
+    );
+
+  }
+
+  return sort(@dependencies);
+}
+
 sub package_deps {
     my $package = shift;
     my $depth   = shift // 1;   # defaults to 1 level of depthness of the tree
@@ -189,17 +331,8 @@ sub package_deps {
             qx/equery -C -q g --depth=$depth '$package'/;    #depth=0 it's all
         chomp @dependencies;
 
-# If an unversioned atom is given, equery returns results for all versions in the portage tree
+        # Get dependencies only of major version
-# leading to duplicates. The sanest thing to do is dedup the list. This gives the superset of all
+        @dependencies = sanitize_deps($package, $atom, \@dependencies);
-# possible dependencies, which isn't perfectly accurate but should be good enough. For completely
-# accurate results, pass in a versioned atom.
-        @dependencies = uniq(
-            sort
-                grep {$_}
-                map { $_ =~ s/\[.*\]|\s//g; &abs_atom($_) if $atom; $_ }
-                @dependencies
-        );
-
         $package_dep_cache{$cache_key} = \@dependencies;
     }
 
@@ -690,6 +823,9 @@ loud "Pre-compilation phase start";
 _system("emerge --info")
     ; #always give detailed information about the building environment, helpful to debug
 
+_system("equo repo list")
+    ; #always give list of repo installed to for reproduce conflicts install issues.
+
 if ( $emerge_remove and $emerge_remove ne "" ) {
     say "Removing with emerge: $emerge_remove";
     _system("emerge -C '$_'") for split( / /, $emerge_remove );

dynlink-scanner

@@ -46,5 +46,5 @@ for cpv in `portageq match $ROOT/ $1`; do
 	# For each file that belongs to package
 	#     run dynlink-scanner --linking-deps <file> to obtain its linking dependencies
 	# Assign all linking deps to packages and print package names
-	qfile -eR $ROOT `portageq contents $ROOT/ $cpv | xargs -r -L 1 "$0" --linking-deps` | cut -f1 -d' ' | sort -u
+	qfile -vR $ROOT `portageq contents $ROOT/ $cpv | xargs -r -L 1 "$0" --linking-deps` | cut -f1 -d' ' | sort -u
 done

sabayon-brokenlinks

@@ -0,0 +1,462 @@
+#!/bin/bash
+# Authors: Sabayon Team 2019
+
+set -e
+
+# Accepted values: x86_64, armv7l
+export SAB_ARCH="${SAB_ARCH:-}"
+export DRY_RUN=${DRY_RUN:-0}
+export DEBUG="${DEBUG:-}"
+export FORCE="${FORCE:-0}"
+export USE_EQUO="${USE_EQUO:-1}"
+
+declare -a MANUALLY_REMOVED_LINKS
+declare -a BROKEN_LINKS2REMOVE
+
+log() {
+  local with_dashes=$1
+  shift || true
+
+  [ "$with_dashes" != 1 ] || \
+    echo "==============================================================="
+  echo "$@"
+  [ "$with_dashes" != 1 ] || \
+    echo "==============================================================="
+}
+
+summary() {
+
+  local msg="SAB_ARCH = ${SAB_ARCH}
+EXAMINED_LIBDIRS = ${EXAMINED_LIBDIRS}
+DRY_RUN = ${DRY_RUN}
+USE_EQUO = ${USE_EQUO}
+FORCE = ${FORCE}"
+
+  log 1 "$msg"
+}
+
+check_arch() {
+  if [ -z "${SAB_ARCH}" ] ; then
+    SAB_ARCH=$(uname -m)
+  fi
+
+  [[ "${SAB_ARCH}" != "x86_64" && "${SAB_ARCH}" != "armv7l" ]] && {
+    log 1 "Unsupported arch ${SAB_ARCH}."
+    exit 1
+  }
+
+  export SAB_ARCH
+}
+
+search_pkg() {
+  local basename=$1
+  # passed by reference
+  local -n p="$2"
+  local -n libdir="$3"
+  local lib="$4"
+  local l=""
+  local ll=""
+
+  # Try to search for package
+  if [ "${USE_EQUO}" = "0" ] ; then
+    p=$(equery -N -q -C belongs ${lib}/$basename --early-out || true)
+  else
+    p=$(equo q belongs ${lib}/${basename} -q || true)
+  fi
+
+  if [ -z "$p" ] ; then
+
+    for l in "/usr/lib64" "/usr/lib32" "/usr/lib" ; do
+      if [ "$lib" = "$l" ] ; then
+        ll="${l/\/usr\///}"
+        break
+      fi
+    done
+
+    if [ -n "${ll}" ] ; then
+      if [ "${USE_EQUO}" = "0" ] ; then
+        p=$(equery -N -q -C belongs ${ll}/$basename --early-out || true)
+      else
+        p=$(equo q belongs ${ll}/${basename} -q || true)
+      fi
+    fi
+
+  fi
+
+  if [ -z "$p" ] ; then
+    libdir=""
+  fi
+}
+
+detect_libs_dir() {
+  local libs=""
+
+  if [ -n "$SCANDIR" ] ; then
+    libs=$SCANDIR
+  else
+    if [ "${SAB_ARCH}" = "x86_64" ] ; then
+      if [ -d "/usr/lib32" ] ; then
+        libs="/usr/lib32 /lib32"
+      fi
+      if [ -d "/usr/lib64" ] ; then
+        libs="$libs /usr/lib64 /lib64"
+      fi
+    else
+      if [ -d "/usr/lib" ] ; then
+        libs="/usr/lib /lib"
+      fi
+    fi
+    if [ -z "$libs" ] ; then
+      log 0 "Error on detect libraries directories."
+      exit 1
+    fi
+  fi
+
+  EXAMINED_LIBDIRS=$libs
+  export EXAMINED_LIBDIRS
+}
+
+examine_libs() {
+  local lib=""
+  local file=""
+  local broken_files=""
+
+  log 1 "The exam could be long sorry for the wait."
+
+  for lib in ${EXAMINED_LIBDIRS} ; do
+    broken_files=""
+
+    log 1 "Checking directory $lib..."
+    broken_files=$(find $lib -type l -and -xtype l -and -name *.so* -or -name *.a -and -type l -and -xtype l 2>/dev/null || true )
+
+    [ -z "$DEBUG" ] || {
+      find $lib -type l -and -xtype l -and -name *.so* -or -name *.a -and -type l -and -xtype l || true
+    }
+
+    for file in ${broken_files} ; do
+      examine_file "${file}" "${lib}"
+    done
+  done
+}
+
+examine_file() {
+  local f=$1
+  local res=""
+  local libdir=$2
+  local is_present_pkg=1
+  local pkg=""
+
+  [ -z "$DEBUG" ] || log 1 "Analyze file $f"
+
+  local base=$(basename $1)
+
+  local targetlink=$(readlink $f)
+  local link2remove=false
+  local link2fix=false
+  local newlibdir=""
+  local tmp=""
+  local current_link=""
+
+  log 1 "Broken link $f => $targetlink"
+
+  # Check if file is without revision (es. libudev.so)
+  res=$(perl -e "if ('$base' =~ /(.so|.a$)/) { print '1' }")
+  if [ -n "$res" ] ; then
+    # 
+    #echo "BASE = $base"
+
+    search_pkg "$base" pkg newlibdir "$lib"
+    if [ -z "$pkg" ] ; then
+      MANUALLY_REMOVED_LINKS+=( "$f" )
+      return
+    else
+
+      if [ -n "$newlibdir" ] ; then
+        log 0 "Link $base available under $newlibdir. I remove it."
+        link2remove=true
+      fi
+    fi
+
+    if [ "${USE_EQUO}" = "0" ] ; then
+      res=$(qlist -IC $pkg 2>/dev/null | grep $f | wc -l)
+    else
+      res=$(equo q files  $pkg -q | grep $f | wc -l)
+    fi
+    if [ "$res" = 1 ] ; then
+      link2fix=true
+      echo "Link $f related to existing package. Try to reinstall $pkg."
+    else
+      MANUALLY_REMOVED_LINKS+=( "$f" )
+    fi
+
+    log 0 "Link $f of the package $pkg"
+
+  else
+
+    # Check if file has only one revision number (es. libudev.so.0)
+    res=$(perl -e "if ('$base' =~ /(.so[.][\d]+$)/) { print '1' }")
+    if [ -n "$res" ] ; then
+
+      search_pkg "$base" pkg newlibdir "$lib"
+      if [ -z "$pkg" ] ; then
+        # Try to search package with .so
+        base=$(echo "$base" | awk 'match($0, /.so/) { print substr($0, 0, RSTART+2) }')
+        search_pkg "$base" pkg newlibdir "$lib"
+        if [ -z "$pkg" ] ; then
+          log 0 "No package found for link $f."
+          MANUALLY_REMOVED_LINKS+=( "$f" )
+          return
+        fi
+
+        if [ -n "$newlibdir" ] ; then
+          log 0 "Link $base available under $newlibdir. I remove it."
+          link2remove=true
+        else
+          # POST: we have a broken link or old link
+
+          current_link=$(readlink $lib/$base)
+          log 0 "Link $f related to package $pkg (with an updated version $current_link). I remove it."
+          link2remove=true
+
+        fi
+
+      else
+
+        log 0 "Link $f of the package $pkg"
+
+        if [ -n "$newlibdir" ] ; then
+          log 0 "Link $base available under $newlibdir. I remove it."
+          link2remove=true
+        else
+
+          # Check new 
+          if [ "${USE_EQUO}" = "0" ] ; then
+            res=$(qlist -IC $pkg 2>/dev/null | grep $f | wc -l)
+          else
+            res=$(equo q files  $pkg -q | grep $f | wc -l)
+          fi
+          if [ "$res" = 1 ] ; then
+            link2fix=true
+            echo "Link $f related to existing package. Try to reinstall $pkg."
+          else
+            MANUALLY_REMOVED_LINKS+=( "$f" )
+          fi
+
+        fi
+
+      fi
+
+    else
+      # Check if file has only one revision number (es. libudev.so.0.63)
+      res=$(perl -e "if ('$base' =~ /(.so[.][\d]+[.][\d]+$)/) { print '1' }")
+      if [ -n "$res" ] ; then
+
+        search_pkg "$base" pkg newlibdir "$lib"
+        if [ -z "$pkg" ] ; then
+
+          base=$(echo "$base" | awk 'match($0, /.so[.][0-9]+/) { print substr($0, 0, RSTART+RLENGTH-1) }')
+          search_pkg "$base" pkg newlibdir "$lib"
+          if [ -z "$pkg" ] ; then
+            log 0 "No package found for link $f."
+            MANUALLY_REMOVED_LINKS+=( "$f" )
+            return
+          fi
+
+          if [ -n "$newlibdir" ] ; then
+            log 0 "Link $base available under $newlibdir. I remove it."
+            link2remove=true
+          else
+
+            current_link=$(readlink $lib/$base)
+            log 0 "Link $f related to package $pkg (with an updated version $current_link). I remove it."
+            link2remove=true
+
+          fi
+
+        else
+
+          log 0 "Link $f of the package $pkg"
+
+          if [ -n "$newlibdir" ] ; then
+            log 0 "Link $base available under $newlibdir. I remove it."
+            link2remove=true
+          else
+
+            # Check new 
+            if [ "${USE_EQUO}" = "0" ] ; then
+              res=$(qlist -IC $pkg 2>/dev/null | grep $f | wc -l)
+            else
+              res=$(equo q files  $pkg -q | grep $f | wc -l)
+            fi
+            if [ "$res" = 1 ] ; then
+              link2fix=true
+              echo "Link $f related to existing package. Try to reinstall $pkg."
+            else
+              MANUALLY_REMOVED_LINKS+=( "$f" )
+            fi
+
+          fi
+
+        fi
+
+      else
+        echo "Link $f not supported".
+        MANUALLY_REMOVED_LINKS+=( "$f" )
+
+      fi
+
+    fi
+
+  fi
+
+  if [ $link2remove = true ] ; then
+    BROKEN_LINKS2REMOVE+=( "$f" )
+  fi
+
+}
+
+rm_broken_links() {
+  for l in ${BROKEN_LINKS2REMOVE[@]} ; do
+    if [ "$DRY_RUN" = 0 ]; then
+      rm -vf ${l}
+    else
+      echo "rm -vf ${l}"
+    fi
+  done
+}
+
+print_links2remove() {
+  for l in ${MANUALLY_REMOVED_LINKS[@]} ; do
+    if [ "$FORCE" = 1 ] ; then
+      if [ "$DRY_RUN" = 0 ] ; then
+        rm -vf ${l}
+      else
+        echo "force: rm -vf ${l}"
+      fi
+    else
+      echo "rm -vf ${l}"
+    fi
+  done
+}
+
+parse_args() {
+
+  help_message() {
+
+    echo "Sabayon Broken Links analyzer.
+Sabayon Team (2019)
+
+$0 [opts]
+
+Available options:
+
+-h|--help         This message.
+--dry-run         Dry run execution.
+--force           Force remove of the links to remove manually.
+--scandir         Define a specific directory to analyze
+--gentoo          Use gentoo tool instead of equo.
+-d|--debug        Enable debug stuff.
+
+Environment variables:
+DRY_RUN           To use instead of --dry-run option. Values: 0 or 1.
+DEBUG             Enable debug output.
+USE_EQUO          To use instead of --gentoo option.
+FORCE             To use instead of --force.
+
+"
+
+  }
+
+  local short_opts="hd"
+  local long_opts="help dry-run force debug scandir: gentoo"
+
+  $(set -- $(getopt -u -q -a -o "$short_opts" -l "$long_opts" -- "$@"))
+
+  while [ $# -gt 0 ] ; do
+    case "$1" in
+
+      -h|--help)
+        help_message
+        exit 1
+        ;;
+      -d|--debug)
+        DEBUG=1
+        ;;
+      --force)
+        FORCE=1
+        ;;
+      --dry-run)
+        DRY_RUN=1
+        ;;
+      --scandir)
+        SCANDIR="$2"
+        shift
+        ;;
+      --gentoo)
+        USE_EQUO=0
+        ;;
+      --)
+        ;;
+      *)
+        echo "Invalid parameter $1."
+        exit 1
+        ;;
+
+    esac
+
+    shift
+
+  done
+
+  unset -f help_message
+}
+
+main() {
+
+  parse_args "$@"
+
+  log 1 "sabayon-brokenlinks: Examine system broken links"
+  check_arch
+  detect_libs_dir
+  summary
+
+  if [[ "$(id -u)" != "0" && "${DRY_RUN}" != "1" ]]; then
+     echo "This script must be run as root" 1>&2
+     exit 1
+  fi
+
+  parse_args
+  unset -f parse_args
+
+  if [ "${USE_EQUO}" = "0" ] ; then
+    which equery 2>&1 >/dev/null || {
+      echo "equery tool is needed. Please install app-portage/gentoolkit."
+      exit 1
+    }
+
+    which qlist 2>&1 >/dev/null || {
+      echo "qlist tool is needed. Please install app-portage/portage-utils."
+      exit 1
+    }
+  fi
+
+  examine_libs
+
+  if [ ${#BROKEN_LINKS2REMOVE[@]} -gt 0 ] ; then
+    log 1 "Remove Broken Links!"
+    rm_broken_links
+  fi
+
+  if [ ${#MANUALLY_REMOVED_LINKS[@]} -gt 0 ] ; then
+    if [ "${FORCE}" = 1 ] ; then
+      log 1 "Hereinafter, links removed with force:"
+    else
+      log 1 "Hereinafter, links to manually remove:"
+    fi
+    print_links2remove
+  fi
+
+  echo "All Done."
+}
+
+main $@

sabayondevkit-functions.sh

@@ -8,7 +8,7 @@ check_docker_requirements(){
   if [ "$(id -u)" != "0" ]; then
     groups | grep -q docker || echo "--> If you are not running the script as root, your user should be in the docker group to use it. (sudo gpasswd -a $USER docker)"
   fi
-  ps aux | grep -q '[d]ocker' || echo "--> Be sure to have the docker daemon running (sudo systemctl start docker) of configure it to run on boot (sudo systemctl enable docker). Trying to start it anyway" && systemctl start docker || true
+  ps aux | grep -q '[d]ocker' || echo "--> Be sure to have the docker daemon running (sudo systemctl start docker) or configure it to run on boot (sudo systemctl enable docker). Trying to start it anyway" && systemctl start docker || true
 }
 
 die() { echo "$@" 1>&2 ; exit 1; }