Attempt to only add/remove enman repos that need action

Since enman was changed to provide exit status 1 [1] if the requested
operation failed, builder attempting to re-add an already existing repo
caused the entire build to fail. Rather than run the command and hope
for the best, this commit changes builder to obtain a list of already
installed repositories, and only add/remove the requested repos that
aren't already in the correct state.

[1] Sabayon/enman@7f3a6c88

builder

@@ -15,7 +15,7 @@ $ENV{FEATURES} = $ENV{FEATURES}
 
 my $equo_install_atoms   = $ENV{EQUO_INSTALL_ATOMS}   // 1;
 my $equo_install_version = $ENV{EQUO_INSTALL_VERSION} // 0;
-my $equo_split_install   = $ENV{EQUO_SPLIT_INSTALL}   // 1;
+my $equo_split_install   = $ENV{EQUO_SPLIT_INSTALL}   // 0;
 my $equo_mirrorsort      = $ENV{EQUO_MIRRORSORT}      // 1;
 my $entropy_repository   = $ENV{ENTROPY_REPOSITORY}
   // "main";    # Can be weekly, main, testing
@@ -123,17 +123,17 @@ auto-sync = yes' > /etc/portage/repos.conf/$reponame.conf
 }
 
 # Input: package, depth, and atom. Package: sys-fs/foobarfs, Depth: 1 (depth of the package tree) , Atom: 1/0 (enable disable atom output)
+my %package_dep_cache;
 sub package_deps {
     my $package = shift;
     my $depth   = shift // 1;    # defaults to 1 level of depthness of the tree
     my $atom    = shift // 0;
 
 # Since we expect this sub to be called multiple times with the same arguments, cache the results
-    state %cache;
     $cache_key = "${package}:${depth}:${atom}";
 
-    if ( !exists $cache{$cache_key} ) {
+    if ( !exists $package_dep_cache{$cache_key} ) {
-        @dependencies =
+        my @dependencies =
           qx/equery -C -q g --depth=$depth $package/;    #depth=0 it's all
         chomp @dependencies;
 
@@ -148,10 +148,10 @@ sub package_deps {
               @dependencies
         );
 
-        $cache{$cache_key} = \@dependencies;
+        $package_dep_cache{$cache_key} = \@dependencies;
     }
 
-    return @{ $cache{$cache_key} };
+    return @{ $package_dep_cache{$cache_key} };
 }
 
 #Input: nothing
@@ -178,18 +178,26 @@ sub calculate_missing {
     my @dependencies = package_deps( $package, $depth, 1 );
 
     if ($prune_virtuals) {
-        say "[$package] Pruning dependencies of virtual packages";
         my %install_dependencies = map { $_ => 1 } @dependencies;
 
         # Look for any virtuals and remove its immediate dependencies to avoid
         # installing multiple conflicting packages one by one
-        my @virtual_deps;
+        my @virtual_deps = ();
         for my $dep (@dependencies) {
-            push( @virtual_deps, package_deps( $dep, 1, 1 ) )
+            if ( $dep =~ /^virtual\// ) {
-              if ( $dep =~ /^virtual\// );
+                my @child_deps = package_deps($dep, 1, 1);
+                push( @virtual_deps, @child_deps );
+            }
         }
+
+        # Deduplicate the list
+        @virtual_deps = uniq(@virtual_deps);
+
+        # Prune the dependencies of the virtual packages from the dependencies list
         for my $dep (@virtual_deps) {
-            $install_dependencies{$dep} = 0 if ( $dep !~ /^virtual\// );
+            if ( $dep !~ /^virtual\// ) {
+                $install_dependencies{$dep} = 0;
+            }
         }
         @dependencies = grep { $install_dependencies{$_} } @dependencies;
     }
@@ -203,6 +211,10 @@ sub calculate_missing {
     my @to_install = grep( defined $available_packs{$_},
         uniq( grep( !defined $installed_packs{$_}, @dependencies ) ) );
     @to_install = grep { length } @to_install;
+
+    # Strip out the target package from the dependency list
+    @to_install = grep { to_atom($_) ne to_atom($package) } @to_install;
+
     say "[$package] packages that will be installed with equo: @to_install"
       if @to_install > 0;
 
@@ -212,6 +224,8 @@ sub calculate_missing {
 # Input : complete gentoo package (sys-fs/foobarfs-1.9.2)
 # Output: atom form (sys-fs/foobarfs)
 sub atom { s/-[0-9]{1,}.*$//; }
+# Same again as a function
+sub to_atom { my $p=shift; $p =~ s/-[0-9]{1,}.*$//; return $p; }
 
 # Input: Array
 # Output: array with unique elements
@@ -314,6 +328,8 @@ if ( $remote_portdir ne "" ) {
     system("chown -R portage:portage /usr/portage");
     system("chmod -R ug+w,a+rX /usr/portage");
 }
+system("mkdir /var/lib/layman") if ( ! -d "/var/lib/layman" );
+system("touch /var/lib/layman/make.conf && layman-updater -R") if ( ! -e "/var/lib/layman/make.conf" );
 
 system("echo 'y' | layman -f -a $_") for @overlays;
 
@@ -377,7 +393,9 @@ unless ( $skip_portage_sync == 1 ) {
 }
 
 # preparing for MOAR automation
-qx|eselect profile set $profile|;
+say "Setting new profile to $profile" if defined $profile;
+qx|eselect profile set $profile| if defined $profile;
+system("eselect profile list") if defined $profile;
 
 if ( $use_equo && $entropy_repository eq "weekly" ) {
     qx|equo repo disable sabayonlinux.org|;
@@ -394,14 +412,24 @@ if ($use_equo) {
     say "Devkit version:";
     system("equo s -vq app-misc/sabayon-devkit");
 
+    my @installed_enman_repos = split(/\n/, chomp(system("enman list --installed -q")));
+
     if ( $enman_repositories and $enman_repositories ne "" ) {
         my @enman_toadd = split( / /, $enman_repositories );
-        safe_call("enman add $_") for @enman_toadd;
+        for my $enman_add (@enman_toadd) {
+            if ! grep ($enman_add, @installed_enman_repos) {
+                safe_call("enman add $_");
+            }
+        }
     }
 
     if ( $remove_enman_repositories and $remove_enman_repositories ne "" ) {
         my @enman_toremove = split( / /, $remove_enman_repositories );
-        system("enman remove $_") for @enman_toremove;
+        for my $enman_remove (@enman_toremove) {
+            if grep ($enman_remove, @installed_enman_repos) {
+                safe_call("enman remove $_");
+            }
+        }
     }
 
     system("enman add $repository_name")

sabayon-buildpackages

@@ -76,11 +76,12 @@ if [ -d "$SAB_WORKSPACE"/specs ]; then
   [ -e "$SAB_WORKSPACE"/specs/custom.env ] && docker_volumes+=(-v "$SAB_WORKSPACE/specs/custom.env:/opt/sabayon-build/conf/$SAB_ARCH/portage/package.env")
   [ -e "$SAB_WORKSPACE"/specs/custom.keywords ] && docker_volumes+=(-v "$SAB_WORKSPACE/specs/custom.keywords:/opt/sabayon-build/conf/$SAB_ARCH/portage/package.keywords/99-custom.keywords")
   [ -d "$SAB_WORKSPACE"/specs/env ] && docker_volumes+=(-v "$SAB_WORKSPACE/specs/env/:/opt/sabayon-build/conf/$SAB_ARCH/portage/env/")
+  [ -d "/vagrant" ] && docker_volumes+=(-v "/vagrant:/vagrant")
 
 fi
 
 echo "Spawning the package builder container for '$@'."
 echo ""
-#echo "docker run $DOCKER_OPTS ${docker_env[@]} ${docker_volumes[@]} $DOCKER_IMAGE $@"
+echo "docker run $DOCKER_OPTS \"${docker_env[@]}\" \"${docker_volumes[@]}\" $DOCKER_IMAGE $@"
 
 docker run $DOCKER_OPTS "${docker_env[@]}" "${docker_volumes[@]}" $DOCKER_IMAGE $@

sabayon-createrepo

@@ -88,8 +88,8 @@ database-format = bz2
 # disabled-eapis = 1,2
 # expiration-based-scope = disable
 # nonfree-packages-directory-support = disable
-rss-feed = disable
+rss-feed = enable
-changelog = disable
+changelog = enable
 rss-name = packages.rss
 rss-base-url = http://packages.sabayon.org/?quicksearch=
 rss-website-url = http://www.sabayon.org/

sabayon-maint-helper

@@ -0,0 +1,94 @@
+#!/bin/bash
+set -e
+. /sbin/sabayondevkit-functions.sh
+
+help_msg() {
+
+   echo "
+ sabayon-maint-helper:
+
+Script for builder boxes maintenance.
+
+-h|--help        This message.
+--installed      Upgrades all the packages installed in the box.
+--obsolete       Upgrades ONLY the packages that are listed obsolete in eix-test-obsolete.
+--all            Tries to upgrade all package without any euristics
+--category       Tries to upgrade/install all the packages in the specified category
+                 e.g. sabayon-maint-helper 'app-office'
+
+Some options could be set also as environment variables:
+EMERGE_DEFAULT_ARGS     Default emerge options
+
+NOTE: This script MUST be execute as root.
+"
+}
+
+
+parse_args() {
+
+   local short_opts="h"
+   local long_opts="help arch: host-arch: unprivileged stagingdir: image-descr: image:"
+
+   $(set -- $(getopt -u -q -a -o "$short_opts" -l "$long_opts" -- "$@"))
+
+   while [ $# -gt 0 ] ; do
+      case "$1" in
+
+         -h|--help)
+            help_msg
+            exit 1
+            ;;
+         --installed)
+            echo "Trying to upgrade all the installed packages"
+            rebuild_all
+            ;;
+         --obsolete)
+            echo "Upgrading the packages that are marked as obsolete"
+            build_obsolete
+            ;;
+         --all)
+            echo "Upgrading/Installing everything available to install"
+            build_all_availables
+            ;;
+         --category)
+            echo "Upgrading/Installing everything inside the '$2' category"
+            build_category_installed "$2"
+            shift
+            ;;
+         --)
+            help_msg
+            exit 1
+            ;;
+         *)
+            echo "Invalid parameter $1."
+            exit 1
+            ;;
+
+      esac
+
+      shift
+   done
+
+   return 0
+}
+
+main() {
+
+  if [ "$(id -u)" != "0" ]; then
+     echo "This script must be run as root" 1>&2
+     exit 1
+  fi
+
+  [ -z "$@" ] && help_msg && exit 1
+
+  (
+    echo "Acquiring lock"
+    flock -x -w 10 200 || ( echo "Failed acquiring lock in 10 seconds" && exit 1 )
+
+    parse_args "$@"
+
+  ) 200>/var/lock/.repomaint
+
+}
+
+main "$@"

sabayondevkit-functions.sh

@@ -9,51 +9,76 @@ check_docker_requirements(){
 
 die() { echo "$@" 1>&2 ; exit 1; }
 
-build() {
-local SEARCH=$1
-local EMERGE_DEFAULT_ARGS=${2:---accept-properties=-interactive --verbose --oneshot --nospinner --noreplace --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg}
-for i in $(EIX_LIMIT=0 eix --only-names --pure-packages "$SEARCH/" | xargs echo | uniq);
-  do
-    echo "Building $i"
-    emerge $EMERGE_DEFAULT_ARGS -o $i && \
-    emerge $EMERGE_DEFAULT_ARGS $i
-  done
-}
-
-build_installed() {
-  local SEARCH=$1
-  local EMERGE_DEFAULT_ARGS=${2:---accept-properties=-interactive --newuse --noreplace --changed-use --update --verbose --oneshot --nospinner --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg}
-
-  for i in $(EIX_LIMIT=0 eix -I --only-names --pure-packages "$SEARCH/" | xargs echo | uniq);
-    do
-      echo "Building $i"
-      emerge $EMERGE_DEFAULT_ARGS $i
-    done
-}
-
-build_all_availables() {
-for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq);
-  do
-    build "$i/"
-  done
-}
-
-rebuild_all() {
-for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq);
-  do
-    build_installed "$i/"
-  done
-}
-
 build_sync() {
-  emerge-webrsync || exit 1
-  emerge --sync
   layman -S
+  emerge --sync
   eix-update || exit 1
   pushd /opt/sabayon-build/
     git stash
     git fetch --all
     git checkout master
-    git reset --heard origin/master
+    git reset --hard origin/master
   popd
 }
+
+build_category() {
+  local SEARCH=$1
+  build_sync
+  for i in $(EIX_LIMIT=0 eix --only-names --pure-packages "$SEARCH/" | xargs echo | uniq);
+    do
+      echo "Building $i"
+      emerge ${EMERGE_DEFAULT_ARGS:---accept-properties=-interactive --verbose --oneshot --nospinner --noreplace --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg} $i
+    done
+}
+
+build_obsolete() {
+  build_sync
+  for i in $(EIX_LIMIT=0 eix-test-obsolete | grep '\[U\]' | awk '{ print $2 }' | xargs echo | uniq);
+    do
+     echo "Build $i"
+     emerge ${EMERGE_DEFAULT_ARGS:---accept-properties=-interactive -u --verbose --oneshot --nospinner --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg --noreplace} $i
+    done
+}
+
+build_category_installed() {
+  local SEARCH=$1
+  build_sync
+  for i in $(EIX_LIMIT=0 eix -I --only-names --pure-packages "$SEARCH/" | xargs echo | uniq);
+    do
+      echo "Building $i"
+      emerge ${EMERGE_DEFAULT_ARGS:---accept-properties=-interactive -u --newuse --noreplace --changed-use --update --verbose --oneshot --nospinner --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg} $i
+    done
+}
+
+build_all_availables() {
+  for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq);
+    do
+      build_category "$i/"
+    done
+}
+
+rebuild_all() {
+for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq);
+  do
+    build_category_installed "$i/"
+  done
+}
+
+get_category() {
+  local CATEGORY=$1
+  local PACKS
+  for i in $(EIX_LIMIT=0 eix --only-names --pure-packages "$CATEGORY/" | xargs echo | uniq);
+    do
+      PACKS+="$i "
+    done
+    echo $PACKS
+}
+
+get_all_availables() {
+  local RES
+  for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq);
+    do
+        RES+="$(get_category $i) "
+    done
+  echo $RES
+}