Makefile: cleanup

* if emerge_split_install is enabled, then build.targets can contain useflags. (e.g. media-tv/kodi[-avahi] )
* for injected packages, this feature is always on

Makefile

@@ -25,4 +25,8 @@ install:
 	install -m 0755 builder $(DESTDIR)/$(USBINDIR)/
 
 	install -d $(DESTDIR)/$(UBINDIR)
+	install -m 0755 depcheck $(DESTDIR)/$(UBINDIR)/
+	install -m 0755 dynlink-scanner $(DESTDIR)/$(UBINDIR)/
+	gcc try_dlopen.c -o try_dlopen -ldl
+	install -m 0755 try_dlopen $(DESTDIR)/$(UBINDIR)/
 	install -m 0755 sabayon-* $(DESTDIR)/$(UBINDIR)/

builder

@@ -15,7 +15,7 @@ $ENV{FEATURES} = $ENV{FEATURES}
 
 my $equo_install_atoms   = $ENV{EQUO_INSTALL_ATOMS}   // 1;
 my $equo_install_version = $ENV{EQUO_INSTALL_VERSION} // 0;
-my $equo_split_install   = $ENV{EQUO_SPLIT_INSTALL}   // 0;
+my $equo_split_install   = $ENV{EQUO_SPLIT_INSTALL}   // 1;
 my $equo_mirrorsort      = $ENV{EQUO_MIRRORSORT}      // 1;
 my $entropy_repository   = $ENV{ENTROPY_REPOSITORY}
   // "main";    # Can be weekly, main, testing
@@ -35,8 +35,11 @@ my $enman_add_self            = $ENV{ENMAN_ADD_SELF} // 0;
 my $build_injected_args       = $ENV{BUILD_INJECTED_ARGS};
 my $equo_masks                = $ENV{EQUO_MASKS};
 my $equo_unmasks              = $ENV{EQUO_UNMASKS};
+my $equo_install_args         = $ENV{EQUO_INSTALL_ARGS} // "--multifetch=10";
 my $remote_overlay            = $ENV{REMOTE_OVERLAY};
 my $repoman_check             = $ENV{QA_CHECKS} // 0;
+my $remote_conf_portdir       = $ENV{REMOTE_CONF_PORTDIR};
+my $remote_portdir            = $ENV{REMOTE_PORTDIR};
 
 my $make_conf = $ENV{MAKE_CONF};
 
@@ -216,6 +219,54 @@ sub uniq {
     keys %{ { map { $_ => 1 } @_ } };
 }
 
+sub detect_useflags
+{ # Detect useflags defined as [-alsa,avahi] in atom, and fill $hash within the $target sub-hash
+    my ( $target, $packages ) = @_;
+    my @packs = @{$packages};
+    for my $i ( 0 .. $#packs ) {
+        if ( $packages->[$i] =~ /\[(.*?)\]/ ) {
+            my $flags = $1;
+            $packages->[$i] =~ s/\[.*?\]//g;
+            $per_package_useflags->{$target}->[$i] = [ +split( /,/, $flags ) ];
+        }
+    }
+}
+
+sub compile_packs {
+    my ( $target, @packages ) = @_;
+    my $extra_arg;
+    $extra_arg = "-B" if $target eq "injected_targets";
+    my $package_counter = 0;
+    my $return;
+    for my $pack (@packages) {
+        my $tmp_rt;
+        say "\n" x 2, "==== Compiling $pack ====", "\n" x 2;
+        if ( defined $per_package_useflags->{$target}->[$package_counter]
+            and @{ $per_package_useflags->{$target}->[$package_counter] } > 0 )
+        {
+            say "USEFLAGS: "
+              . join( " ",
+                @{ $per_package_useflags->{$target}->[$package_counter] } );
+            $tmp_rt = system(
+                "USE=\""
+                  . join( " ",
+                    @{ $per_package_useflags->{$target}->[$package_counter] } )
+                  . "\" emerge $emerge_defaults_args -j $jobs $extra_arg $pack"
+            );
+        }
+        else {
+            $tmp_rt =
+              system("emerge $emerge_defaults_args -j $jobs $extra_arg $pack");
+        }
+        $package_counter++;
+
+        $return = $tmp_rt
+          if ( $? == -1 or $? & 127 or !$rt )
+          ;    # if one fails, the build should be considered failed!
+    }
+    return $return;
+}
+
 sub help {
     say "-> You should feed me with something", "", "Examples:", "",
       "\t$0 app-text/tree", "\t$0 plasma-meta --layman kde", "",
@@ -244,7 +295,25 @@ say "[*] Syncing configurations files, Layman and Portage";
 
 # Syncronizing portage configuration and adding overlays
 system("echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen");    #be sure about that.
-system("cd /etc/portage/;git checkout master; git stash; git pull");
+
+# If defined, fetch a remote /etc/portage
+if ( $remote_conf_portdir ne "" ) {
+    system("rm -rf /etc/portage");
+    system("git clone $remote_conf_portdir /etc/portage");
+    system("chown -R portage:portage /etc/portage");
+    system("chmod -R ug+w,a+rX /etc/portage");
+}
+else {
+    system("cd /etc/portage/;git checkout master; git stash; git pull");
+}
+
+# If defined, fetch a remote /usr/portage
+if ( $remote_portdir ne "" ) {
+    system("rm -rf /usr/portage");
+    system("git clone $remote_portdir /usr/portage");
+    system("chown -R portage:portage /usr/portage");
+    system("chmod -R ug+w,a+rX /usr/portage");
+}
 
 system("echo 'y' | layman -f -a $_") for @overlays;
 
@@ -343,13 +412,29 @@ if ($use_equo) {
 
 system("cp -rf $make_conf /etc/portage/make.conf") if $make_conf;
 
-
+my $per_package_useflags;
 my @packages          = @ARGV;
 my @injected_packages = ();
 if ($build_injected_args) {
     @injected_packages = split( / /, $build_injected_args );
+    say "[*] Injected installs:";
+
+    say "\t* " . $_ for @injected_packages;
+
 }
 
+# Allow users to specify atoms as: media-tv/kodi[-alsa,avahi]
+if ($emerge_split_install)
+{    # For targets this will be available only if split_install is enabled
+    detect_useflags( "targets", \@packages );
+}
+else {
+    map { $_ =~ s/\[.*?\]//g; $_; }
+      @packages
+      ; # Clean up [] if user didn't specified split_install, but specified a useflag combination
+}
+detect_useflags( "injected_targets", \@injected_packages );
+
 if ($use_equo) {
 
     # best effort mask
@@ -393,7 +478,7 @@ if ($use_equo) {
     say "", "[install] Those dependencies will be installed with equo :",
       @packages_deps, "";
     if ($equo_split_install) {
-        safe_call("equo i --bdeps $_")
+        safe_call("equo i $equo_install_args --bdeps $_")
           for ( @packages_deps, @equo_install )
           ; ## bail out here, if installs fails. emerge will compile a LOT of stuff
         if ( @equo_remove > 0 ) {
@@ -401,7 +486,8 @@ if ($use_equo) {
         }
     }
     else {
-        safe_call("equo i --bdeps @packages_deps @equo_install")
+        safe_call(
+            "equo i $equo_install_args --bdeps @packages_deps @equo_install")
           if ( @packages_deps > 0 or @equo_install > 0 )
           ; ## bail out here, if installs fails. emerge will compile a LOT of stuff
         system("equo rm --nodeps @equo_remove") if ( @equo_remove > 0 );
@@ -409,8 +495,12 @@ if ($use_equo) {
 }
 
 if ( $repoman_check == 1 ) {
-     say "*** Repoman checks ***";
-     say "*** QA checks for $_" && system("pushd \$(dirname \$(equery which $_ 2>/dev/null)); repoman; popd") for ( @packages, @injected_packages );
+    say "*** Repoman checks ***";
+    for ( @packages, @injected_packages ) {
+        say "*** QA checks for $_";
+        system(
+            "pushd \$(dirname \$(equery which $_ 2>/dev/null)); repoman; popd");
+    }
 }
 
 say "*** Ready to compile, finger crossed ***";
@@ -425,12 +515,7 @@ if ( $emerge_remove and $emerge_remove ne "" ) {
 }
 
 if ($emerge_split_install) {
-    for my $pack (@packages) {
-        say "\n" x 2, "==== Compiling $pack ====", "\n" x 2;
-        my $tmp_rt = system("emerge $emerge_defaults_args -j $jobs $pack");
-
-#  $rt=$tmp_rt if ($? == -1 or $? & 127 or !$rt); # if one fails, the build should be considered failed!
-    }
+    compile_packs( "targets", @packages );
     $rt = 0;    #consider the build good anyway, like a "keep-going"
 }
 else {
@@ -440,7 +525,7 @@ else {
 my $return = $rt >> 8;
 
 # best effort -B
-system("emerge $emerge_defaults_args -j $jobs -B $_") for (@injected_packages);
+compile_packs( "injected_targets", @injected_packages );
 
 if ($preserved_rebuild) {
 

depcheck

@@ -0,0 +1,303 @@
+#!/bin/bash
+# PODNAME: depcheck
+#
+# Copyright (c) 2014 Michael Palimaka <kensington@gentoo.org>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+#
+# A tool to report both undeclared and potentially-unused runtime dependencies.
+#
+# Depends on app-misc/pax-utils, app-portage/portage-utils and
+# sys-apps/gentoo-functions.
+
+DEBUG=FALSE
+IGNORE_DEPS=( "sys-libs/glibc" "sys-devel/gcc" )
+IGNORE_LINKS=( "/lib64/libgcc_s.so.1" )
+PKG_DIR="/var/db/pkg/"
+
+. /lib/gentoo/functions.sh
+
+bold() {
+    echo $@
+    return
+    local bold=$(tput bold)
+    local normal=$(tput sgr0)
+    echo "${bold}${@}${normal}"
+}
+
+debug() {
+    if [ $DEBUG = TRUE ]; then
+        # write to stderr so as not to interfere with function returns
+        echo "$@" 1>&2
+    fi
+}
+
+# app-foo/bar-1.2.3-r1 -> app-foo/bar
+remove_atom_version() {
+    local atom=`qatom "${1}" | cut -d " " -f 1-2 | tr " " /`
+    echo $atom
+}
+
+virtualcheck() {
+    debug Checking if ${libowner_pn} is provided by a virtual
+    for virtual in $(qdepends --nocolor --name-only --${1} --query ${libowner_pn} | grep ^virtual/)
+    do
+        debug Checking if ${virtual} is in dependencies
+        local isvirtualdep=$(qdepends --${1} ${atom} | grep ${virtual})
+
+        if [ $? -eq 0 ]; then
+            used_virtuals+=( ${virtual} )
+            continue 2
+        fi
+    done
+
+    if [ "${1}" = "depend" ]; then
+        eerror "${obj} links to ${link}"
+    fi
+    eindent
+    eerror Missing ${1^^} on $(bold ${libowner_pn})
+    eoutdent
+    errors=1
+}
+
+check_atom() {
+
+    local errors=0
+    local atom=$1
+    local checked=()
+    local rdepends=()
+    local used_virtuals=()
+
+    local objects=`qlist -qo ${atom}`
+
+    if [ ! "${objects}" ]; then
+        einfo ${atom} does not have any objects installed, skipping...
+        echo
+        return
+    fi
+
+    if [ -z "${INSIDE_PORTAGE}" ]; then
+        einfo Checking ${atom} for undeclared dependencies
+        eindent
+    fi
+
+    local obj
+    for obj in $objects
+    do
+        debug "Checking ${obj}"
+
+        readelf -h ${obj} > /dev/null 2>&1
+
+        # can it have stuff linked to it?
+        if [ $? -ne 0 ]; then
+            debug "It can't have links, skipping"
+            continue
+        fi
+
+        local elf=`scanelf --format "#f%n" --nobanner --use-ldpath ${obj} 2>&1`
+        local links=`echo ${elf} | tr "," " "`
+
+        # get a list of everything that it's linked to
+        local link
+        for link in $links
+        do
+            local ignore
+            for ignore in "${IGNORE_LINKS[@]}"; do
+                if [ "${ignore}" = "${link}" ]; then
+                    debug "Ignoring ${link} due to blacklist"
+                    continue 2
+                fi
+            done
+
+            # only check a library once per atom for performance reasons
+            local check
+            for check in "${checked[@]}"; do
+                if [ "${check}" = "${link}" ]; then
+                    debug "Already checked ${link} for this atom, skipping"
+                    continue 2
+                fi
+            done
+
+            checked=( "${checked[@]}" "${link}" )
+
+            debug "Found ${link}"
+
+            local libowner=`qfile -vqC ${link} | uniq`
+
+            if [ ! "${libowner}" ]; then
+                ewarn "Warning: installed file ${obj} is linked to ${link} which is not owned by any installed atom."
+                continue
+            fi
+
+            debug "Owning package for ${link} is ${libowner}"
+
+            local libowner_pn=$(remove_atom_version ${libowner})
+            local my_pn=$(remove_atom_version ${atom})
+
+            rdepends+=( "${libowner_pn}" )
+
+            if [ "${libowner_pn}" = "${my_pn}" ]; then
+                debug "Owning package is self, ignoring"
+                continue
+            fi
+
+            local ignorelib
+            for ignorelib in "${IGNORE_DEPS[@]}"
+            do
+                if [ "${libowner_pn}" = "${ignorelib}" ]; then
+                    debug "Ignoring objects belonging to ${ignorelib}"
+                    continue 2
+                fi
+            done
+
+            debug "Checking if ${libowner_pn} is in the [R]DEPEND list of ${atom}"
+
+            local isdep
+            isdep=`qdepends -d ${atom} | grep ${libowner_pn}`
+            if [ $? -ne 0 ]; then
+                virtualcheck depend
+            fi
+
+            local isrdep
+            isrdep=`qdepends -r ${atom} | grep ${libowner_pn}`
+            if [ $? -ne 0 ]; then
+                virtualcheck rdepend
+            fi
+
+
+        done
+
+    done
+
+    local ebuild_rdepends=()
+    for rdepend in $(qdepends --nocolor --quiet --rdepend ${atom} | sed -e "s/\[[^]]*\]//g" | cut -d : -f 2-)
+    do
+        if [[ ${rdepend} = !* ]] ; then
+            debug Skipping blocker: ${rdepend}
+            continue
+        elif [[ ${rdepend} = virtual/* ]] ; then
+            for virtual in "${used_virtuals[@]}"
+            do
+                if [[ ${virtual} == $(remove_atom_version ${rdepend}) ]]; then
+                    debug Skipping virtual: ${rdepend}
+                    continue 2
+                fi
+            done
+        fi
+        ebuild_rdepends+=( $(remove_atom_version $rdepend) )
+    done
+
+    debug "Ebuild RDEPENDS: ${ebuild_rdepends[@]}"
+    debug "Linked RDEPENDS: ${rdepends[@]}"
+
+    local suspect_rdepends=$(comm -13 <(echo ${rdepends[@]} | sed 's/ /\n/g' | sort -u) <(echo ${ebuild_rdepends[@]} | sed 's/ /\n/g' | sort -u))
+    if [ "${suspect_rdepends}" ]; then
+        ewarn "Suspect RDEPEND: $(bold ${suspect_rdepends})"
+    fi
+
+    if [ -z "${INSIDE_PORTAGE}" ]; then
+        eoutdent
+        if [ -n "${CAT}" ]; then
+            echo
+        fi
+    fi
+
+    return $errors
+
+}
+
+check_package() {
+
+    local package=$1
+    local atoms=`qlist -IcCS ${package} | tr ' ' '-' | cut -d : -f1`
+
+    debug Package ${package} own atoms: ${atoms}
+
+    if [ ! "${atoms}" ]; then
+        eerror ERROR: ${package} is not a valid atom
+        exit 1
+    fi
+
+    for atom in ${atoms}; do
+        check_atom ${atom}
+    done
+
+}
+
+check_category() {
+
+    local errors=0
+
+    for package in `ls "${PKG_DIR}/${1}"`; do
+
+        check_package "${1}/${package}"
+
+        if [ $? -ne 0 ]; then
+            errors=1
+        fi
+
+    done;
+
+    return $errors
+
+}
+
+BINNAME=`basename ${0}`
+if [ -z $1 ]; then
+    echo "Checks emerged package(s) for undeclared depedencies"
+    echo
+        echo "Usage: ${BINNAME} [ atom | -c category | -a ]"
+    echo
+        echo "Examples:"
+    echo "Check a single package: ${BINNAME} app-foo/bar-1.2.3"
+    echo "Check a category: ${BINNAME} -c app-foo"
+    echo "Check everything: ${BINNAME} -a"
+    exit 1
+fi
+
+# check a particular category
+if [ "$1" = "-c" ]; then
+
+    CAT=$2
+
+    if [ ! -d "${PKG_DIR}/${CAT}" ]; then
+        eerror ERROR: No packages from the category ${CAT} have been emerged yet
+        exit 1
+    fi
+
+    check_category $CAT
+
+# check everything
+elif [ "$1" = "-a" ]; then
+
+    for category in `ls ${PKG_DIR}`; do
+        check_category $category
+    done;
+
+else
+    check_package $1
+fi
+
+if [ $DEBUG = TRUE ]; then
+    if [ $? -eq 0 ]; then
+        einfo Checking complete, no errors found
+    else
+        eerror Checking complete, errors found
+    fi
+fi

dynlink-scanner

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Maciej Mrozowski <reavertm@gentoo.org>
+ 
+# Returns colon separated list of linking dependencies
+get_link_deps()
+{
+	KEY=
+	for dep in `scanelf --use-ldpath -yBF '%f %n' "$1"`; do
+		if [[ -z $KEY ]]; then
+			KEY="$dep"
+			continue
+		fi
+		echo $dep
+	done
+}
+
+# Print linking deps for given executable or shared object in alphabetical order.
+# Also try to dlopen shared objects in order to detect missing/broken dependencies.
+if [[ "$1" = --linking-deps ]]; then
+	# Sanity check, file-5.12 is broken, bail out early
+	if [[ `file --version | grep --color=never file- | cut -d'-' -f2` == '5.12' ]]; then
+		echo "file-5.12 is broken, bailing out"
+		exit 1
+	fi
+	mime=`file -b --mime-type "$2"`
+	if [[ "$mime" == 'application/x-executable' ]] || [[ "${mime}" == 'application/x-sharedlib' ]]; then
+		LINK=`get_link_deps "$2"`
+		[[ "$mime" == 'application/x-sharedlib' ]] && /usr/bin/try_dlopen "$2"
+		[[ -n $LINK ]] && echo -e ${LINK//,/\\n} | sort -u
+		exit 0
+	fi
+	exit 1
+fi
+
+[[ -z $* ]] && echo "usage: `basename $0` <package>" && exit 0
+
+# Check whether package is installed
+if ! portageq has_version $ROOT/ $1; then
+	echo "$1 is not installed"
+	exit 1
+fi
+
+# For each installed slot
+for cpv in `portageq match $ROOT/ $1`; do
+	echo "Processing $cpv"
+	# For each file that belongs to package
+	#     run dynlink-scanner --linking-deps <file> to obtain its linking dependencies
+	# Assign all linking deps to packages and print package names
+	qfile -eR $ROOT `portageq contents $ROOT/ $cpv | xargs -r -L 1 "$0" --linking-deps` | cut -f1 -d' ' | sort -u
+done

sabayon-buildpackages

@@ -53,7 +53,9 @@ docker_volumes=( -v "$OUTPUT_DIR:/usr/portage/packages" )
 [ -z "$REMOVE_LAYMAN_OVERLAY" ] || docker_env+=(-e "REMOVE_LAYMAN_OVERLAY=$REMOVE_LAYMAN_OVERLAY")
 [ -z "$REMOVE_REMOTE_OVERLAY" ] || docker_env+=(-e "REMOVE_REMOTE_OVERLAY=$REMOVE_REMOTE_OVERLAY")
 [ -z "$QA_CHECKS" ] || docker_env+=(-e "QA_CHECKS=$QA_CHECKS")
-
+[ -z "$REMOTE_CONF_PORTDIR" ] || docker_env+=(-e "REMOTE_CONF_PORTDIR=$REMOTE_CONF_PORTDIR")
+[ -z "$REMOTE_PORTDIR" ] || docker_env+=(-e "REMOTE_PORTDIR=$REMOTE_PORTDIR")
+[ -z "$ETP_NOCACHE" ] || docker_env+=(-e "ETP_NOCACHE=$ETP_NOCACHE")
 [ -d "$LOCAL_OVERLAY" ] && docker_volumes+=(-v "$LOCAL_OVERLAY:/usr/local/local_portage")
 
 if [ -e $MAKE_CONF ]; then

try_dlopen.c

@@ -0,0 +1,23 @@
+#include <stdio.h>
+#include <dlfcn.h>
+
+/**
+ * Attempts to load shared libraries
+ * @param argc
+ * @param argv files to check (pass only shared libs)
+ * @return 0 - all shared libs loaded, 1 - error occured (invalid or broken files passed)
+ */
+int main(int argc, char* argv[])
+{
+	int i;
+	for (i = 1; i < argc; ++i) {
+		void* handle = dlopen(argv[i], RTLD_NOW);
+		if (!handle) {
+			fprintf(stderr, "%s\n", dlerror());
+			return 1;
+		} else {
+			dlclose(handle);
+		}
+	}
+	return 0;
+}