Files
devkit/builder
T
Ben Roberts 4dbbc6b34a Strip out package itself from package_deps
Package_deps returned the target package in the output which is not
strictly true (the package is not a dependency of itself really).
In calcuate_missing also means we install the package once using entropy
(if available), and then immediately replace it with portage,
which just wastes effort.

It breaks installation in the cases where the older equo version blocks the
new portage version (e.g. change of slot).

This should remove the target package from the list of packages to be
installed with entropy.
2016-09-13 22:08:43 +01:00

570 lines
18 KiB
Perl
Executable File

#!/usr/bin/env perl
use Getopt::Long;
use v5.10;
no feature "say";
my $profile = $ENV{BUILDER_PROFILE} // 3;
my $jobs = $ENV{BUILDER_JOBS} // 1;
my $use_equo = $ENV{USE_EQUO} // 1;
my $preserved_rebuild = $ENV{PRESERVED_REBUILD} // 0;
my $emerge_defaults_args = $ENV{EMERGE_DEFAULTS_ARGS}
// "--accept-properties=-interactive --verbose --oneshot --complete-graph --buildpkg";
$ENV{FEATURES} = $ENV{FEATURES}
// "parallel-fetch protect-owned compressdebug splitdebug -userpriv";
my $equo_install_atoms = $ENV{EQUO_INSTALL_ATOMS} // 1;
my $equo_install_version = $ENV{EQUO_INSTALL_VERSION} // 0;
my $equo_split_install = $ENV{EQUO_SPLIT_INSTALL} // 1;
my $equo_mirrorsort = $ENV{EQUO_MIRRORSORT} // 1;
my $entropy_repository = $ENV{ENTROPY_REPOSITORY}
// "main"; # Can be weekly, main, testing
my $artifacts_folder = $ENV{ARTIFACTS_DIR};
my $dep_scan_depth = $ENV{DEPENDENCY_SCAN_DEPTH} // 2;
my $skip_portage_sync = $ENV{SKIP_PORTAGE_SYNC} // 0;
my $emerge_split_install = $ENV{EMERGE_SPLIT_INSTALL} // 0;
my $webrsync = $ENV{WEBRSYNC} // 0;
my $enman_repositories = $ENV{ENMAN_REPOSITORIES};
my $emerge_remove = $ENV{EMERGE_REMOVE};
my $remove_enman_repositories = $ENV{REMOVE_ENMAN_REPOSITORIES};
my $remove_remote_overlay = $ENV{REMOVE_REMOTE_OVERLAY};
my $remove_layman_overlay = $ENV{REMOVE_LAYMAN_OVERLAY};
my $prune_virtuals = $ENV{PRUNE_VIRTUALS} // 0;
my $repository_name = $ENV{REPOSITORY_NAME};
my $enman_add_self = $ENV{ENMAN_ADD_SELF} // 0;
my $build_injected_args = $ENV{BUILD_INJECTED_ARGS};
my $equo_masks = $ENV{EQUO_MASKS};
my $equo_unmasks = $ENV{EQUO_UNMASKS};
my $equo_install_args = $ENV{EQUO_INSTALL_ARGS} // "--multifetch=10";
my $remote_overlay = $ENV{REMOTE_OVERLAY};
my $qualityassurance_checks = $ENV{QA_CHECKS} // 0;
my $remote_conf_portdir = $ENV{REMOTE_CONF_PORTDIR};
my $remote_portdir = $ENV{REMOTE_PORTDIR};
my $make_conf = $ENV{MAKE_CONF};
my @overlays;
my $help = 0;
GetOptions(
'layman|overlay:s{,}' => \@overlays,
'equo|install:s{,}' => \@equo_install,
'equorm|remove:s{,}' => \@equo_remove,
'help|?' => \$help
);
help() if $help;
$ENV{LC_ALL} = "en_US.UTF-8"; #here be dragons
$ENV{ETP_NONINTERACTIVE} = 1;
$ENV{ACCEPT_LICENSE} = "*"; # we can use wildcard since entropy 302
# A barely print replacement
sub say { print join( "\n", @_ ) . "\n"; }
sub safe_call {
my $cmd = shift;
my $rt = system($cmd);
my $return = $rt >> 8;
exit($return) if ($rt);
}
sub append_to_file {
my ( $file_name, $package ) = @_;
if ( -f $file_name ) {
# Check that the package was not already there
open my $fh_ro, '<:encoding(UTF-8)', $file_name
or die("Cannot open: $file_name");
while ( my $line = <$fh_ro> ) {
return if $line eq $package . "\n";
}
close $fh_ro;
}
open( my $fh_a, '>>', $file_name )
or die "Could not open file '$filename' $!";
print $fh_a $package . "\n";
close $fh_a;
}
sub add_portage_repository {
my $repo = $_[0];
my $reponame;
my $sync_type;
my @repodef = split( /\|/, $repo );
( $reponame, $repo ) = @repodef if ( @repodef == 2 );
( $reponame, $sync_type, $repo ) = @repodef if ( @repodef == 3 );
# try to detect sync-type
if ( !$sync_type ) {
$sync_type = ( split( /\:/, $repo ) )[0];
$sync_type = "git"
if $repo =~ /github|bitbucket/
or $sync_type eq "https"
or $sync_type eq "http";
$sync_type = "svn" if $repo =~ /\/svn\//;
}
$reponame = ( split( /\//, $repo ) )[-1] if !$reponame;
$reponame =~ s/\.|//g; #clean
system("mkdir -p /etc/portage/repos.conf/")
if ( !-d "/etc/portage/repos.conf/" );
say "==== Adding $reponame ====";
qx{
echo '[$reponame]
location = /usr/local/overlay/$reponame
sync-type = $sync_type
sync-uri = $repo
auto-sync = yes' > /etc/portage/repos.conf/$reponame.conf
}; # Declaring the repo and giving priority
system("emaint sync -r $reponame");
}
# Input: package, depth, and atom. Package: sys-fs/foobarfs, Depth: 1 (depth of the package tree) , Atom: 1/0 (enable disable atom output)
my %package_dep_cache;
sub package_deps {
my $package = shift;
my $depth = shift // 1; # defaults to 1 level of depthness of the tree
my $atom = shift // 0;
# Since we expect this sub to be called multiple times with the same arguments, cache the results
$cache_key = "${package}:${depth}:${atom}";
if ( !exists $package_dep_cache{$cache_key} ) {
my @dependencies =
qx/equery -C -q g --depth=$depth $package/; #depth=0 it's all
chomp @dependencies;
# If an unversioned atom is given, equery returns results for all versions in the portage tree
# leading to duplicates. The sanest thing to do is dedup the list. This gives the superset of all
# possible dependencies, which isn't perfectly accurate but should be good enough. For completely
# accurate results, pass in a versioned atom.
@dependencies = uniq(
sort
grep { $_ }
map { $_ =~ s/\[.*\]|\s//g; &atom($_) if $atom; $_ }
@dependencies
);
$package_dep_cache{$cache_key} = \@dependencies;
}
return @{ $package_dep_cache{$cache_key} };
}
#Input: nothing
#Output: returns all available packages across all the repository installed in the machine
sub available_packages {
my @packages;
my @repos = qx|equo repo list -q|;
chomp(@repos);
push( @packages, qx|equo q list available -q $_| ) for @repos;
chomp(@packages);
return @packages;
}
# Input: package (sys-fs/foobarfs)
# Output: Array of packages to be installed, that aren't installed in a Sabayon machine
sub calculate_missing {
my $package = shift;
my $depth = shift;
my @Installed_Packages = @{ +shift };
my @Available_Packages = @{ +shift };
my $prune_virtuals = shift;
# Getting the package dependencies and the installed packages
my @dependencies = package_deps( $package, $depth, 1 );
if ($prune_virtuals) {
my %install_dependencies = map { $_ => 1 } @dependencies;
# Look for any virtuals and remove its immediate dependencies to avoid
# installing multiple conflicting packages one by one
my @virtual_deps = ();
for my $dep (@dependencies) {
if ( $dep =~ /^virtual\// ) {
my @child_deps = package_deps($dep, 1, 1);
push( @virtual_deps, @child_deps );
}
}
# Deduplicate the list
@virtual_deps = uniq(@virtual_deps);
# Prune the dependencies of the virtual packages from the dependencies list
for my $dep (@virtual_deps) {
if ( $dep !~ /^virtual\// ) {
$install_dependencies{$dep} = 0;
}
}
@dependencies = grep { $install_dependencies{$_} } @dependencies;
}
#taking only the 4th column of output as key of the hashmap
my %installed_packs =
map { $_ => 1 } @Installed_Packages;
my %available_packs = map { $_ => 1 } @Available_Packages;
# removing from packages the one that are already installed and keeping only the available in the entropy repositories
my @to_install = grep( defined $available_packs{$_},
uniq( grep( !defined $installed_packs{$_}, @dependencies ) ) );
@to_install = grep { length } @to_install;
# Strip out the target package from the dependency list
@to_install = grep { to_atom($_) ne to_atom($package) } @to_install;
say "[$package] packages that will be installed with equo: @to_install"
if @to_install > 0;
return @to_install;
}
# Input : complete gentoo package (sys-fs/foobarfs-1.9.2)
# Output: atom form (sys-fs/foobarfs)
sub atom { s/-[0-9]{1,}.*$//; }
# Same again as a function
sub to_atom { my $p=shift; $p =~ s/-[0-9]{1,}.*$//; return $p; }
# Input: Array
# Output: array with unique elements
sub uniq {
keys %{ { map { $_ => 1 } @_ } };
}
sub detect_useflags
{ # Detect useflags defined as [-alsa,avahi] in atom, and fill $hash within the $target sub-hash
my ( $target, $packages ) = @_;
my @packs = @{$packages};
for my $i ( 0 .. $#packs ) {
if ( $packages->[$i] =~ /\[(.*?)\]/ ) {
my $flags = $1;
$packages->[$i] =~ s/\[.*?\]//g;
$per_package_useflags->{$target}->[$i] = [ +split( /,/, $flags ) ];
}
}
}
sub compile_packs {
my ( $target, @packages ) = @_;
my $extra_arg;
$extra_arg = "-B" if $target eq "injected_targets";
my $package_counter = 0;
my $return;
for my $pack (@packages) {
my $tmp_rt;
say "\n" x 2, "==== Compiling $pack ====", "\n" x 2;
if ( defined $per_package_useflags->{$target}->[$package_counter]
and @{ $per_package_useflags->{$target}->[$package_counter] } > 0 )
{
say "USEFLAGS: "
. join( " ",
@{ $per_package_useflags->{$target}->[$package_counter] } );
$tmp_rt = system(
"USE=\""
. join( " ",
@{ $per_package_useflags->{$target}->[$package_counter] } )
. "\" emerge $emerge_defaults_args -j $jobs $extra_arg $pack"
);
}
else {
$tmp_rt =
system("emerge $emerge_defaults_args -j $jobs $extra_arg $pack");
}
$package_counter++;
$return = $tmp_rt
if ( $? == -1 or $? & 127 or !$rt )
; # if one fails, the build should be considered failed!
}
return $return;
}
sub help {
say "-> You should feed me with something", "", "Examples:", "",
"\t$0 app-text/tree", "\t$0 plasma-meta --layman kde", "",
"\t$0 app-foo/foobar --equo foo-misc/foobar --equo net-foo/foobar --layman foo --layman bar foo",
"**************************", "",
"You can supply multiple overlays as well: $0 plasma-meta --layman kde plab",
"The documentation is available at https://github.com/Sabayon/devkit",
"";
exit 0;
}
say "****************************************************";
if ( @overlays > 0 ) {
say "Overlay(s) to add";
foreach my $overlay (@overlays) {
say "\t- $overlay";
}
}
say "[*] Installing:";
say "\t* " . $_ for @ARGV;
say "[*] Syncing configurations files, Layman and Portage";
# Syncronizing portage configuration and adding overlays
system("echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen"); #be sure about that.
# If defined, fetch a remote /etc/portage
if ( $remote_conf_portdir ne "" ) {
system("rm -rf /etc/portage");
system("git clone $remote_conf_portdir /etc/portage");
system("chown -R portage:portage /etc/portage");
system("chmod -R ug+w,a+rX /etc/portage");
}
else {
system("cd /etc/portage/;git checkout master; git stash; git pull");
}
# If defined, fetch a remote /usr/portage
if ( $remote_portdir ne "" ) {
system("rm -rf /usr/portage");
system("git clone $remote_portdir /usr/portage");
system("chown -R portage:portage /usr/portage");
system("chmod -R ug+w,a+rX /usr/portage");
}
system("mkdir /var/lib/layman") if ( ! -d "/var/lib/layman" );
system("touch /var/lib/layman/make.conf && layman-updater -R") if ( ! -e "/var/lib/layman/make.conf" );
system("echo 'y' | layman -f -a $_") for @overlays;
my $reponame = "LocalOverlay";
# Setting up a local overlay if doesn't exists
system(
"rm -rf /usr/local/portage;cp -rf /usr/local/local_portage /usr/local/portage"
) if ( -d "/usr/local/local_portage" );
if ( !-f "/usr/local/portage/profiles/repo_name" ) {
system("mkdir -p /usr/local/portage/{metadata,profiles}");
system("echo 'LocalOverlay' > /usr/local/portage/profiles/repo_name");
system("echo 'masters = gentoo' > /usr/local/portage/metadata/layout.conf");
}
else {
open FILE, "</usr/local/portage/profiles/repo_name";
my @FILE = <FILE>;
close FILE;
chomp(@FILE);
$reponame = $FILE[0];
}
system("chown -R portage:portage /usr/local/portage");
system("chmod -R 755 /usr/local/portage");
qx{
echo '[$reponame]
location = /usr/local/portage
masters = gentoo
priority=9999
auto-sync = no' > /etc/portage/repos.conf/local.conf
}; # Declaring the repo and giving priority
if ( $remote_overlay and $remote_overlay ne "" ) {
add_portage_repository($_) for ( split( / /, $remote_overlay ) );
}
if ( $remove_layman_overlay and $remove_layman_overlay ne "" ) {
say "===== Removing overlays: $remove_remote_overlay =====";
system("layman -d $_") for ( split( / /, $remove_layman_overlay ) );
}
if ( $remove_remote_overlay and $remove_remote_overlay ne "" ) {
say "===== Removing overlays: $remove_remote_overlay =====";
system( "rm -rfv /etc/portage/" . $_ . ".conf" )
for ( split( / /, $remote_overlay ) );
}
system("mkdir -p /usr/portage/distfiles/git3-src");
unless ( $skip_portage_sync == 1 ) {
# sync portage and overlays
system("layman -S");
if ( $webrsync == 1 ) {
system("emerge-webrsync");
}
else {
system("emerge --sync");
}
}
# preparing for MOAR automation
say "Setting new profile to $profile" if defined $profile;
qx|eselect profile set $profile| if defined $profile;
system("eselect profile list") if defined $profile;
if ( $use_equo && $entropy_repository eq "weekly" ) {
qx|equo repo disable sabayonlinux.org|;
qx|equo repo enable sabayon-weekly|;
}
elsif ( $use_equo && $entropy_repository eq "testing" ) {
qx|equo repo disable sabayon-weekly|;
qx|equo repo enable sabayonlinux.org|;
qx|equo repo enable sabayon-limbo|;
}
if ($use_equo) {
say "Devkit version:";
system("equo s -vq app-misc/sabayon-devkit");
if ( $enman_repositories and $enman_repositories ne "" ) {
my @enman_toadd = split( / /, $enman_repositories );
safe_call("enman add $_") for @enman_toadd;
}
if ( $remove_enman_repositories and $remove_enman_repositories ne "" ) {
my @enman_toremove = split( / /, $remove_enman_repositories );
system("enman remove $_") for @enman_toremove;
}
system("enman add $repository_name")
if ( $enman_add_self and $repository_name and $repository_name ne "" );
system("equo repo mirrorsort sabayonlinux.org") if $equo_mirrorsort;
system("equo up && equo u");
}
system("cp -rf $make_conf /etc/portage/make.conf") if $make_conf;
my $per_package_useflags;
my @packages = @ARGV;
my @injected_packages = ();
if ($build_injected_args) {
@injected_packages = split( / /, $build_injected_args );
say "[*] Injected installs:";
say "\t* " . $_ for @injected_packages;
}
# Allow users to specify atoms as: media-tv/kodi[-alsa,avahi]
if ($emerge_split_install)
{ # For targets this will be available only if split_install is enabled
detect_useflags( "targets", \@packages );
}
else {
map { $_ =~ s/\[.*?\]//g; $_; }
@packages
; # Clean up [] if user didn't specified split_install, but specified a useflag combination
}
detect_useflags( "injected_targets", \@injected_packages );
if ($use_equo) {
# best effort mask
if ($equo_masks) {
append_to_file( "/etc/entropy/packages/package.mask", $_ )
for ( split( / /, $equo_masks ) );
}
# best effort unmask
if ($equo_unmasks) {
append_to_file( "/etc/entropy/packages/package.unmask", $_ )
for ( split( / /, $equo_unmasks ) );
}
my @packages_deps;
my @Installed_Packages = qx/equo q list installed --quiet/;
my @Available_Packages = available_packages();
chomp(@Installed_Packages);
my %installed_packs =
map { $_ => 1 } @Installed_Packages;
# Remove any already installed packages from the list of entropy packages to install in the build spec
@equo_install = grep { !exists $installed_packs{$_} } @equo_install;
foreach my $p (@packages) {
say
"[$p] Getting the package dependencies which aren't already installed on the system.. ";
push(
@packages_deps,
calculate_missing(
$p, $dep_scan_depth,
\@Installed_Packages, \@Available_Packages,
$prune_virtuals
)
) if $equo_install_atoms;
push( @packages_deps, package_deps( $p, $dep_scan_depth, 0 ) )
if $equo_install_version;
}
@packages_deps = grep { defined() and length() } @packages_deps; #cleaning
say "", "[install] Those dependencies will be installed with equo :",
@packages_deps, "";
if ($equo_split_install) {
safe_call("equo i $equo_install_args --bdeps $_")
for ( @packages_deps, @equo_install )
; ## bail out here, if installs fails. emerge will compile a LOT of stuff
if ( @equo_remove > 0 ) {
system("equo rm --nodeps $_") for (@equo_remove);
}
}
else {
safe_call(
"equo i $equo_install_args --bdeps @packages_deps @equo_install")
if ( @packages_deps > 0 or @equo_install > 0 )
; ## bail out here, if installs fails. emerge will compile a LOT of stuff
system("equo rm --nodeps @equo_remove") if ( @equo_remove > 0 );
}
}
if ( $qualityassurance_checks == 1 ) {
say "*** Repoman checks ***";
for ( @packages, @injected_packages ) {
say "*** QA checks for $_";
system(
"pushd \$(dirname \$(equery which $_ 2>/dev/null)); repoman; popd");
}
}
say "*** Ready to compile, finger crossed ***";
system("emerge --info")
; #always give detailed information about the building environment, helpful to debug
my $rt;
if ( $emerge_remove and $emerge_remove ne "" ) {
system("emerge -C $_") for split( / /, $emerge_remove );
}
if ($emerge_split_install) {
compile_packs( "targets", @packages );
$rt = 0; #consider the build good anyway, like a "keep-going"
}
else {
$rt = system("emerge $emerge_defaults_args -j $jobs @packages");
}
my $return = $rt >> 8;
# best effort -B
compile_packs( "injected_targets", @injected_packages );
if ($preserved_rebuild) {
system("emerge -j $jobs --buildpkg \@preserved-rebuild");
system("revdep-rebuild");
}
if ( $qualityassurance_checks == 1 ) {
say "*** Missing dependencies checks ***";
for ( @packages ) {
say "*** DEPEND for $_";
system("dynlink-scanner $_");
system("depcheck $_");
}
}
# Copy files to artifacts folder
system(
"mkdir -p $artifacts_folder;cp -rfv /usr/portage/packages $artifacts_folder"
) if ( $artifacts_folder and !$return );
exit($return);