From aa45473eda86e45aec6cdea2665da041862ae551 Mon Sep 17 00:00:00 2001 From: mudler Date: Sat, 6 Feb 2016 14:58:50 +0100 Subject: [PATCH] first commit --- Makefile | 29 ++++++ builder | 207 +++++++++++++++++++++++++++++++++++++ sabayon-buildpackages | 15 +++ sabayon-createrepo | 77 ++++++++++++++ sabayondevkit-functions.sh | 49 +++++++++ 5 files changed, 377 insertions(+) create mode 100644 Makefile create mode 100755 builder create mode 100755 sabayon-buildpackages create mode 100755 sabayon-createrepo create mode 100644 sabayondevkit-functions.sh diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a4197e9 --- /dev/null +++ b/Makefile @@ -0,0 +1,29 @@ +SUBDIRS = +DESTDIR = +UBINDIR ?= /usr/bin +LIBDIR ?= /usr/lib +SBINDIR ?= /sbin +USBINDIR ?= /usr/sbin +BINDIR ?= /bin +LIBEXECDIR ?= /usr/libexec +SYSCONFDIR ?= /etc + +all: + for d in $(SUBDIRS); do $(MAKE) -C $$d; done + +clean: + for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done + +install: + for d in $(SUBDIRS); do $(MAKE) -C $$d install; done + + install -d $(DESTDIR)/$(SBINDIR) + install -d $(DESTDIR)/$(BINDIR) + install -m 0755 *-functions.sh $(DESTDIR)/$(SBINDIR)/ + + install -d $(DESTDIR)/$(USBINDIR) + install -m 0755 builder $(DESTDIR)/$(USBINDIR)/ + + install -d $(DESTDIR)/$(UBINDIR) + install -m 0755 sabayon-buildpackages $(DESTDIR)/$(UBINDIR)/ + install -m 0755 sabayon-createrepo $(DESTDIR)/$(UBINDIR)/ diff --git a/builder b/builder new file mode 100755 index 0000000..ce31e64 --- /dev/null +++ b/builder @@ -0,0 +1,207 @@ +#!/usr/bin/env perl + +use Getopt::Long; + +my $profile = $ENV{BUILDER_PROFILE} // 3; +my $jobs = $ENV{BUILDER_JOBS} // 1; +my $use_equo = $ENV{USE_EQUO} // 1; +my $preserved_rebuild = $ENV{PRESERVED_REBUILD} // 0; +my $emerge_defaults_args = $ENV{EMERGE_DEFAULTS_ARGS} + // "--accept-properties=-interactive --verbose --oneshot --nospinner --quiet-build=y --complete-graph --buildpkg"; +$ENV{FEATURES} = $ENV{FEATURES} + // "parallel-fetch protect-owned compressdebug splitdebug -userpriv"; + +my $equo_install_atoms = $ENV{EQUO_INSTALL_ATOMS} // 1; +my $equo_install_version = $ENV{EQUO_INSTALL_VERSION} // 0; +my $equo_split_install = $ENV{EQUO_SPLIT_INSTALL} // 0; +my $artifacts_folder = $ENV{ARTIFACTS_DIR}; + +my @overlays; + +GetOptions( 'layman|overlay:s{,}' => \@overlays, 'equo|install:s{,}' => \@equo_install ); + +if ( @ARGV == 0 ) { + help(); + die(); +} + +$ENV{LC_ALL} = "en_US.UTF-8"; #here be dragons + +# A barely print replacement +sub say { print join( "\n", @_ ) . "\n"; } + +# Input: package, depth, and atom. Package: sys-fs/foobarfs, Depth: 1 (depth of the package tree) , Atom: 1/0 (enable disable atom output) +sub package_deps { + my $package = shift; + my $depth = shift // 1; # defaults to 1 level of depthness of the tree + my $atom = shift // 0; + return + map { $_ =~ s/\[.*\]|\s//g; &atom($_) if $atom; $_ } + qx/equery -C -q g --depth=$depth $package/; #depth=0 it's all +} + +#Input: nothing +#Output: returns all available packages across all the repository installed in the machine +sub available_packages { + my @packages; + my @repos = qx|equo repo list -q|; + chomp(@repos); + push( @packages, qx|equo q list available -q $_| ) for @repos; + chomp(@packages); + return @packages; +} + +# Input: package (sys-fs/foobarfs) +# Output: Array of packages to be installed, that aren't installed in a Sabayon machine +sub calculate_missing { + my $package = shift; + my $depth = shift; + + # Getting the package dependencies and the installed packages + say "[$package] Getting the package dependencies and the installed packages"; + my @Packages = package_deps( $package, $depth, 1 ); + my @Installed_Packages = qx/equo q --quiet list installed/; + chomp(@Installed_Packages); + say "[$package] Getting the package dependencies and the installed packages"; + + #taking only the 4th column of output as key of the hashmap + my %installed_packs = + map { ( split( /\s/, $_ ) )[3] => 1 } @Installed_Packages; + my %available_packs = map { $_ => 1 } available_packages(); + +# removing from packages the one that are already installed and keeping only the available in the entropy repositories + my @to_install = grep( defined $available_packs{$_}, + uniq( grep( !defined $installed_packs{$_}, @Packages ) ) ); + @to_install=grep { length } @to_install; + say "[$package] packages that will be installed with equo: @to_install" if @to_install>0; + + return @to_install; +} + +# Input : complete gentoo package (sys-fs/foobarfs-1.9.2) +# Output: atom form (sys-fs/foobarfs) +sub atom { s/-[0-9]{1,}.*$//; } + +# Input: Array +# Output: array with unique elements +sub uniq { + keys %{ { map { $_ => 1 } @_ } }; +} + +sub help { + say "-> You should feed me with something", "", "Examples:", "", + "\t$0 app-text/tree", "\t$0 plasma-meta --layman kde", "", + "\t$0 app-foo/foobar --equo foo-misc/foobar --equo net-foo/foobar --layman foo --layman bar foo", + "**************************", "", +"You can supply multiple overlays as well: $0 plasma-meta --layman kde plab", + ""; +} + +say +"************* IF YOU WANT TO SUPPLY ADDITIONAL ARGS TO EMERGE, pass to docker EMERGE_DEFAULT_OPTS env with your options *************"; + +if ( @overlays > 0 ) { + say "Overlay(s) to add"; + foreach my $overlay (@overlays) { + say "\t- $overlay"; + } +} + +say "Installing:"; + +say "\t* " . $_ for @ARGV; + +say "* Syncing stuff for you, if it's the first time, can take a while"; + +# Syncronizing portage configuration and adding overlays +system("echo 'en_US.UTF-8 UTF-8' > /etc/locale.gen"); #be sure about that. +system( +"cd /etc/portage/;git checkout master; git stash; git pull" +); + +if (`uname -m` eq "x86_64\n"){ + system( + "cd /etc/portage/;rm -rfv make.conf;ln -s make.conf.amd64 make.conf" + ); +} + +system("echo 'y' | layman -f -a $_") for @overlays; + +my $reponame = "LocalOverlay"; + +# Setting up a local overlay if doesn't exists +if ( !-f "/usr/local/portage/profiles/repo_name" ) { + system("mkdir -p /usr/local/portage/{metadata,profiles}"); + system("echo 'LocalOverlay' > /usr/local/portage/profiles/repo_name"); + system("echo 'masters = gentoo' > /usr/local/portage/metadata/layout.conf"); + system("chown -R portage:portage /usr/local/portage"); +} +else { + open FILE, "; + close FILE; + chomp(@FILE); + $reponame = $FILE[0]; +} + +qx{ +echo '[$reponame] +location = /usr/local/portage +masters = gentoo +priority=9999 +auto-sync = no' > /etc/portage/repos.conf/local.conf +}; # Declaring the repo and giving priority + +system("mkdir -p /usr/portage/distfiles/git3-src"); + +# sync portage and overlays +system("layman -S;emerge --sync"); + +# preparing for MOAR automation +qx|eselect profile set $profile|; +qx{ls /usr/portage/licenses -1 | xargs -0 > /etc/entropy/packages/license.accept} + ; #HAHA +system("equo repo mirrorsort sabayonlinux.org;equo up && equo u") + if $use_equo; # Better don't be behind +qx|echo 'ACCEPT_LICENSE="*"' >> /etc/portage/make.conf|; #just plain evil + +my @packages = @ARGV; + +if ($use_equo) { + my @packages_deps; + foreach my $p (@packages) { + say "[$p] Getting the package dependencies which aren't already installed on the system.. ''"; + push( @packages_deps, calculate_missing( $p , 2) ) + if $equo_install_atoms; + push( @packages_deps, package_deps( $p, 2, 0 ) ) + if $equo_install_version; + say "[$p] Done ''" + } + @packages_deps = grep { defined() and length() } @packages_deps; #cleaning + say "", "[install] Installing missing dependencies with equo", @packages_deps, ""; + if ($equo_split_install) { + system("equo i --bdeps $_") for (@packages_deps,@equo_install); + } + else { + system("equo i --bdeps @packages_deps @equo_install"); + } +} + +say "* Ready to compile, finger crossed"; + +my $rt = system("emerge $emerge_defaults_args -j $jobs @packages"); + +my $return = $rt >> 8; + +if ($preserved_rebuild) { + + system("emerge -j $jobs --buildpkg \@preserved-rebuild"); + +} + +# Copy files to artifacts folder +system( + "mkdir -p $artifacts_folder;cp -rfv /usr/portage/packages $artifacts_folder" +) if ( $artifacts_folder and !$return ); + +exit($return); diff --git a/sabayon-buildpackages b/sabayon-buildpackages new file mode 100755 index 0000000..c097628 --- /dev/null +++ b/sabayon-buildpackages @@ -0,0 +1,15 @@ +#!/bin/bash +set -e +ROOT="${ROOT:-$PWD}" + +if [ $# -eq 0 ] + then + echo "No arguments supplied, at least you should provide some package you wish to compile. e.g. $0 app-text/tree" + exit 1 +fi + +if [ "$(id -u)" != "0" ]; then + echo "!! If you are not running the script as root, your user should be in the docker group to use it. (sudo gpasswd -a $USER docker) !!" +fi + +docker run --rm -v "$ROOT"/portage_artifacts:/usr/portage/packages -ti sabayon/builder-amd64 $@ diff --git a/sabayon-createrepo b/sabayon-createrepo new file mode 100755 index 0000000..5584b5f --- /dev/null +++ b/sabayon-createrepo @@ -0,0 +1,77 @@ +#!/bin/bash +set -e +ROOT="${ROOT:-$PWD}" +entropysrv=$(mktemp) +createrepo=$(mktemp) +REPOSITORY_NAME="${REPOSITORY_NAME:-default}" +REPOSITORY_DESCRIPTION="${REPOSITORY_DESCRIPTION:-My Sabayon repository}" +EDITOR=cat + +if [ "$(id -u)" != "0" ]; then + echo "!! If you are not running the script as root, your user should be in the docker group to use it. (sudo gpasswd -a $USER docker) !!" +fi + +echo "Repository: $REPOSITORY_NAME" +echo "Repository Description: $REPOSITORY_DESCRIPTION" + + +# Creating the building script on-the-fly +cat >$createrepo <$entropysrv < +community-mode = enable +weak-package-files = disable +database-format = bz2 +# sync-speed-limit = +# server-basic-languages = en_US C +# disabled-eapis = 1,2 +# expiration-based-scope = disable +# nonfree-packages-directory-support = disable +rss-feed = disable +changelog = disable +rss-name = packages.rss +rss-base-url = http://packages.sabayon.org/?quicksearch= +rss-website-url = http://www.sabayon.org/ +max-rss-entries = 10000 +# max-rss-light-entries = 100 +rss-light-name = updates.rss +managing-editor = +broken-reverse-deps = disable +default-repository = $REPOSITORY_NAME +repository=$REPOSITORY_NAME|$REPOSITORY_DESCRIPTION|file:///sabayon/artifacts +EOF + +docker run --rm --entrypoint /bin/bash -e REPOSITORY=$REPOSITORY_NAME -e EDITOR=cat -e LC_ALL=en_US.UTF-8 \ + -v "$ROOT"/entropy_artifacts:/sabayon/artifacts \ + -v $createrepo:/sabayon/bin/create_repo.sh \ + -v $entropysrv:/etc/entropy/server.conf \ + -v "$ROOT"/portage_artifacts:/usr/portage/packages \ + -ti sabayon/builder-amd64 /sabayon/bin/create_repo.sh || true +rm -rf $createrepo +rm -rf $entropysrv diff --git a/sabayondevkit-functions.sh b/sabayondevkit-functions.sh new file mode 100644 index 0000000..6cef70d --- /dev/null +++ b/sabayondevkit-functions.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +build() { +local SEARCH=$1 +local EMERGE_DEFAULT_ARGS=${2:---accept-properties=-interactive --verbose --oneshot --nospinner --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg} +for i in $(EIX_LIMIT=0 eix --only-names --pure-packages "$SEARCH/" | xargs echo | uniq); + do + emerge $EMERGE_DEFAULT_ARGS -o $i && \ + emerge $EMERGE_DEFAULT_ARGS $i + done +} + +build_installed() { + local SEARCH=$1 + local EMERGE_DEFAULT_ARGS=${2:---accept-properties=-interactive --newuse --changed-use --update --verbose --oneshot --nospinner --quiet-build=y --quiet-fail --fail-clean=y --complete-graph --buildpkg} + + for i in $(EIX_LIMIT=0 eix -I --only-names --pure-packages "$SEARCH/" | xargs echo | uniq); + do + #emerge $EMERGE_DEFAULT_ARGS -o $i && \ + emerge $EMERGE_DEFAULT_ARGS $i + done +} + +build_all_availables() { +for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq); + do + build "$i/" + done +} + +rebuild_all() { +for i in $(cat /usr/portage/profiles/categories | xargs echo | uniq); + do + build_installed "$i/" + done +} + +build_sync() { + emerge-webrsync || exit 1 + emerge --sync + layman -S + eix-update || exit 1 + pushd /opt/sabayon-build/ + git stash + git fetch --all + git checkout master + git reset --heard origin/master + popd +}