From a5b08b3dfbf47376e6fb967c06aa9ce7cd1d1328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C5=82awomir=20Nizio?= Date: Mon, 21 May 2012 18:32:23 +0200 Subject: [PATCH] [scripts] add file listing generator, bug 3314 --- scripts/gen_html/daily.tmpl | 197 +++++++++++++++++ scripts/gen_html/gen.sh | 72 ++++++ scripts/gen_html/main.tmpl | 147 +++++++++++++ scripts/gen_html/style.css | 28 +++ scripts/gen_html/templ+abc.pl | 398 ++++++++++++++++++++++++++++++++++ 5 files changed, 842 insertions(+) create mode 100644 scripts/gen_html/daily.tmpl create mode 100755 scripts/gen_html/gen.sh create mode 100644 scripts/gen_html/main.tmpl create mode 100644 scripts/gen_html/style.css create mode 100755 scripts/gen_html/templ+abc.pl diff --git a/scripts/gen_html/daily.tmpl b/scripts/gen_html/daily.tmpl new file mode 100644 index 0000000..f779cfc --- /dev/null +++ b/scripts/gen_html/daily.tmpl @@ -0,0 +1,197 @@ + + + + + Sabayon Linux + + + + +

+ skip to download links +

+ +

+Sabayon comes in different flavours to choose.
+To distinguish between them, a suffix is appended to file names.
+Here's a short description. More details can be found in this thread on Sabayon forum. +

+

+This directory contains "daily" images, which are generated automatically every few days (as the name suggests), and features even more editions to choose from.
+

+

+Note: "daily" snapshots are generated automatically.
+They are not as much tested as the "normal" releases.
+Even though they should usually work, consider downloading
+one of the official releases if you are new to Sabayon. +

+

hilights:

+
+
G
+
This version contains the well known GNOME 3.
+
K
+
This version comes with KDE SC, another popular desktop environment.
+
Xfce
+
Contains Xfce - lightweight yet very functional and friendly desktop.
+
LXDE
+
With LXDE. Small, pretty and usable.
+
E17
+
With Enlightenment DR17.
+
+ +

Core releases:

+ +
+
SpinBase
+
It's a very minimal environment that can be used for many different purposes: didactical, home server deployment, and even for custom Sabayon ISO images creation, using our tool called Molecule. Any Sabayon release we make is based on SpinBase.
+
CoreCDX
+
It's geared towards very minimal graphical environment setup, no fancy tools, browsers, whatever, just Fluxbox and command line.
+
ServerBase
+
It's very similar to SpinBase, but powered by a server-optimized Linux kernel (package: sys-kernel/linux-server).
+
OpenVZ
+
It's our official OpenVZ template.
+
+ +

+All of them have a smaller footprint making them fit into a single CD, or USB memory sticks.
+SpinBase and ServerBase are provided with a very minimal Anaconda Installer and CoreCDX should be preferred for non-standard filesystem/partition layouts. +

+ +

the rest:

+
+
ForensicsXfce
+
Editions for forensics related tasks. More.
+
ARM images
+
Images for ARM7. Those without "rootfs" in their name are for BeagleBone/BeagleBoard devices and the other ones should work on any ARM7 device.
+
+ +

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
NameArchitectureDownload linkmd5sumPackage listSizeDate
"> + + + + + + ">download + + ??? + + + + ">md5sum + + N/A + + + + ">package list + + N/A + + + + + + N/A + + + + + + N/A + +
+

All dates are in day-month-year format.

+ + + + + + + + + + + + + + + + + + + + + + +
NameDownload linkmd5sumSizeDate
"> + + + + ">download () + + ??? + + + + ">md5sum + + N/A + + + + + + N/A + + + + + + N/A + +
+

All dates are in day-month-year format.

+ + +

+ Other:
+ + "> + + +
+ +

+ + + + diff --git a/scripts/gen_html/gen.sh b/scripts/gen_html/gen.sh new file mode 100755 index 0000000..53cef65 --- /dev/null +++ b/scripts/gen_html/gen.sh @@ -0,0 +1,72 @@ +#!/bin/sh + +script_dir=${0%/*} + +script=$script_dir/templ+abc.pl + +warn() { + echo "$*" >&2 +} + +if [ ! -f "$script" ]; then + warn "error, file templ+abc.pl cannot be found in directory '$script_dir'" + warn "(PWD=$PWD)" + exit 2 +fi + +iso_dir_prefix="/sabayon/rsync/rsync.sabayon.org/iso" +main_dir=$iso_dir_prefix +dailies_dir=$iso_dir_prefix/daily +unset iso_dir_prefix + +for dir in "$main_dir" "$dailies_dir"; do + if [ ! -d "$dir" ]; then + warn "'$dir' doesn't exist or is not a directory" + exit 2 + fi +done + +common_args="--skip .. --skip main.html --skip daily.html --skip style.css" + +tmpfile="$script_dir/tmp.html" + +if ! rm -f "$tmpfile"; then + warn "rm -f $tmpfile failed!" + exit 2 +fi + +# main +if ! "$script" $common_args \ + main --template main.tmpl --dir "$main_dir" \ + > "$tmpfile"; then + warn "script for main releases failed" + rm -f "$tmpfile" + exit 2 +fi + +if ! mv "$tmpfile" "$main_dir/main.html"; then + warn "mv $tmpfile $main_dir/main.html failed" + rm -f "$tmpfile" + exit 2 +fi + +# dailies +if ! "$script" $common_args \ + daily --template daily.tmpl --dir "$dailies_dir" \ + > "$tmpfile"; then + warn "script for daily releases failed" + rm -f "$tmpfile" + exit 2 +fi + +if ! mv "$tmpfile" "$dailies_dir/daily.html"; then + warn "mv $tmpfile $dailies_dir/daily.html failed" + rm -f "$tmpfile" + exit 2 +fi + +# CSS +if ! cp "$script_dir/style.css" "$main_dir/"; then + warn "cp $script_dir/style.css $main_dir/ failed" + exit 2 +fi diff --git a/scripts/gen_html/main.tmpl b/scripts/gen_html/main.tmpl new file mode 100644 index 0000000..3674973 --- /dev/null +++ b/scripts/gen_html/main.tmpl @@ -0,0 +1,147 @@ + + + + + Sabayon Linux + + + + +

+ skip to download links +

+ +

+Sabayon comes in different flavours to choose.
+To distinguish between them, a suffix is appended to file names.
+Here's a short description. More details can be found in this thread on Sabayon forum. +

+ +

main releases:

+
+
G
+
This version contains the well known GNOME 3.
+
K
+
This version comes with KDE SC, another popular desktop environment.
+
Xfce
+
Contains Xfce - lightweight yet very functional and friendly desktop.
+
+ + + +

additionally, these Core releases are available:

+ +
+
SpinBase
+
It's a very minimal environment that can be used for many different purposes: didactical, home server deployment, and even for custom Sabayon ISO images creation, using our tool called Molecule. Any Sabayon release we make is based on SpinBase.
+
CoreCDX
+
It's geared towards very minimal graphical environment setup, no fancy tools, browsers, whatever, just Fluxbox and command line.
+
ServerBase
+
It's very similar to SpinBase, but powered by a server-optimized Linux kernel (package: sys-kernel/linux-server).
+ +
+ +

+All of them have a smaller footprint making them fit into a single CD, or USB memory sticks.
+SpinBase and ServerBase are provided with a very minimal Anaconda Installer and CoreCDX should be preferred for non-standard filesystem/partition layouts. +

+ +

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameArchitectureDownload linkTorrentmd5sumPackage listSizeDate
"> + + + + + + ">download + + ??? + + + + ">torrent + + N/A + + + + ">md5sum + + N/A + + + + ">package list + + N/A + + + + + + N/A + + + + + + N/A + +
+

All dates are in day-month-year format.

+ + +

+ Other:
+ + "> + + +
+ +

+ + + + diff --git a/scripts/gen_html/style.css b/scripts/gen_html/style.css new file mode 100644 index 0000000..c17fcdb --- /dev/null +++ b/scripts/gen_html/style.css @@ -0,0 +1,28 @@ +body +{ +font-size: 14px; +font-family: Verdana,sans-serif,"Times New Roman" +} + +table +{ +border: 1px solid #A0522D; +border-collapse:collapse; +width: 80%; +} + +td +{ +border:1px solid #A0522D; +} + +th +{ +background-color: #90EE90; +color:black; +} + +td:hover +{ +background-color: #CACACA; +} diff --git a/scripts/gen_html/templ+abc.pl b/scripts/gen_html/templ+abc.pl new file mode 100755 index 0000000..d493c89 --- /dev/null +++ b/scripts/gen_html/templ+abc.pl @@ -0,0 +1,398 @@ +#!/usr/bin/env perl + +# Copyright (C) 2012, Enlik + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +use warnings; +use strict; +use Getopt::Long; +use HTML::Template; +use 5.010; + +# This makes accessing edition's data more clear. +# Class EditionItem stores URL, size and date for an edition for one +# architecture, for one file type (which means that one EditionType +# is needed for an .iso, another one for a .md5sum file and so on). +# We don't use, for example, size for .md5 files, but just in case it changes... +{ + package EditionItem; + my ($href, $size, $date); + sub new { + my $class = shift; + my %self = @_; + for my $item (qw(url size date)) { + die "EditionItem::new(): $item not defined" + unless defined $self{$item} + } + bless \%self, $class; + } + sub url { $_[0]->{url} } + sub size { $_[0]->{size} } + sub date { $_[0]->{date} } + sub toString { + my $self = shift; + my ($url, $size, $date) = ($self->{url}, $self->{size}, $self->{date}); + return "$url (size = $size, date = $date)"; + } +} + +# $sab{edition}->{arch}->{type} = EditionItem +# type is one of: ".md5", ".pkglist", ".torrent", "" + +my %sab; +my @oth; +my $tmpl; + +sub print_other { + return unless @oth; + my @others_loop_data; + for my $item (@oth) { + push @others_loop_data, { url => $item->[0], name => $item->[1] } + } + $tmpl->param(others_loop => \@others_loop_data); +} + +# Adds elements to template which will be printed. +# $regex is a regular expression to specify which arches to select; +# $negate - if true, $regex will be negated; +# $editions_loop_arch - name of the loop in template to use; +# $extra_fields - what to pass to template besides for "" and ".md5", +# for example ".pkglist" to pass .pkglist for arches/editions which have +# such files; supported types: ".pkglist" and ".torrent" +sub print_items { + my ($regex, $negate, $editions_loop_arch, $extra_fields) = @_; + my $match; + my %extra_fields = map { $_ => 1 } @{ $extra_fields }; + my @editions_loop_data = (); + + for my $edition (sort keys %sab) { + my $displayed_edition = 0; + my @arches_loop_data = (); + + for my $arch (sort keys %{ $sab{$edition} }) { + $match = $arch =~ $regex; + $match = !$match if $negate; + next unless $match; + # these are EditionItem objects + my $ISO = $sab{$edition}->{$arch}->{""}; + my $md5 = $sab{$edition}->{$arch}->{".md5"}; + my $pkglist = $sab{$edition}->{$arch}->{".pkglist"} + if $extra_fields{".pkglist"}; + my $torrent = $sab{$edition}->{$arch}->{".torrent"} + if $extra_fields{".torrent"}; + + for my $item (qw(.torrent .pkglist)) { + if (defined $sab{$edition}->{$arch}->{$item} + and not $extra_fields{$item}) { + warn "Warning: file type $item not specified to print, ", + "but it is available for $edition -> $arch: ", + $sab{$edition}->{$arch}->{$item}->url + } + } + + my %arches_loop_row; # fresh hash for new arch + + $arches_loop_row{name} = $edition; + $arches_loop_row{arch} = $arch; + + # any item is counted, even if there's no ISO but only md5sum + # not too pretty, but fast enough + $arches_loop_row{arches_per_edition} = 0; + for my $arch (keys %{ $sab{$edition} }) { + my $match = $arch =~ $regex; + $match = !$match if $negate; + $arches_loop_row{arches_per_edition}++ if $match; + } + + # Some of them (ISO, md5, pkglist) may be not defined, + # depending on available file types. + # If x is defined, then x->url, x->size and x->time are defined. + # Note: size and date for non-ISO-like files are discarded here. + if (defined $ISO) { + $arches_loop_row{mainfile_url} = $ISO->url; + $arches_loop_row{mainfile_size} = $ISO->size; + $arches_loop_row{mainfile_date} = $ISO->date; + } + if (defined $md5) { + $arches_loop_row{md5_url} = $md5->url; + #$arches_loop_row{md5_size} = $md5->size; + #$arches_loop_row{md5_date} = $md5->date; + } + if (defined $pkglist) { + $arches_loop_row{pkglist_url} = $pkglist->url; + #$arches_loop_row{pkglist_size} = $pkglist->size; + #$arches_loop_row{pkglist_date} = $pkglist->date; + } + if (defined $torrent) { + $arches_loop_row{torrent_url} = $torrent->url; + } + + # push data (name, download links, ...) for an architecture + push @arches_loop_data, \%arches_loop_row; + } + # say "

" if $displayed_edition; + # with nested loops this template thingy goes a little obscure... + push @editions_loop_data, { arches_loop => \@arches_loop_data } + if (@arches_loop_data); + } + $tmpl->param($editions_loop_arch => \@editions_loop_data); +} + +sub add_item { + my ($edition, $arch, $type, $href, $size, $mdate) = @_; + die "add_item: args!\n" unless @_ == 6; + + if (defined $sab{$edition}->{$arch}->{$type}) { + my $ei = $sab{$edition}->{$arch}->{$type}; + warn "Warning: already defined! ($edition, $arch, $type), item = ", + $ei->toString, "\n"; + } + + $sab{$edition}->{$arch}->{$type} = + EditionItem->new (url => $href, size => $size, date => $mdate); +} + +# Parse file name and add using add_item. First argument is file name; second +# argument is prefix (see help for --prefix). +# The rest will be passed to add_item (it's used to pass additional attributes +# gathered by caller). +sub parse_entry { + my $href = shift; + my $prefix = shift; + my @extra_args = @_; + my $href_full = $href; + my $type_ext = ""; + + # find file type + for my $ext (qw(.md5 .pkglist .torrent)) { + if ($href =~ /\Q${ext}\E$/) { + $type_ext = $ext; + $href =~ s/\Q${ext}\E$//; # part without "well known extension" + last; + } + } + + my $href_link = $prefix . $href_full; + + my $re_pref = "Sabayon_Linux"; + my $re_arch = "(?[^_]+)"; + # If numeration schema changes, look at $re_ver. :) + # $re_ver part can't be too loose because it would match something else + # than release type... + my $re_ver = "(?DAILY|[0-9]+)"; + + my $fmt = sub { + my ($ver, $ed) = @_; + $ver eq "DAILY" ? $ed : qq{Sabayon $ver "$ed"}; + }; + + # Sabayon_Linux_DAILY_amd64_E17.iso + if ($href =~ /^${re_pref}_${re_ver}_${re_arch}_(?[^_]+)\.iso$/) { + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ($ed, $+{arch}, $type_ext, $href_link, @extra_args); + } + + # Sabayon_Linux_CoreCDX_DAILY_amd64.iso + elsif ($href =~ /^${re_pref}_(?[^_]+)_${re_ver}_${re_arch}\.iso$/) { + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ($ed, $+{arch}, $type_ext, $href_link, @extra_args); + } + + # Sabayon_Linux_SpinBase_DAILY_x86_openvz.tar.gz + elsif ($href =~ /^${re_pref}_(?[^_]+)_${re_ver}_${re_arch}_(?.+)\.tar\.gz$/) { + # there's no such non-daily rel., but just in case; same for a few others + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ("$ed ($+{ed_misc}; .tar.gz)", $+{arch}, + $type_ext, $href_link, @extra_args); + } + + # Sabayon_Linux_DAILY_armv7a_(misc).img.xz + elsif ($href =~ /^${re_pref}_${re_ver}_${re_arch}_(?.+)\.img\.xz$/) { + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ("$ed (.img.xz)", $+{arch}, + $type_ext, $href_link, @extra_args); + } + + # Sabayon_Linux_DAILY_armv7a_BeagleBone_Base_4GB.img.rootfs.tar.xz + elsif ($href =~ /^${re_pref}_${re_ver}_${re_arch}_(?.+)(?\.img\.[^.]+\.tar\.xz)$/) { + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ("$ed ($+{ext})", $+{arch}, + $type_ext, $href_link, @extra_args); + } + + # Sabayon_Linux_DAILY_armv7a_BeagleBone_Base_16GB.img + elsif ($href =~ /^${re_pref}_${re_ver}_${re_arch}_(?.+)\.img$/) { + my $ed = $fmt->($+{ver}, $+{ed}); + add_item ("$ed (.img)", $+{arch}, + $type_ext, $href_link, @extra_args); + } + + else { + push @oth, [ $href_link, $href_full ] + } +} + +# Read directory and call subroutines to build data structures. +sub generate { + if (1) { + my ($dir_to_open, $prefix, $opt_skip) = @_; + if (not defined $dir_to_open or not defined $prefix) { + die "generate: need 2 arguments!"; + } + + opendir (my $dh, $dir_to_open) + or die "Cannot open directory $dir_to_open: $!\n"; + my %opt_skip = map { $_ => 1 } @{ $opt_skip }; + + while (my $item = readdir ($dh)) { + # next if $item eq ".."; # we want "." I think + next if $opt_skip{$item}; + + my $item_path = $dir_to_open . "/" . $item; + my ($size, $mdate); + my @st = stat($item_path); + if (@st) { + ($size, $mdate) = @st[7,9]; + $size = "" if (-d _); + } + else { + say STDERR "Cannot stat $item_path: $!"; + $mdate = $size = "???"; + } + + # make pretty $size if it's a number + if ($size =~ /^[0-9]/) { + my @units = qw(B KiB MiB); + my $level = 0; + while ($size >= 2000) { + last if $level == $#units; + $size /= 1024; + $level++; + } + if ($level > 0 and $size != int($size)) { + $size = int (100*$size + 0.5) / 100; + $size = sprintf ("%.2f", $size); + } + $size .= " " . $units[$level]; + } + + # make pretty $mdate if it's not unknown + if ($mdate =~ /^[0-9]/) { + my ($day, $mth, $year) = (localtime($mdate)) [3,4,5]; + $mth++; + $year += 1900; + $mth = "0" . $mth if $mth < 10; + $mdate = "$day.$mth.$year"; + } + + parse_entry ($item, $prefix, $size, $mdate); + } + closedir ($dh); + } + # read from downloaded listing (wget .../iso), good for testing purposes + # ignores any argument + else { + my $filename = "daily.html"; + open my $fh, "<", $filename or die "Cannot open file $filename! $!\n"; + + while (my $line = <$fh>) { + if ($line =~ /\/) { + parse_entry ($1, "", "(size)", "(date)"); + } + } + close $fh; + } +} + +my ($arg_type, $opt_help, $opt_template, $opt_dir, $opt_prefix) = ("") x 5; +my @opt_skip; +my $opts = GetOptions( + "template=s" => \$opt_template, + "help" => \$opt_help, + "dir=s" => \$opt_dir, + "prefix=s" => \$opt_prefix, + "skip=s" => \@opt_skip, +); + +if ($opt_help) { + print <.tmpl +--dir - directory with file entries to process (.ISO files and so on; + defaults to current directory) +--prefix - prefix appended to each link; for example if you want to place + resulting HTML document under /var/www and you use --dir /var/www/s/iso, + you may need to specify --prefix s/iso/ so download links point to the + proper location (note the trailing slash) +--skip - file name to skip (especially useful to skip unwanted files listed + under "Others" section); this option can be specified multiple times +--help - shows this help +END + exit 0; +} + +if (@ARGV != 1) { + die "You must provide one argument TYPE. See --help.\n" +} +$arg_type = shift; + +unless ($arg_type ~~ [ qw(main daily) ]) { + die "Invalid argument TYPE. See --help.\n" +} + +my $template_name; +if ($opt_template ne "") { + $template_name = $opt_template; +} +else { + $template_name = $arg_type . ".tmpl"; +} + +# Open template itself to avoid 2-argument open() idiocy. +open my $tmpl_fh, "<", $template_name + or die "Cannot open file $template_name: $!\n"; +$tmpl = HTML::Template->new(filehandle => $tmpl_fh, + loop_context_vars => 1); +close $tmpl_fh; + +generate($opt_dir || ".", $opt_prefix, \@opt_skip); + +if ($arg_type eq "daily") { + # x86 & amd86 first + print_items ( qr/amd64|x86/, 0, "intel_editions_loop", [ qw(.pkglist) ] ); + print_items ( qr/amd64|x86/, 1, "nonintel_editions_loop", [] ); +} +else { + print_items ( qr/.*/, 0, "intel_editions_loop", [ qw(.pkglist .torrent) ] ); +} +print_other (); + +# get rid of those stupid blank lines (leaving /^$/ ones which may be used +# to prettify output) for some small penalty cost +for my $line (split /\n/,$tmpl->output) { + say "$line" unless $line =~ /^\s+$/; +}