* OSSetup install now uses the uclib environment

 * minor fix for suse install

 * added script for fetching suse mirrors


git-svn-id: http://svn.openslx.org/svn/openslx/openslx/trunk@2699 95ad53e4-c205-0410-b2fa-d234c58c8868

3 files changed, 363 insertions, 15 deletions

diff --git a/installer/OpenSLX/OSSetup/Distro/Suse.pm b/installer/OpenSLX/OSSetup/Distro/Suse.pm
index e41e919a..e8e1c7c8 100644
--- a/installer/OpenSLX/OSSetup/Distro/Suse.pm
+++ b/installer/OpenSLX/OSSetup/Distro/Suse.pm
@@ -43,7 +43,7 @@ sub initialize
     my $defaultMetaPackager = 'zypper';
     if ($self->{'engine'}->{'distro-name'} =~ m{-([^-]+)$}) {
         my $distroVersion = 0.0 + $1;
-        if ($distroVersion < 10.2) {
+        if ($distroVersion <= 10.2) {
             $defaultMetaPackager = 'smart';
         }
     }
diff --git a/installer/OpenSLX/OSSetup/Engine.pm b/installer/OpenSLX/OSSetup/Engine.pm
index 14843f0b..821df9a0 100644
--- a/installer/OpenSLX/OSSetup/Engine.pm
+++ b/installer/OpenSLX/OSSetup/Engine.pm
@@ -51,9 +51,9 @@ use vars qw(%supportedDistros);
     'suse-10.2_x86_64'  => 'clone,install,update,shell',
     'suse-10.3'         => 'clone,update,shell',
     'suse-10.3_x86_64'  => 'clone,update,shell',
-    'suse-11.0'         => 'clone,update,shell',
+    'suse-11.0'         => 'clone,update,shell,install',
     'suse-11.0_x86_64'  => 'clone,update,shell',
-    'suse-11.1'         => 'clone,update,shell',
+    'suse-11.1'         => 'clone,update,shell,install',
     'suse-11.1_x86_64'  => 'clone,update,shell',
     'scilin-4.7'        => 'clone,update,shell',
     'scilin-5.1'        => 'clone,update,shell',
@@ -1047,10 +1047,11 @@ sub _touchVendorOS
 sub _copyUclibcRootfs
 {
     my $self = shift;
+    my $targetRoot = shift || $self->{'vendor-os-path'};
 
     vlog(0, _tr("copying uclibc-rootfs into vendor-OS ...\n"));
 
-    my $targetRoot = $self->{'vendor-os-path'};
+    # my $targetRoot = $self->{'vendor-os-path'};
     my $target     = "$targetRoot/opt/openslx/uclib-rootfs";
 
     if (system("mkdir -p $target")) {
@@ -1073,6 +1074,8 @@ sub _copyUclibcRootfs
     my $rsyncCmd
         = "rsync -aq --delete-excluded --exclude-from=- $uclibcRootfs/ $target";
     vlog(2, "executing: $rsyncCmd\n");
+    # if we're doing a fresh install we need to create /lib first
+    system("mkdir -p $targetRoot/lib");
     # link uClibc from the uclib-rootfs to /lib to make LD_PRELOAD=... working
     my $uClibCmd = "ln -sf /opt/openslx/uclib-rootfs/lib/ld-uClibc.so.0";
     $uClibCmd .= " $targetRoot/lib/ld-uClibc.so.0";
@@ -1169,7 +1172,7 @@ try_next_url:
         foreach my $file (split '\s+', $fileVariantStr) {
             my $basefile = basename($file);
             vlog(2, "fetching <$file>...");
-            if (slxsystem("wget", "-c", "-O", "$basefile", "$url/$file") == 0) {
+            if (system("wget", "-c", "-O", "$basefile", "$url/$file") == 0) {
                 $foundFile = $basefile;
                 last;
             }
@@ -1262,13 +1265,29 @@ sub _setupStage1A
             $stage1cDir, $!);
     }
 
-    $self->_stage1A_createBusyboxEnvironment();
+    #$self->_stage1A_createBusyboxEnvironment();
+    $self->_stage1A_setupUclibcEnvironment();
     $self->_stage1A_copyPrerequiredFiles();
     $self->_stage1A_copyTrustedPackageKeys();
     $self->_stage1A_createRequiredFiles();
     return;
 }
 
+sub _stage1A_setupUclibcEnvironment
+{
+    my $self = shift;
+    $self->_copyUclibcRootfs("$self->{stage1aDir}/$self->{stage1bSubdir}");
+    my $source = "$self->{stage1bSubdir}/opt/openslx/uclib-rootfs";
+    my $target = "$self->{stage1aDir}";
+    system("ln -sf $source/bin $target/bin");
+    system("ln -sf $source/lib $target/lib");
+    system("ln -sf $source/usr $target/usr");
+    $self->_stage1A_setupResolver();
+    
+    return;
+}
+
+
 sub _stage1A_createBusyboxEnvironment
 {
     my $self = shift;
@@ -1321,9 +1340,9 @@ sub _stage1A_setupResolver
     copyFile('/etc/resolv.conf', "$self->{stage1aDir}/etc");
     copyFile('/etc/nsswitch.conf', "$self->{stage1aDir}/etc");
     spitFile("$self->{stage1aDir}/etc/hosts", "127.0.0.1 localhost\n");
-    copyFile("$libcFolder/libresolv*",    "$self->{stage1aDir}$libcFolder");
-    copyFile("$libcFolder/libnss_dns*",   "$self->{stage1aDir}$libcFolder");
-    copyFile("$libcFolder/libnss_files*", "$self->{stage1aDir}$libcFolder");
+    #copyFile("$libcFolder/libresolv*",    "$self->{stage1aDir}$libcFolder");
+    #copyFile("$libcFolder/libnss_dns*",   "$self->{stage1aDir}$libcFolder");
+    #copyFile("$libcFolder/libnss_files*", "$self->{stage1aDir}$libcFolder");
 
     my $stage1cDir 
         = "$self->{'stage1aDir'}/$self->{'stage1bSubdir'}/$self->{'stage1cSubdir'}";
@@ -1350,6 +1369,7 @@ sub _stage1A_copyPrerequiredFiles
             $stage1cDir, $!
         );
     }
+    vlog(2, "fix pre-required files...");
     $self->{distro}->fixPrerequiredFiles($stage1cDir);
     return;
 }
@@ -1523,12 +1543,13 @@ sub _stage1C_cleanupBasicVendorOS
             $self->{'vendor-os-path'}, $!
         );
     }
-    if (slxsystem("rm -rf $self->{stage1aDir}")) {
-        die _tr(
-            "unable to remove temporary folder '%s' (%s)\n",
-            $self->{stage1aDir}, $!
-        );
-    }
+    die();
+    #if (slxsystem("rm -rf $self->{stage1aDir}")) {
+    #    die _tr(
+    #        "unable to remove temporary folder '%s' (%s)\n",
+    #        $self->{stage1aDir}, $!
+    #    );
+    #}
     return;
 }
 
diff --git a/systemtools/updateDistroInfo.suse b/systemtools/updateDistroInfo.suse
new file mode 100755
index 00000000..025f102d
--- /dev/null
+++ b/systemtools/updateDistroInfo.suse
@@ -0,0 +1,327 @@
+#!/usr/bin/perl
+# Copyright (c) 2009 - OpenSLX GmbH
+#
+# This program/file is free software distributed under the GPL version 2.
+# See http://openslx.org/COPYING
+#
+# If you have any feedback please consult http://openslx.org/feedback and
+# send your feedback to feedback@openslx.org
+#
+# General information about OpenSLX can be found at http://openslx.org
+#
+
+use strict;
+use warnings;
+
+use threads ('yield',
+             'stack_size' => 64*4096,
+             'exit' => 'threads_only',
+             'stringify');
+use Time::HiRes qw(sleep);
+use Switch;
+use Net::HTTP;
+use Net::FTP;
+#use Data::Dumper;
+
+
+# Configuration ###############################################################
+
+my @supportedVersions = (
+    {
+        ver => 
+            '10.3',
+        source => 
+            'http://mirrors.opensuse.org/list/all.html',
+        supporedArch => 
+            'i586 x86_64',
+        packageKeysLocation => 
+            'http://download.opensuse.org/distribution/10.3/repo/oss/'
+    },
+    {
+        ver => 
+            '11.0',
+        source => 
+            'http://mirrors.opensuse.org/list/11.0.html',
+        supporedArch => 
+            'i586 x86_64',
+        packageKeysLocation => 
+            'http://download.opensuse.org/distribution/11.0/repo/oss/'
+    },
+    {
+        ver => 
+            '11.1',
+        source => 
+            'http://mirrors.opensuse.org/list/11.1.html',
+        supporedArch => 
+            'i586 x86_64',
+        packageKeysLocation => 
+            'http://download.opensuse.org/distribution/11.1/repo/oss/'
+    }
+);
+
+my %sourceLists = (
+    'base' =>
+         {
+             path => 'distribution/#ver#/repo/oss',
+             archpath => 'distribution/#ver#/repo/oss/suse/#arch#'
+         },
+    'base_non-oss' =>
+         {
+             path => 'distribution/#ver#/repo/non-oss',
+             archpath => 'distribution/#ver#/repo/oss/suse/#arch#'
+         },
+    'base_update' =>
+         {
+             path => 'update/#ver#',
+             archpath => 'update/#ver#/rpm/#arch#'
+         },
+);
+
+###############################################################################
+
+my $cmd; 
+my $out = {};
+my $statistics = {};
+
+# set autoflush
+$| = 1;
+
+sub checkHttpMirror {
+    my $checkurl = shift;
+    my $mirrorurl = shift;
+    my $outfile = shift;
+
+    $SIG{'KILL'} = sub { threads->exit(); };
+
+    $checkurl = m/http:\/\/([^\/]*)(\/.*)/;
+    my $s = Net::HTTP->new(Host => $1, Timeout => 10, Debug => 0) or return ( "status:1" );
+    $s->write_request(GET => $2, 'User-Agent' => "Mozilla/5.0");
+
+    my($code, $mess, %h) = $s->read_response_headers;
+
+    if ($code == 200) {
+        return ( "status:0", "mirror:$mirrorurl", "outfile:$outfile" );
+    } else {
+        return ( "status:2" );
+    }
+}
+
+sub checkFtpMirror {
+    my $checkurl = shift;
+    my $mirrorurl = shift;
+    my $outfile = shift;
+
+    $SIG{'KILL'} = sub { threads->exit(); };
+
+    $checkurl = m/ftp:\/\/([^\/]*)(\/.*)/;
+    my $ftp = Net::FTP->new(Host => $1, Timeout => 10, Debug => 0) or return ( "status:1" );
+    $ftp->login("anonymous",'-anonymous@') or return ( "status:2" );
+    $ftp->cwd($2) or return ( "status:3" );
+    $ftp->quit;
+
+    return ( "status:0", "mirror:$mirrorurl", "outfile:$outfile" );
+}
+
+sub cleanupThreads {
+
+    my @joinable = threads->list(threads::joinable);
+    while (@joinable) {
+        my @ret = shift(@joinable)->join();
+        my $response = {};
+
+        while (my $param = shift(@ret)) {
+            $param =~ m/^([^:]*):(.*)$/;
+            $response->{$1} = $2;
+        }
+
+        switch ($response->{status}) {
+            case 0 {
+                 push(@{$out->{$response->{outfile}}},
+                 $response->{mirror});
+                 print "o"
+            }
+            else { 
+                 print "x";
+                 $statistics->{errors}++;
+            }
+         }
+         #print "j";
+    }
+
+    my $running = threads->list(threads::running);
+    return $running;
+}
+
+my $startTime = time();
+my $endTime;
+
+my $oldtime;
+if ( -e "$ENV{'HOME'}/.update-suse-distro-info" ) {
+   $oldtime = qx(cat $ENV{'HOME'}/.update-suse-distro-info);
+}
+
+# cleanup
+system ("rm -rf out");
+
+print "\n              ** OpenSLX distro-info Updater - OpenSUSE **\n\n";
+print "Notice: running the checks takes some time, so it's enough time for a coffee\n        break ;)";
+
+if ($oldtime) {
+    my ($min, $sec) = split(/:/, $oldtime);
+    printf (" - last run took %i min %i sec ..\n", $min, $sec);
+} else {
+    print "\n";
+}
+
+while (my $version = shift(@supportedVersions)) {
+    print "\nopenSUSE $version->{ver} \n";
+
+    print "fetching mirror list..\n";
+    system("wget -q -O suse-$version->{ver} $version->{source}");
+
+    print "extracting mirrors.. \n";
+    $cmd  = "grep -P -e \"<a href=.*>(HT|F)TP</a>\" suse-$version->{ver} | ";
+    $cmd .= "sed -e \"s/^[^\\\"]*\\\"//\" -e \"s/\\\".*\$//\" >";
+    $cmd .= "suse-$version->{ver}-mirrors && rm suse-$version->{ver}";
+    system $cmd;
+
+    my @sa = split(/ /,$version->{supporedArch});
+    
+    # empty out
+    $out = {};
+    $statistics = {};
+
+
+    while (my $arch = shift(@sa)) {
+        $arch = "_$arch";
+        # if we have ix86 arch name is not used..
+        $arch =~ s/_i.?86//; 
+        $cmd  = "mkdir -p out/suse-$version->{ver}$arch/mirrors";
+        $cmd .= "         out/suse-$version->{ver}$arch/trusted-package-keys";
+        system ($cmd);
+    }
+
+    print "check mirrors .. \n";
+    my $running;
+    $statistics->{numMirrors} = qx(cat suse-$version->{ver}-mirrors | wc -l);
+    open FILE, "< suse-$version->{ver}-mirrors", or die "couldn't open file: $!";
+    while (<FILE>) {
+        chomp;
+        my $mirror = $_;
+        my $skipmirror = 0;
+        @sa = split(/ /,$version->{supporedArch});
+        while (my $arch = shift(@sa)) {
+             while (my($type, $parameters) = each %sourceLists) {
+                 my $checkpath = $parameters->{archpath};
+                    $checkpath =~ s/#ver#/$version->{ver}/;
+                    $checkpath =~ s/#arch#/$arch/;
+
+                 my $mirrorpath = $parameters->{path};
+                    $mirrorpath =~ s/#ver#/$version->{ver}/;
+
+                 my $local_arch = "_$arch";
+                 # if we have ix86 arch name is not used..
+                 $local_arch =~ s/_i.?86//;
+
+                 my @running = threads->list(threads::running);
+                 my $numRunning = scalar @running;
+                 $running = cleanupThreads();
+
+                 while( $running > 50 ) { sleep 0.1; $running = cleanupThreads();}
+                 if ($mirror =~ m/^http:/) {
+                     threads->create({context => 'list'},
+                                     'checkHttpMirror', 
+                                     "$mirror$checkpath", 
+                                     "$mirror$mirrorpath", 
+                                     "suse-$version->{ver}$local_arch/mirrors/$type"
+                                    );
+                 }
+                 if ($mirror =~ m/^ftp:/) {
+                     threads->create({context => 'list'},
+                                     'checkFtpMirror', 
+                                     "$mirror$checkpath", 
+                                     "$mirror$mirrorpath", 
+                                     "suse-$version->{ver}$local_arch/mirrors/$type"
+                                    );
+                 }
+             }
+        }
+        #print "$mirror \n";
+    }
+    close FILE;
+    system ("rm suse-$version->{ver}-mirrors");
+
+    $running = cleanupThreads();
+    printf ("\n-- waiting for %i unfinished check(s) ..\n", $running);
+    my $counter = 120;
+    my $exit = 0;
+    while( $exit == 0 ) { 
+        sleep 0.5; 
+        $running = cleanupThreads();
+        $counter--;
+        if ( $counter < 1 && $running > 0 ) {
+            printf ("\n-- %i check(s) still running.. \n", $running);
+            print ("\n-- cleanup remaining check(s)..");
+            my @runningthreads = threads->list();
+            while (@runningthreads) {
+                shift(@runningthreads)->kill('KILL')->detach();
+            }
+            $exit = 1;
+        }
+    }
+    
+    print "\nfinished.\n\n";
+    print "writing active mirrors to file.\n";
+
+    while (my($file, $mirrorlist) = each %{$out}) {
+        open(OUTFILE, ">>", "out/$file") or die ("something went wrong");
+        while (@$mirrorlist) {
+            #print Dumper($mirrorlist);
+            $statistics->{files}->{$file}++;
+            print OUTFILE shift(@$mirrorlist);
+            print OUTFILE "\n";
+        }
+        close(OUTFILE);
+    }
+
+    print "\nStatistics:\n";
+    printf ("-- checked %i mirrors \n", $statistics->{numMirrors});
+    while (my($f, $num) = each(%{$statistics->{files}})) {
+        printf ("-- found %i entries for: %s \n", $num, $f);
+    }
+
+    print "\nGet package keys\n";
+    system("wget -q -O index http://download.opensuse.org/distribution/$version->{ver}/repo/oss/");
+    my @files = qx(cat index | grep -P -e "(gpg-pubkey|pubring)");
+    while (@files) {
+        my $file = shift(@files);
+        $file =~ m/href=\"([^\"]*)\"/;
+        my $cmd  = "wget -q -O out/suse-$version->{ver}/trusted-package-keys/$1 ";
+           $cmd .= "http://download.opensuse.org/distribution/$version->{ver}/repo/oss/$1";
+        system ("sh", "-c", $cmd);
+    }
+    system ("rm index");
+
+    @sa = split(/ /,$version->{supporedArch});
+    while (@sa) {
+        my $arch = shift(@sa);
+        my $cmd  = "cp out/suse-$version->{ver}/trusted-package-keys/* ";
+           $cmd .= "   out/suse-$version->{ver}_$arch/trusted-package-keys/";
+        $arch =~ m/i.?86/ or system ($cmd);
+    }
+}
+
+$endTime = time();
+my $totalTime = $endTime - $startTime;
+
+system("cd out && tar cf ../distro-info.suse.tar * && cd ..");
+system("rm -rf out");
+
+printf ("\nUpdated distro info in %i min %i sec \n", $totalTime/60, $totalTime%60);
+printf ("Data written to distro-info.suse.tar, go to <oslx-source>/lib/distro-info \n and extract it.");
+
+open(FH, ">", "$ENV{'HOME'}/.update-suse-distro-info") or die ($!);
+printf FH ("%i:%i", $totalTime/60, $totalTime%60);
+close(FH);
+
+exit 1;