From 416ab8a37f1b07dc9f6c0fb3ff1a8ff2036510b5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmelzer Date: Thu, 2 Sep 2010 17:50:49 +0200 Subject: change dir structure --- tools/README.busybox | 10 -- tools/mksquashfs | Bin 88416 -> 0 bytes tools/mksquashfs-filter-patch | 201 ------------------------ tools/uclibc-wrapper | 28 ---- tools/updateDistroInfo.suse | 347 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 347 insertions(+), 239 deletions(-) delete mode 100644 tools/README.busybox delete mode 100755 tools/mksquashfs delete mode 100644 tools/mksquashfs-filter-patch delete mode 100755 tools/uclibc-wrapper create mode 100755 tools/updateDistroInfo.suse (limited to 'tools') diff --git a/tools/README.busybox b/tools/README.busybox deleted file mode 100644 index 9c12a65b..00000000 --- a/tools/README.busybox +++ /dev/null @@ -1,10 +0,0 @@ -How busybox (version 1.13.2) is created: - -1. Get our current sources from openslx-src-tools/trunk/busybox -2. bind mount the busybox folder to the correspoding uclibc-build-environment - folder (see http://www.openslx.org/wiki/openslx/GentooUclibcChroot for - how to create such an environment) -3. chroot into the uclibc-build-environment -4. cd into the bind-mounted busybox folder -5. make oldconfig and adjust the configuration if required. -6. build busybox (make) diff --git a/tools/mksquashfs b/tools/mksquashfs deleted file mode 100755 index e65a35a5..00000000 Binary files a/tools/mksquashfs and /dev/null differ diff --git a/tools/mksquashfs-filter-patch b/tools/mksquashfs-filter-patch deleted file mode 100644 index 5cb4ba24..00000000 --- a/tools/mksquashfs-filter-patch +++ /dev/null @@ -1,201 +0,0 @@ ---- squashfs-tools/mksquashfs.c 2007-01-16 09:24:40.000000000 +0100 -+++ squashfs-tools-vito/mksquashfs.c 2007-01-17 08:34:17.000000000 +0100 -@@ -44,6 +44,7 @@ - #include - #include - #include -+#include - - #ifndef linux - #define __BYTE_ORDER BYTE_ORDER -@@ -162,15 +163,19 @@ - struct file_info *dupl[65536]; - int dup_files = 0; - --/* list of exclude dirs/files */ --struct exclude_info { -+/* list of include/exclude files/regexes */ -+struct filter_info { - dev_t st_dev; - ino_t st_ino; -+ regex_t *preg; -+ int action; - }; - --#define EXCLUDE_SIZE 8192 --int exclude = 0; --struct exclude_info *exclude_paths = NULL; -+#define EXCLUDE 0 -+#define INCLUDE 1 -+#define FILTER_SIZE 8192 -+int filter_count = 0; -+struct filter_info *filters = NULL; - int excluded(char *filename, struct stat *buf); - - /* fragment block data structures */ -@@ -2606,20 +2611,32 @@ - { - int i; - -- for(i = 0; i < exclude; i++) -- if((exclude_paths[i].st_dev == buf->st_dev) && (exclude_paths[i].st_ino == buf->st_ino)) -- return TRUE; -+ for(i = 0; i < filter_count; i++) { -+ struct filter_info f = filters[i]; -+ -+ if(f.preg == NULL) { -+ if((f.st_dev == buf->st_dev) && (f.st_ino == buf->st_ino)) -+ return TRUE; -+ } else if(regexec(f.preg, filename, (size_t)0, NULL, 0) == 0) -+ if(f.action == EXCLUDE) -+ return TRUE; -+ else -+ return FALSE; -+ } -+ - return FALSE; - } - - --#define ADD_ENTRY(buf) \ -- if(exclude % EXCLUDE_SIZE == 0) {\ -- if((exclude_paths = (struct exclude_info *) realloc(exclude_paths, (exclude + EXCLUDE_SIZE) * sizeof(struct exclude_info))) == NULL)\ -- BAD_ERROR("Out of memory in exclude dir/file table\n");\ -+#define ADD_ENTRY(b, a, p) \ -+ if(filter_count % FILTER_SIZE == 0) {\ -+ if((filters = (struct filter_info *) realloc(filters, (filter_count + FILTER_SIZE) * sizeof(struct filter_info))) == NULL)\ -+ BAD_ERROR("Out of memory in the include/exclude file/regex table\n");\ - }\ -- exclude_paths[exclude].st_dev = buf.st_dev;\ -- exclude_paths[exclude++].st_ino = buf.st_ino; -+ filters[filter_count].action = a;\ -+ filters[filter_count].preg = p;\ -+ filters[filter_count].st_dev = b.st_dev;\ -+ filters[filter_count++].st_ino = b.st_ino; - int add_exclude(char *path) - { - int i; -@@ -2632,7 +2649,7 @@ - perror(buffer); - return TRUE; - } -- ADD_ENTRY(buf); -+ ADD_ENTRY(buf, EXCLUDE, NULL); - return TRUE; - } - -@@ -2645,11 +2662,47 @@ - } - continue; - } -- ADD_ENTRY(buf); -+ ADD_ENTRY(buf, EXCLUDE, NULL); - } - return TRUE; - } - -+void add_pattern(char *pattern) -+{ -+ struct stat dummy; -+ int action = -1; -+ -+ if(pattern[0] == '#') -+ return; -+ -+ if(strlen(pattern) < 3) -+ goto bad_pattern; -+ -+ if(strncmp(pattern, "+ ", 2) == 0) -+ action = INCLUDE; -+ -+ if(strncmp(pattern, "- ", 2) == 0) -+ action = EXCLUDE; -+ -+ if(action != -1) { -+ regex_t *preg = (regex_t*)malloc(sizeof(regex_t)); -+ char message[1024]; -+ int error = regcomp(preg, pattern+2, REG_EXTENDED|REG_NOSUB); -+ regerror(error, preg, message, 1024); -+ -+ if(error) { -+ fprintf(stderr, "Ignoring invalid regex \"%s\" (%s)\n", pattern+2, message); -+ return; -+ } -+ -+ ADD_ENTRY(dummy, action, preg); -+ return; -+ } -+ -+bad_pattern: -+ fprintf(stderr, "Ignoring wrong filter pattern \"%s\"\n", pattern); -+} -+ - - void add_old_root_entry(char *name, squashfs_inode inode, int inode_number, int type) - { -@@ -2849,6 +2902,11 @@ - ERROR("%s: -ef missing filename\n", argv[0]); - exit(1); - } -+ } else if(strcmp(argv[i], "-ff") == 0) { -+ if(++i == argc) { -+ ERROR("%s: -ff missing filename\n", argv[0]); -+ exit(1); -+ } - } else if(strcmp(argv[i], "-no-duplicates") == 0) - duplicate_checking = FALSE; - -@@ -2991,6 +3049,10 @@ - ERROR("\t\t\tfile or dir with priority per line. Priority -32768 to\n"); - ERROR("\t\t\t32767, default priority 0\n"); - ERROR("-ef \tlist of exclude dirs/files. One per line\n"); -+ ERROR("-ff \tlist of include/exclude patterns. One per line.\n"); -+ ERROR("\t\t\tThe first pattern that matches a dir/file is applied.\n"); -+ ERROR("\t\t\tExclude patterns are written '+ regex', include patterns\n"); -+ ERROR("\t\t\tare written '- regex'. Comments start with #\n"); - exit(1); - } - } -@@ -3048,7 +3110,7 @@ - signal(SIGINT, sighandler2); - } - -- /* process the exclude files - must be done afer destination file has been possibly created */ -+ /* process the include/exclude files/patterns - must be done afer destination file has been possibly created */ - for(i = source + 2; i < argc; i++) - if(strcmp(argv[i], "-ef") == 0) { - FILE *fd; -@@ -3060,6 +3122,16 @@ - while(fscanf(fd, "%16384[^\n]\n", filename) != EOF) - add_exclude(filename); - fclose(fd); -+ } else if(strcmp(argv[i], "-ff") == 0) { -+ FILE *fd; -+ char pattern[16385]; -+ if((fd = fopen(argv[++i], "r")) == NULL) { -+ perror("Could not open filter file..."); -+ EXIT_MKSQUASHFS(); -+ } -+ while(fscanf(fd, "%16384[^\n]\n", pattern) != EOF) -+ add_pattern(pattern); -+ fclose(fd); - } else if(strcmp(argv[i], "-e") == 0) - break; - else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-sort") == 0) -@@ -3073,14 +3145,14 @@ - while(i < argc && add_exclude(argv[i++])); - } - -- /* process the sort files - must be done afer the exclude files */ -+ /* process the sort files - must be done afer processing the include/exclude files/patterns */ - for(i = source + 2; i < argc; i++) - if(strcmp(argv[i], "-sort") == 0) { - read_sort_file(argv[++i], source, source_path); - sorted ++; - } else if(strcmp(argv[i], "-e") == 0) - break; -- else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-ef") == 0) -+ else if(strcmp(argv[i], "-b") == 0 || strcmp(argv[i], "-root-becomes") == 0 || strcmp(argv[i], "-ef") == 0 || strcmp(argv[i], "-ff") == 0) - i++; - - initialise_threads(); diff --git a/tools/uclibc-wrapper b/tools/uclibc-wrapper deleted file mode 100755 index 08934f32..00000000 --- a/tools/uclibc-wrapper +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# Copyright (c) 2008..2009 - OpenSLX GmbH -# -# This program is free software distributed under the GPL version 2. -# See http://openslx.org/COPYING -# -# If you have any feedback please consult http://openslx.org/feedback and -# send your feedback to feedback@openslx.org -# -# General information about OpenSLX can be found under http://openslx.org -# -# Wrapper script for OpenSLX environments used with server and client stage4 - -if [ -d /opt/openslx/share/uclib-rootfs ]; then - SLX_UCLIB_PATH=/opt/openslx/share/uclib-rootfs -elif [ -d /opt/openslx/uclib-rootfs ]; then - SLX_UCLIB_PATH=/opt/openslx/uclib-rootfs -else - echo "Can't find OpenSLX Environment." && exit 1 -fi - -[ -f /lib/ld-uClibc.so.0 ] || \ - ln -sf $SLX_PATH/uclib-rootfs/lib/ld-uClibc.so.0 /lib/ld-uClibc.so.0 - -PATH=$SLX_UCLIB_PATH/bin:$SLX_UCLIB_PATH/usr/bin:$SLX_UCLIB_PATH/sbin:\ -$SLX_UCLIB_PATH/usr/sbin -LD_LIBRARY_PATH=$SLX_UCLIB_PATH/lib:$SLX_UCLIB_PATH/usr/lib -LD_LIBRARY_PATH=$LD_LIBRARY_PATH PATH=$PATH PS1="#" $@ diff --git a/tools/updateDistroInfo.suse b/tools/updateDistroInfo.suse new file mode 100755 index 00000000..b877ab55 --- /dev/null +++ b/tools/updateDistroInfo.suse @@ -0,0 +1,347 @@ +#!/usr/bin/perl +# Copyright (c) 2009 - OpenSLX GmbH +# +# This program/file is free software distributed under the GPL version 2. +# See http://openslx.org/COPYING +# +# If you have any feedback please consult http://openslx.org/feedback and +# send your feedback to feedback@openslx.org +# +# General information about OpenSLX can be found at http://openslx.org +# + +use strict; +use warnings; + +use threads ('yield', + 'stack_size' => 64*4096, + 'exit' => 'threads_only', + 'stringify'); +use Time::HiRes qw(sleep); +use Switch; +use Net::HTTP; +use Net::FTP; +#use Data::Dumper; + + +# Configuration ############################################################### + +my @supportedVersions = ( + { + ver => + '10.3', + source => + 'http://mirrors.opensuse.org/list/all.html', + supporedArch => + 'i586 x86_64', + packageKeysLocation => + 'http://download.opensuse.org/distribution/10.3/repo/oss/' + }, + { + ver => + '11.0', + source => + 'http://mirrors.opensuse.org/list/11.0.html', + supporedArch => + 'i586 x86_64', + packageKeysLocation => + 'http://download.opensuse.org/distribution/11.0/repo/oss/' + }, + { + ver => + '11.1', + source => + 'http://mirrors.opensuse.org/list/11.1.html', + supporedArch => + 'i586 x86_64', + packageKeysLocation => + 'http://download.opensuse.org/distribution/11.1/repo/oss/' + } + { + ver => + '11.2', + source => + 'http://mirrors.opensuse.org/list/11.2.html', + supporedArch => + 'i586 x86_64', + packageKeysLocation => + 'http://download.opensuse.org/distribution/11.2/repo/oss/' + } + { + ver => + '11.3', + source => + 'http://mirrors.opensuse.org/list/11.3.html', + supporedArch => + 'i586 x86_64', + packageKeysLocation => + 'http://download.opensuse.org/distribution/11.3/repo/oss/' + } +); + +my %sourceLists = ( + 'base' => + { + path => 'distribution/#ver#/repo/oss', + archpath => 'distribution/#ver#/repo/oss/suse/#arch#' + }, + 'base_non-oss' => + { + path => 'distribution/#ver#/repo/non-oss', + archpath => 'distribution/#ver#/repo/oss/suse/#arch#' + }, + 'base_update' => + { + path => 'update/#ver#', + archpath => 'update/#ver#/rpm/#arch#' + }, +); + +############################################################################### + +my $cmd; +my $out = {}; +my $statistics = {}; + +# set autoflush +$| = 1; + +sub checkHttpMirror { + my $checkurl = shift; + my $mirrorurl = shift; + my $outfile = shift; + + $SIG{'KILL'} = sub { threads->exit(); }; + + $checkurl = m/http:\/\/([^\/]*)(\/.*)/; + my $s = Net::HTTP->new(Host => $1, Timeout => 10, Debug => 0) or return ( "status:1" ); + $s->write_request(GET => $2, 'User-Agent' => "Mozilla/5.0"); + + my($code, $mess, %h) = $s->read_response_headers; + + if ($code == 200) { + return ( "status:0", "mirror:$mirrorurl", "outfile:$outfile" ); + } else { + return ( "status:2" ); + } +} + +sub checkFtpMirror { + my $checkurl = shift; + my $mirrorurl = shift; + my $outfile = shift; + + $SIG{'KILL'} = sub { threads->exit(); }; + + $checkurl = m/ftp:\/\/([^\/]*)(\/.*)/; + my $ftp = Net::FTP->new(Host => $1, Timeout => 10, Debug => 0) or return ( "status:1" ); + $ftp->login("anonymous",'-anonymous@') or return ( "status:2" ); + $ftp->cwd($2) or return ( "status:3" ); + $ftp->quit; + + return ( "status:0", "mirror:$mirrorurl", "outfile:$outfile" ); +} + +sub cleanupThreads { + + my @joinable = threads->list(threads::joinable); + while (@joinable) { + my @ret = shift(@joinable)->join(); + my $response = {}; + + while (my $param = shift(@ret)) { + $param =~ m/^([^:]*):(.*)$/; + $response->{$1} = $2; + } + + switch ($response->{status}) { + case 0 { + push(@{$out->{$response->{outfile}}}, + $response->{mirror}); + print "o" + } + else { + print "x"; + $statistics->{errors}++; + } + } + #print "j"; + } + + my $running = threads->list(threads::running); + return $running; +} + +my $startTime = time(); +my $endTime; + +my $oldtime; +if ( -e "$ENV{'HOME'}/.update-suse-distro-info" ) { + $oldtime = qx(cat $ENV{'HOME'}/.update-suse-distro-info); +} + +# cleanup +system ("rm -rf out"); + +print "\n ** OpenSLX distro-info Updater - OpenSUSE **\n\n"; +print "Notice: running the checks takes some time, so it's enough time for a coffee\n break ;)"; + +if ($oldtime) { + my ($min, $sec) = split(/:/, $oldtime); + printf (" - last run took %i min %i sec ..\n", $min, $sec); +} else { + print "\n"; +} + +while (my $version = shift(@supportedVersions)) { + print "\nopenSUSE $version->{ver} \n"; + + print "fetching mirror list..\n"; + system("wget -q -O suse-$version->{ver} $version->{source}"); + + print "extracting mirrors.. \n"; + $cmd = "grep -P -e \"(HT|F)TP\" suse-$version->{ver} | "; + $cmd .= "sed -e \"s/^[^\\\"]*\\\"//\" -e \"s/\\\".*\$//\" >"; + $cmd .= "suse-$version->{ver}-mirrors && rm suse-$version->{ver}"; + system $cmd; + + my @sa = split(/ /,$version->{supporedArch}); + + # empty out + $out = {}; + $statistics = {}; + + + while (my $arch = shift(@sa)) { + $arch = "_$arch"; + # if we have ix86 arch name is not used.. + $arch =~ s/_i.?86//; + $cmd = "mkdir -p out/suse-$version->{ver}$arch/mirrors"; + $cmd .= " out/suse-$version->{ver}$arch/trusted-package-keys"; + system ($cmd); + } + + print "check mirrors .. \n"; + my $running; + $statistics->{numMirrors} = qx(cat suse-$version->{ver}-mirrors | wc -l); + open FILE, "< suse-$version->{ver}-mirrors", or die "couldn't open file: $!"; + while () { + chomp; + my $mirror = $_; + my $skipmirror = 0; + @sa = split(/ /,$version->{supporedArch}); + while (my $arch = shift(@sa)) { + while (my($type, $parameters) = each %sourceLists) { + my $checkpath = $parameters->{archpath}; + $checkpath =~ s/#ver#/$version->{ver}/; + $checkpath =~ s/#arch#/$arch/; + + my $mirrorpath = $parameters->{path}; + $mirrorpath =~ s/#ver#/$version->{ver}/; + + my $local_arch = "_$arch"; + # if we have ix86 arch name is not used.. + $local_arch =~ s/_i.?86//; + + my @running = threads->list(threads::running); + my $numRunning = scalar @running; + $running = cleanupThreads(); + + while( $running > 50 ) { sleep 0.1; $running = cleanupThreads();} + if ($mirror =~ m/^http:/) { + threads->create({context => 'list'}, + 'checkHttpMirror', + "$mirror$checkpath", + "$mirror$mirrorpath", + "suse-$version->{ver}$local_arch/mirrors/$type" + ); + } + if ($mirror =~ m/^ftp:/) { + threads->create({context => 'list'}, + 'checkFtpMirror', + "$mirror$checkpath", + "$mirror$mirrorpath", + "suse-$version->{ver}$local_arch/mirrors/$type" + ); + } + } + } + #print "$mirror \n"; + } + close FILE; + system ("rm suse-$version->{ver}-mirrors"); + + $running = cleanupThreads(); + printf ("\n-- waiting for %i unfinished check(s) ..\n", $running); + my $counter = 120; + my $exit = 0; + while( $exit == 0 ) { + sleep 0.5; + $running = cleanupThreads(); + $counter--; + if ( $counter < 1 && $running > 0 ) { + printf ("\n-- %i check(s) still running.. \n", $running); + print ("\n-- cleanup remaining check(s).."); + my @runningthreads = threads->list(); + while (@runningthreads) { + shift(@runningthreads)->kill('KILL')->detach(); + } + $exit = 1; + } + } + + print "\nfinished.\n\n"; + print "writing active mirrors to file.\n"; + + while (my($file, $mirrorlist) = each %{$out}) { + open(OUTFILE, ">>", "out/$file") or die ("something went wrong"); + while (@$mirrorlist) { + #print Dumper($mirrorlist); + $statistics->{files}->{$file}++; + print OUTFILE shift(@$mirrorlist); + print OUTFILE "\n"; + } + close(OUTFILE); + } + + print "\nStatistics:\n"; + printf ("-- checked %i mirrors \n", $statistics->{numMirrors}); + while (my($f, $num) = each(%{$statistics->{files}})) { + printf ("-- found %i entries for: %s \n", $num, $f); + } + + print "\nGet package keys\n"; + system("wget -q -O index http://download.opensuse.org/distribution/$version->{ver}/repo/oss/"); + my @files = qx(cat index | grep -P -e "(gpg-pubkey|pubring)"); + while (@files) { + my $file = shift(@files); + $file =~ m/href=\"([^\"]*)\"/; + my $cmd = "wget -q -O out/suse-$version->{ver}/trusted-package-keys/$1 "; + $cmd .= "http://download.opensuse.org/distribution/$version->{ver}/repo/oss/$1"; + system ("sh", "-c", $cmd); + } + system ("rm index"); + + @sa = split(/ /,$version->{supporedArch}); + while (@sa) { + my $arch = shift(@sa); + my $cmd = "cp out/suse-$version->{ver}/trusted-package-keys/* "; + $cmd .= " out/suse-$version->{ver}_$arch/trusted-package-keys/"; + $arch =~ m/i.?86/ or system ($cmd); + } +} + +$endTime = time(); +my $totalTime = $endTime - $startTime; + +system("cd out && tar cf ../distro-info.suse.tar * && cd .."); +system("rm -rf out"); + +printf ("\nUpdated distro info in %i min %i sec \n", $totalTime/60, $totalTime%60); +printf ("Data written to distro-info.suse.tar, go to /lib/distro-info \n and extract it."); + +open(FH, ">", "$ENV{'HOME'}/.update-suse-distro-info") or die ($!); +printf FH ("%i:%i", $totalTime/60, $totalTime%60); +close(FH); + +exit 1; -- cgit v1.2.3-55-g7522